diff --git a/sys/compat/linux/linux_socket.c b/sys/compat/linux/linux_socket.c index 36cffc979802..15431bf3127c 100644 --- a/sys/compat/linux/linux_socket.c +++ b/sys/compat/linux/linux_socket.c @@ -1,2755 +1,2755 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1995 Søren Schmidt * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET6 #include #include #endif #ifdef COMPAT_LINUX32 #include #include #include #else #include #include #endif #include #include #include #include #include #include #include _Static_assert(offsetof(struct l_ifreq, ifr_ifru) == offsetof(struct ifreq, ifr_ifru), "Linux ifreq members names should be equal to FreeeBSD"); _Static_assert(offsetof(struct l_ifreq, ifr_index) == offsetof(struct ifreq, ifr_index), "Linux ifreq members names should be equal to FreeeBSD"); _Static_assert(offsetof(struct l_ifreq, ifr_name) == offsetof(struct ifreq, ifr_name), "Linux ifreq members names should be equal to FreeeBSD"); #define SECURITY_CONTEXT_STRING "unconfined" static int linux_sendmsg_common(struct thread *, l_int, struct l_msghdr *, l_uint); static int linux_recvmsg_common(struct thread *, l_int, struct l_msghdr *, l_uint, struct msghdr *); static int linux_set_socket_flags(int, int *); #define SOL_NETLINK 270 static int linux_to_bsd_sockopt_level(int level) { if (level == LINUX_SOL_SOCKET) return (SOL_SOCKET); /* Remaining values are RFC-defined protocol numbers. */ return (level); } static int bsd_to_linux_sockopt_level(int level) { if (level == SOL_SOCKET) return (LINUX_SOL_SOCKET); return (level); } static int linux_to_bsd_ip_sockopt(int opt) { switch (opt) { /* known and translated sockopts */ case LINUX_IP_TOS: return (IP_TOS); case LINUX_IP_TTL: return (IP_TTL); case LINUX_IP_HDRINCL: return (IP_HDRINCL); case LINUX_IP_OPTIONS: return (IP_OPTIONS); case LINUX_IP_RECVOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_RECVOPTS"); return (IP_RECVOPTS); case LINUX_IP_RETOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_REETOPTS"); return (IP_RETOPTS); case LINUX_IP_RECVTTL: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_RECVTTL"); return (IP_RECVTTL); case LINUX_IP_RECVTOS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_RECVTOS"); return (IP_RECVTOS); case LINUX_IP_FREEBIND: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_FREEBIND"); return (IP_BINDANY); case LINUX_IP_IPSEC_POLICY: /* we have this option, but not documented in ip(4) manpage */ LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_IPSEC_POLICY"); return (IP_IPSEC_POLICY); case LINUX_IP_MINTTL: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_MINTTL"); return (IP_MINTTL); case LINUX_IP_MULTICAST_IF: return (IP_MULTICAST_IF); case LINUX_IP_MULTICAST_TTL: return (IP_MULTICAST_TTL); case LINUX_IP_MULTICAST_LOOP: return (IP_MULTICAST_LOOP); case LINUX_IP_ADD_MEMBERSHIP: return (IP_ADD_MEMBERSHIP); case LINUX_IP_DROP_MEMBERSHIP: return (IP_DROP_MEMBERSHIP); case LINUX_IP_UNBLOCK_SOURCE: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_UNBLOCK_SOURCE"); return (IP_UNBLOCK_SOURCE); case LINUX_IP_BLOCK_SOURCE: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_BLOCK_SOURCE"); return (IP_BLOCK_SOURCE); case LINUX_IP_ADD_SOURCE_MEMBERSHIP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_ADD_SOURCE_MEMBERSHIP"); return (IP_ADD_SOURCE_MEMBERSHIP); case LINUX_IP_DROP_SOURCE_MEMBERSHIP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_DROP_SOURCE_MEMBERSHIP"); return (IP_DROP_SOURCE_MEMBERSHIP); case LINUX_MCAST_JOIN_GROUP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_MCAST_JOIN_GROUP"); return (MCAST_JOIN_GROUP); case LINUX_MCAST_LEAVE_GROUP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_MCAST_LEAVE_GROUP"); return (MCAST_LEAVE_GROUP); case LINUX_MCAST_JOIN_SOURCE_GROUP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_MCAST_JOIN_SOURCE_GROUP"); return (MCAST_JOIN_SOURCE_GROUP); case LINUX_MCAST_LEAVE_SOURCE_GROUP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv4 socket option IP_MCAST_LEAVE_SOURCE_GROUP"); return (MCAST_LEAVE_SOURCE_GROUP); case LINUX_IP_RECVORIGDSTADDR: return (IP_RECVORIGDSTADDR); /* known but not implemented sockopts */ case LINUX_IP_ROUTER_ALERT: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_ROUTER_ALERT (%d), you can not do user-space routing from linux programs", opt); return (-2); case LINUX_IP_PKTINFO: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_PKTINFO (%d), you can not get extended packet info for datagram sockets in linux programs", opt); return (-2); case LINUX_IP_PKTOPTIONS: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_PKTOPTIONS (%d)", opt); return (-2); case LINUX_IP_MTU_DISCOVER: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_MTU_DISCOVER (%d), your linux program can not control path-MTU discovery", opt); return (-2); case LINUX_IP_RECVERR: /* needed by steam */ LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_RECVERR (%d), you can not get extended reliability info in linux programs", opt); return (-2); case LINUX_IP_MTU: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_MTU (%d), your linux program can not control the MTU on this socket", opt); return (-2); case LINUX_IP_XFRM_POLICY: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_XFRM_POLICY (%d)", opt); return (-2); case LINUX_IP_PASSSEC: /* needed by steam */ LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_PASSSEC (%d), you can not get IPSEC related credential information associated with this socket in linux programs -- if you do not use IPSEC, you can ignore this", opt); return (-2); case LINUX_IP_TRANSPARENT: /* IP_BINDANY or more? */ LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_TRANSPARENT (%d), you can not enable transparent proxying in linux programs -- note, IP_FREEBIND is supported, no idea if the FreeBSD IP_BINDANY is equivalent to the Linux IP_TRANSPARENT or not, any info is welcome", opt); return (-2); case LINUX_IP_NODEFRAG: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_NODEFRAG (%d)", opt); return (-2); case LINUX_IP_CHECKSUM: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_CHECKSUM (%d)", opt); return (-2); case LINUX_IP_BIND_ADDRESS_NO_PORT: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_BIND_ADDRESS_NO_PORT (%d)", opt); return (-2); case LINUX_IP_RECVFRAGSIZE: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_RECVFRAGSIZE (%d)", opt); return (-2); case LINUX_MCAST_MSFILTER: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_MCAST_MSFILTER (%d)", opt); return (-2); case LINUX_IP_MULTICAST_ALL: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_MULTICAST_ALL (%d), your linux program will not see all multicast groups joined by the entire system, only those the program joined itself on this socket", opt); return (-2); case LINUX_IP_UNICAST_IF: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv4 socket option IP_UNICAST_IF (%d)", opt); return (-2); /* unknown sockopts */ default: return (-1); } } static int linux_to_bsd_ip6_sockopt(int opt) { switch (opt) { /* known and translated sockopts */ case LINUX_IPV6_2292PKTINFO: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_2292PKTINFO"); return (IPV6_2292PKTINFO); case LINUX_IPV6_2292HOPOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_2292HOPOPTS"); return (IPV6_2292HOPOPTS); case LINUX_IPV6_2292DSTOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_2292DSTOPTS"); return (IPV6_2292DSTOPTS); case LINUX_IPV6_2292RTHDR: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_2292RTHDR"); return (IPV6_2292RTHDR); case LINUX_IPV6_2292PKTOPTIONS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_2292PKTOPTIONS"); return (IPV6_2292PKTOPTIONS); case LINUX_IPV6_CHECKSUM: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_CHECKSUM"); return (IPV6_CHECKSUM); case LINUX_IPV6_2292HOPLIMIT: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_2292HOPLIMIT"); return (IPV6_2292HOPLIMIT); case LINUX_IPV6_NEXTHOP: return (IPV6_NEXTHOP); case LINUX_IPV6_UNICAST_HOPS: return (IPV6_UNICAST_HOPS); case LINUX_IPV6_MULTICAST_IF: return (IPV6_MULTICAST_IF); case LINUX_IPV6_MULTICAST_HOPS: return (IPV6_MULTICAST_HOPS); case LINUX_IPV6_MULTICAST_LOOP: return (IPV6_MULTICAST_LOOP); case LINUX_IPV6_ADD_MEMBERSHIP: return (IPV6_JOIN_GROUP); case LINUX_IPV6_DROP_MEMBERSHIP: return (IPV6_LEAVE_GROUP); case LINUX_IPV6_V6ONLY: return (IPV6_V6ONLY); case LINUX_IPV6_IPSEC_POLICY: /* we have this option, but not documented in ip6(4) manpage */ LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_IPSEC_POLICY"); return (IPV6_IPSEC_POLICY); case LINUX_MCAST_JOIN_GROUP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_JOIN_GROUP"); return (IPV6_JOIN_GROUP); case LINUX_MCAST_LEAVE_GROUP: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_LEAVE_GROUP"); return (IPV6_LEAVE_GROUP); case LINUX_IPV6_RECVPKTINFO: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RECVPKTINFO"); return (IPV6_RECVPKTINFO); case LINUX_IPV6_PKTINFO: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_PKTINFO"); return (IPV6_PKTINFO); case LINUX_IPV6_RECVHOPLIMIT: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RECVHOPLIMIT"); return (IPV6_RECVHOPLIMIT); case LINUX_IPV6_HOPLIMIT: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_HOPLIMIT"); return (IPV6_HOPLIMIT); case LINUX_IPV6_RECVHOPOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RECVHOPOPTS"); return (IPV6_RECVHOPOPTS); case LINUX_IPV6_HOPOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_HOPOPTS"); return (IPV6_HOPOPTS); case LINUX_IPV6_RTHDRDSTOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RTHDRDSTOPTS"); return (IPV6_RTHDRDSTOPTS); case LINUX_IPV6_RECVRTHDR: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RECVRTHDR"); return (IPV6_RECVRTHDR); case LINUX_IPV6_RTHDR: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RTHDR"); return (IPV6_RTHDR); case LINUX_IPV6_RECVDSTOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RECVDSTOPTS"); return (IPV6_RECVDSTOPTS); case LINUX_IPV6_DSTOPTS: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_DSTOPTS"); return (IPV6_DSTOPTS); case LINUX_IPV6_RECVPATHMTU: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_RECVPATHMTU"); return (IPV6_RECVPATHMTU); case LINUX_IPV6_PATHMTU: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_PATHMTU"); return (IPV6_PATHMTU); case LINUX_IPV6_DONTFRAG: return (IPV6_DONTFRAG); case LINUX_IPV6_AUTOFLOWLABEL: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_AUTOFLOWLABEL"); return (IPV6_AUTOFLOWLABEL); case LINUX_IPV6_ORIGDSTADDR: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_ORIGDSTADDR"); return (IPV6_ORIGDSTADDR); case LINUX_IPV6_FREEBIND: LINUX_RATELIMIT_MSG_NOTTESTED("IPv6 socket option IPV6_FREEBIND"); return (IPV6_BINDANY); /* known but not implemented sockopts */ case LINUX_IPV6_ADDRFORM: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_ADDRFORM (%d), you linux program can not convert the socket to IPv4", opt); return (-2); case LINUX_IPV6_AUTHHDR: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_AUTHHDR (%d), your linux program can not get the authentication header info of IPv6 packets", opt); return (-2); case LINUX_IPV6_FLOWINFO: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_FLOWINFO (%d), your linux program can not get the flowid of IPv6 packets", opt); return (-2); case LINUX_IPV6_ROUTER_ALERT: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_ROUTER_ALERT (%d), you can not do user-space routing from linux programs", opt); return (-2); case LINUX_IPV6_MTU_DISCOVER: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_MTU_DISCOVER (%d), your linux program can not control path-MTU discovery", opt); return (-2); case LINUX_IPV6_MTU: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_MTU (%d), your linux program can not control the MTU on this socket", opt); return (-2); case LINUX_IPV6_JOIN_ANYCAST: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_JOIN_ANYCAST (%d)", opt); return (-2); case LINUX_IPV6_LEAVE_ANYCAST: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_LEAVE_ANYCAST (%d)", opt); return (-2); case LINUX_IPV6_MULTICAST_ALL: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_MULTICAST_ALL (%d)", opt); return (-2); case LINUX_IPV6_ROUTER_ALERT_ISOLATE: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_ROUTER_ALERT_ISOLATE (%d)", opt); return (-2); case LINUX_IPV6_FLOWLABEL_MGR: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_FLOWLABEL_MGR (%d)", opt); return (-2); case LINUX_IPV6_FLOWINFO_SEND: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_FLOWINFO_SEND (%d)", opt); return (-2); case LINUX_IPV6_XFRM_POLICY: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_XFRM_POLICY (%d)", opt); return (-2); case LINUX_IPV6_HDRINCL: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_HDRINCL (%d)", opt); return (-2); case LINUX_MCAST_BLOCK_SOURCE: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option MCAST_BLOCK_SOURCE (%d), your linux program may see more multicast stuff than it wants", opt); return (-2); case LINUX_MCAST_UNBLOCK_SOURCE: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option MCAST_UNBLOCK_SOURCE (%d), your linux program may not see all the multicast stuff it wants", opt); return (-2); case LINUX_MCAST_JOIN_SOURCE_GROUP: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option MCAST_JOIN_SOURCE_GROUP (%d), your linux program is not able to join a multicast source group", opt); return (-2); case LINUX_MCAST_LEAVE_SOURCE_GROUP: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option MCAST_LEAVE_SOURCE_GROUP (%d), your linux program is not able to leave a multicast source group -- but it was also not able to join one, so no issue", opt); return (-2); case LINUX_MCAST_MSFILTER: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option MCAST_MSFILTER (%d), your linux program can not manipulate the multicast filter, it may see more multicast data than it wants to see", opt); return (-2); case LINUX_IPV6_ADDR_PREFERENCES: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_ADDR_PREFERENCES (%d)", opt); return (-2); case LINUX_IPV6_MINHOPCOUNT: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_MINHOPCOUNT (%d)", opt); return (-2); case LINUX_IPV6_TRANSPARENT: /* IP_BINDANY or more? */ LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_TRANSPARENT (%d), you can not enable transparent proxying in linux programs -- note, IP_FREEBIND is supported, no idea if the FreeBSD IP_BINDANY is equivalent to the Linux IP_TRANSPARENT or not, any info is welcome", opt); return (-2); case LINUX_IPV6_UNICAST_IF: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_UNICAST_IF (%d)", opt); return (-2); case LINUX_IPV6_RECVFRAGSIZE: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_RECVFRAGSIZE (%d)", opt); return (-2); case LINUX_IPV6_RECVERR: LINUX_RATELIMIT_MSG_OPT1( "unsupported IPv6 socket option IPV6_RECVERR (%d), you can not get extended reliability info in linux programs", opt); return (-2); /* unknown sockopts */ default: return (-1); } } static int linux_to_bsd_so_sockopt(int opt) { switch (opt) { case LINUX_SO_DEBUG: return (SO_DEBUG); case LINUX_SO_REUSEADDR: return (SO_REUSEADDR); case LINUX_SO_TYPE: return (SO_TYPE); case LINUX_SO_ERROR: return (SO_ERROR); case LINUX_SO_DONTROUTE: return (SO_DONTROUTE); case LINUX_SO_BROADCAST: return (SO_BROADCAST); case LINUX_SO_SNDBUF: case LINUX_SO_SNDBUFFORCE: return (SO_SNDBUF); case LINUX_SO_RCVBUF: case LINUX_SO_RCVBUFFORCE: return (SO_RCVBUF); case LINUX_SO_KEEPALIVE: return (SO_KEEPALIVE); case LINUX_SO_OOBINLINE: return (SO_OOBINLINE); case LINUX_SO_LINGER: return (SO_LINGER); case LINUX_SO_REUSEPORT: return (SO_REUSEPORT_LB); case LINUX_SO_PASSCRED: return (LOCAL_CREDS_PERSISTENT); case LINUX_SO_PEERCRED: return (LOCAL_PEERCRED); case LINUX_SO_RCVLOWAT: return (SO_RCVLOWAT); case LINUX_SO_SNDLOWAT: return (SO_SNDLOWAT); case LINUX_SO_RCVTIMEO: return (SO_RCVTIMEO); case LINUX_SO_SNDTIMEO: return (SO_SNDTIMEO); case LINUX_SO_TIMESTAMPO: case LINUX_SO_TIMESTAMPN: return (SO_TIMESTAMP); case LINUX_SO_TIMESTAMPNSO: case LINUX_SO_TIMESTAMPNSN: return (SO_BINTIME); case LINUX_SO_ACCEPTCONN: return (SO_ACCEPTCONN); case LINUX_SO_PROTOCOL: return (SO_PROTOCOL); case LINUX_SO_DOMAIN: return (SO_DOMAIN); } return (-1); } static int linux_to_bsd_tcp_sockopt(int opt) { switch (opt) { case LINUX_TCP_NODELAY: return (TCP_NODELAY); case LINUX_TCP_MAXSEG: return (TCP_MAXSEG); case LINUX_TCP_CORK: return (TCP_NOPUSH); case LINUX_TCP_KEEPIDLE: return (TCP_KEEPIDLE); case LINUX_TCP_KEEPINTVL: return (TCP_KEEPINTVL); case LINUX_TCP_KEEPCNT: return (TCP_KEEPCNT); case LINUX_TCP_INFO: LINUX_RATELIMIT_MSG_OPT1( "unsupported TCP socket option TCP_INFO (%d)", opt); return (-2); case LINUX_TCP_MD5SIG: return (TCP_MD5SIG); } return (-1); } static int linux_to_bsd_msg_flags(int flags) { int ret_flags = 0; if (flags & LINUX_MSG_OOB) ret_flags |= MSG_OOB; if (flags & LINUX_MSG_PEEK) ret_flags |= MSG_PEEK; if (flags & LINUX_MSG_DONTROUTE) ret_flags |= MSG_DONTROUTE; if (flags & LINUX_MSG_CTRUNC) ret_flags |= MSG_CTRUNC; if (flags & LINUX_MSG_TRUNC) ret_flags |= MSG_TRUNC; if (flags & LINUX_MSG_DONTWAIT) ret_flags |= MSG_DONTWAIT; if (flags & LINUX_MSG_EOR) ret_flags |= MSG_EOR; if (flags & LINUX_MSG_WAITALL) ret_flags |= MSG_WAITALL; if (flags & LINUX_MSG_NOSIGNAL) ret_flags |= MSG_NOSIGNAL; if (flags & LINUX_MSG_PROXY) LINUX_RATELIMIT_MSG_OPT1("socket message flag MSG_PROXY (%d) not handled", LINUX_MSG_PROXY); if (flags & LINUX_MSG_FIN) LINUX_RATELIMIT_MSG_OPT1("socket message flag MSG_FIN (%d) not handled", LINUX_MSG_FIN); if (flags & LINUX_MSG_SYN) LINUX_RATELIMIT_MSG_OPT1("socket message flag MSG_SYN (%d) not handled", LINUX_MSG_SYN); if (flags & LINUX_MSG_CONFIRM) LINUX_RATELIMIT_MSG_OPT1("socket message flag MSG_CONFIRM (%d) not handled", LINUX_MSG_CONFIRM); if (flags & LINUX_MSG_RST) LINUX_RATELIMIT_MSG_OPT1("socket message flag MSG_RST (%d) not handled", LINUX_MSG_RST); if (flags & LINUX_MSG_ERRQUEUE) LINUX_RATELIMIT_MSG_OPT1("socket message flag MSG_ERRQUEUE (%d) not handled", LINUX_MSG_ERRQUEUE); return (ret_flags); } static int linux_to_bsd_cmsg_type(int cmsg_type) { switch (cmsg_type) { case LINUX_SCM_RIGHTS: return (SCM_RIGHTS); case LINUX_SCM_CREDENTIALS: return (SCM_CREDS); } return (-1); } static int bsd_to_linux_ip_cmsg_type(int cmsg_type) { switch (cmsg_type) { case IP_RECVORIGDSTADDR: return (LINUX_IP_RECVORIGDSTADDR); } return (-1); } static int bsd_to_linux_cmsg_type(struct proc *p, int cmsg_type, int cmsg_level) { struct linux_pemuldata *pem; if (cmsg_level == IPPROTO_IP) return (bsd_to_linux_ip_cmsg_type(cmsg_type)); if (cmsg_level != SOL_SOCKET) return (-1); pem = pem_find(p); switch (cmsg_type) { case SCM_RIGHTS: return (LINUX_SCM_RIGHTS); case SCM_CREDS: return (LINUX_SCM_CREDENTIALS); case SCM_CREDS2: return (LINUX_SCM_CREDENTIALS); case SCM_TIMESTAMP: return (pem->so_timestamp); case SCM_BINTIME: return (pem->so_timestampns); } return (-1); } static int linux_to_bsd_msghdr(struct msghdr *bhdr, const struct l_msghdr *lhdr) { if (lhdr->msg_controllen > INT_MAX) return (ENOBUFS); bhdr->msg_name = PTRIN(lhdr->msg_name); bhdr->msg_namelen = lhdr->msg_namelen; bhdr->msg_iov = PTRIN(lhdr->msg_iov); bhdr->msg_iovlen = lhdr->msg_iovlen; bhdr->msg_control = PTRIN(lhdr->msg_control); /* * msg_controllen is skipped since BSD and LINUX control messages * are potentially different sizes (e.g. the cred structure used * by SCM_CREDS is different between the two operating system). * * The caller can set it (if necessary) after converting all the * control messages. */ bhdr->msg_flags = linux_to_bsd_msg_flags(lhdr->msg_flags); return (0); } static int bsd_to_linux_msghdr(const struct msghdr *bhdr, struct l_msghdr *lhdr) { lhdr->msg_name = PTROUT(bhdr->msg_name); lhdr->msg_namelen = bhdr->msg_namelen; lhdr->msg_iov = PTROUT(bhdr->msg_iov); lhdr->msg_iovlen = bhdr->msg_iovlen; lhdr->msg_control = PTROUT(bhdr->msg_control); /* * msg_controllen is skipped since BSD and LINUX control messages * are potentially different sizes (e.g. the cred structure used * by SCM_CREDS is different between the two operating system). * * The caller can set it (if necessary) after converting all the * control messages. */ /* msg_flags skipped */ return (0); } static int linux_set_socket_flags(int lflags, int *flags) { if (lflags & ~(LINUX_SOCK_CLOEXEC | LINUX_SOCK_NONBLOCK)) return (EINVAL); if (lflags & LINUX_SOCK_NONBLOCK) *flags |= SOCK_NONBLOCK; if (lflags & LINUX_SOCK_CLOEXEC) *flags |= SOCK_CLOEXEC; return (0); } static int linux_copyout_sockaddr(const struct sockaddr *sa, void *uaddr, size_t len) { struct l_sockaddr *lsa; int error; error = bsd_to_linux_sockaddr(sa, &lsa, len); if (error != 0) return (error); error = copyout(lsa, uaddr, len); free(lsa, M_LINUX); return (error); } static int linux_sendit(struct thread *td, int s, struct msghdr *mp, int flags, struct mbuf *control, enum uio_seg segflg) { struct sockaddr *to; int error, len; if (mp->msg_name != NULL) { len = mp->msg_namelen; error = linux_to_bsd_sockaddr(mp->msg_name, &to, &len); if (error != 0) return (error); mp->msg_name = to; } else to = NULL; error = kern_sendit(td, s, mp, linux_to_bsd_msg_flags(flags), control, segflg); if (to) free(to, M_SONAME); return (error); } /* Return 0 if IP_HDRINCL is set for the given socket. */ static int linux_check_hdrincl(struct thread *td, int s) { int error, optval; socklen_t size_val; size_val = sizeof(optval); error = kern_getsockopt(td, s, IPPROTO_IP, IP_HDRINCL, &optval, UIO_SYSSPACE, &size_val); if (error != 0) return (error); return (optval == 0); } /* * Updated sendto() when IP_HDRINCL is set: * tweak endian-dependent fields in the IP packet. */ static int linux_sendto_hdrincl(struct thread *td, struct linux_sendto_args *linux_args) { /* * linux_ip_copysize defines how many bytes we should copy * from the beginning of the IP packet before we customize it for BSD. * It should include all the fields we modify (ip_len and ip_off). */ #define linux_ip_copysize 8 struct ip *packet; struct msghdr msg; struct iovec aiov[1]; int error; /* Check that the packet isn't too big or too small. */ if (linux_args->len < linux_ip_copysize || linux_args->len > IP_MAXPACKET) return (EINVAL); packet = (struct ip *)malloc(linux_args->len, M_LINUX, M_WAITOK); /* Make kernel copy of the packet to be sent */ if ((error = copyin(PTRIN(linux_args->msg), packet, linux_args->len))) goto goout; /* Convert fields from Linux to BSD raw IP socket format */ packet->ip_len = linux_args->len; packet->ip_off = ntohs(packet->ip_off); /* Prepare the msghdr and iovec structures describing the new packet */ msg.msg_name = PTRIN(linux_args->to); msg.msg_namelen = linux_args->tolen; msg.msg_iov = aiov; msg.msg_iovlen = 1; msg.msg_control = NULL; msg.msg_flags = 0; aiov[0].iov_base = (char *)packet; aiov[0].iov_len = linux_args->len; error = linux_sendit(td, linux_args->s, &msg, linux_args->flags, NULL, UIO_SYSSPACE); goout: free(packet, M_LINUX); return (error); } static const char *linux_netlink_names[] = { [LINUX_NETLINK_ROUTE] = "ROUTE", [LINUX_NETLINK_SOCK_DIAG] = "SOCK_DIAG", [LINUX_NETLINK_NFLOG] = "NFLOG", [LINUX_NETLINK_SELINUX] = "SELINUX", [LINUX_NETLINK_AUDIT] = "AUDIT", [LINUX_NETLINK_FIB_LOOKUP] = "FIB_LOOKUP", [LINUX_NETLINK_NETFILTER] = "NETFILTER", [LINUX_NETLINK_KOBJECT_UEVENT] = "KOBJECT_UEVENT", }; int linux_socket(struct thread *td, struct linux_socket_args *args) { int retval_socket, type; sa_family_t domain; type = args->type & LINUX_SOCK_TYPE_MASK; if (type < 0 || type > LINUX_SOCK_MAX) return (EINVAL); retval_socket = linux_set_socket_flags(args->type & ~LINUX_SOCK_TYPE_MASK, &type); if (retval_socket != 0) return (retval_socket); domain = linux_to_bsd_domain(args->domain); if (domain == AF_UNKNOWN) { /* Mask off SOCK_NONBLOCK / CLOEXEC for error messages. */ type = args->type & LINUX_SOCK_TYPE_MASK; if (args->domain == LINUX_AF_NETLINK && args->protocol == LINUX_NETLINK_AUDIT) { ; /* Do nothing, quietly. */ } else if (args->domain == LINUX_AF_NETLINK) { const char *nl_name; if (args->protocol >= 0 && args->protocol < nitems(linux_netlink_names)) nl_name = linux_netlink_names[args->protocol]; else nl_name = NULL; if (nl_name != NULL) linux_msg(curthread, "unsupported socket(AF_NETLINK, %d, " "NETLINK_%s)", type, nl_name); else linux_msg(curthread, "unsupported socket(AF_NETLINK, %d, %d)", type, args->protocol); } else { linux_msg(curthread, "unsupported socket domain %d, " "type %d, protocol %d", args->domain, type, args->protocol); } return (EAFNOSUPPORT); } retval_socket = kern_socket(td, domain, type, args->protocol); if (retval_socket) return (retval_socket); if (type == SOCK_RAW && (args->protocol == IPPROTO_RAW || args->protocol == 0) && domain == PF_INET) { /* It's a raw IP socket: set the IP_HDRINCL option. */ int hdrincl; hdrincl = 1; /* We ignore any error returned by kern_setsockopt() */ kern_setsockopt(td, td->td_retval[0], IPPROTO_IP, IP_HDRINCL, &hdrincl, UIO_SYSSPACE, sizeof(hdrincl)); } #ifdef INET6 /* * Linux AF_INET6 socket has IPV6_V6ONLY setsockopt set to 0 by default * and some apps depend on this. So, set V6ONLY to 0 for Linux apps. * For simplicity we do this unconditionally of the net.inet6.ip6.v6only * sysctl value. */ if (domain == PF_INET6) { int v6only; v6only = 0; /* We ignore any error returned by setsockopt() */ kern_setsockopt(td, td->td_retval[0], IPPROTO_IPV6, IPV6_V6ONLY, &v6only, UIO_SYSSPACE, sizeof(v6only)); } #endif return (retval_socket); } int linux_bind(struct thread *td, struct linux_bind_args *args) { struct sockaddr *sa; int error; error = linux_to_bsd_sockaddr(PTRIN(args->name), &sa, &args->namelen); if (error != 0) return (error); error = kern_bindat(td, AT_FDCWD, args->s, sa); free(sa, M_SONAME); /* XXX */ if (error == EADDRNOTAVAIL && args->namelen != sizeof(struct sockaddr_in)) return (EINVAL); return (error); } int linux_connect(struct thread *td, struct linux_connect_args *args) { struct socket *so; struct sockaddr *sa; struct file *fp; int error; error = linux_to_bsd_sockaddr(PTRIN(args->name), &sa, &args->namelen); if (error != 0) return (error); error = kern_connectat(td, AT_FDCWD, args->s, sa); free(sa, M_SONAME); if (error != EISCONN) return (error); /* * Linux doesn't return EISCONN the first time it occurs, * when on a non-blocking socket. Instead it returns the * error getsockopt(SOL_SOCKET, SO_ERROR) would return on BSD. */ error = getsock(td, args->s, &cap_connect_rights, &fp); if (error != 0) return (error); error = EISCONN; so = fp->f_data; if (atomic_load_int(&fp->f_flag) & FNONBLOCK) { SOCK_LOCK(so); if (so->so_emuldata == 0) error = so->so_error; so->so_emuldata = (void *)1; SOCK_UNLOCK(so); } fdrop(fp, td); return (error); } int linux_listen(struct thread *td, struct linux_listen_args *args) { return (kern_listen(td, args->s, args->backlog)); } static int linux_accept_common(struct thread *td, int s, l_uintptr_t addr, l_uintptr_t namelen, int flags) { struct sockaddr_storage ss = { .ss_len = sizeof(ss) }; struct file *fp, *fp1; struct socket *so; socklen_t len; int bflags, error, error1; bflags = 0; fp = NULL; error = linux_set_socket_flags(flags, &bflags); if (error != 0) return (error); if (PTRIN(addr) != NULL) { error = copyin(PTRIN(namelen), &len, sizeof(len)); if (error != 0) return (error); if (len < 0) return (EINVAL); } else len = 0; error = kern_accept4(td, s, (struct sockaddr *)&ss, bflags, &fp); /* * Translate errno values into ones used by Linux. */ if (error != 0) { /* * XXX. This is wrong, different sockaddr structures * have different sizes. */ switch (error) { case EFAULT: if (namelen != sizeof(struct sockaddr_in)) error = EINVAL; break; case EINVAL: error1 = getsock(td, s, &cap_accept_rights, &fp1); if (error1 != 0) { error = error1; break; } so = fp1->f_data; if (so->so_type == SOCK_DGRAM) error = EOPNOTSUPP; fdrop(fp1, td); break; } return (error); } if (PTRIN(addr) != NULL) { len = min(ss.ss_len, len); error = linux_copyout_sockaddr((struct sockaddr *)&ss, PTRIN(addr), len); if (error == 0) { len = ss.ss_len; error = copyout(&len, PTRIN(namelen), sizeof(len)); } if (error != 0) { fdclose(td, fp, td->td_retval[0]); td->td_retval[0] = 0; } } if (fp != NULL) fdrop(fp, td); return (error); } int linux_accept(struct thread *td, struct linux_accept_args *args) { return (linux_accept_common(td, args->s, args->addr, args->namelen, 0)); } int linux_accept4(struct thread *td, struct linux_accept4_args *args) { return (linux_accept_common(td, args->s, args->addr, args->namelen, args->flags)); } int linux_getsockname(struct thread *td, struct linux_getsockname_args *args) { struct sockaddr_storage ss = { .ss_len = sizeof(ss) }; socklen_t len; int error; error = copyin(PTRIN(args->namelen), &len, sizeof(len)); if (error != 0) return (error); error = kern_getsockname(td, args->s, (struct sockaddr *)&ss); if (error != 0) return (error); len = min(ss.ss_len, len); error = linux_copyout_sockaddr((struct sockaddr *)&ss, PTRIN(args->addr), len); if (error == 0) { len = ss.ss_len; error = copyout(&len, PTRIN(args->namelen), sizeof(len)); } return (error); } int linux_getpeername(struct thread *td, struct linux_getpeername_args *args) { struct sockaddr_storage ss = { .ss_len = sizeof(ss) }; socklen_t len; int error; error = copyin(PTRIN(args->namelen), &len, sizeof(len)); if (error != 0) return (error); error = kern_getpeername(td, args->s, (struct sockaddr *)&ss); if (error != 0) return (error); len = min(ss.ss_len, len); error = linux_copyout_sockaddr((struct sockaddr *)&ss, PTRIN(args->addr), len); if (error == 0) { len = ss.ss_len; error = copyout(&len, PTRIN(args->namelen), sizeof(len)); } return (error); } int linux_socketpair(struct thread *td, struct linux_socketpair_args *args) { int domain, error, sv[2], type; domain = linux_to_bsd_domain(args->domain); if (domain != PF_LOCAL) return (EAFNOSUPPORT); type = args->type & LINUX_SOCK_TYPE_MASK; if (type < 0 || type > LINUX_SOCK_MAX) return (EINVAL); error = linux_set_socket_flags(args->type & ~LINUX_SOCK_TYPE_MASK, &type); if (error != 0) return (error); if (args->protocol != 0 && args->protocol != PF_UNIX) { /* * Use of PF_UNIX as protocol argument is not right, * but Linux does it. * Do not map PF_UNIX as its Linux value is identical * to FreeBSD one. */ return (EPROTONOSUPPORT); } error = kern_socketpair(td, domain, type, 0, sv); if (error != 0) return (error); error = copyout(sv, PTRIN(args->rsv), 2 * sizeof(int)); if (error != 0) { (void)kern_close(td, sv[0]); (void)kern_close(td, sv[1]); } return (error); } #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) struct linux_send_args { register_t s; register_t msg; register_t len; register_t flags; }; static int linux_send(struct thread *td, struct linux_send_args *args) { struct sendto_args /* { int s; caddr_t buf; int len; int flags; caddr_t to; int tolen; } */ bsd_args; struct file *fp; int error; bsd_args.s = args->s; bsd_args.buf = (caddr_t)PTRIN(args->msg); bsd_args.len = args->len; bsd_args.flags = linux_to_bsd_msg_flags(args->flags); bsd_args.to = NULL; bsd_args.tolen = 0; error = sys_sendto(td, &bsd_args); if (error == ENOTCONN) { /* * Linux doesn't return ENOTCONN for non-blocking sockets. * Instead it returns the EAGAIN. */ error = getsock(td, args->s, &cap_send_rights, &fp); if (error == 0) { if (atomic_load_int(&fp->f_flag) & FNONBLOCK) error = EAGAIN; fdrop(fp, td); } } return (error); } struct linux_recv_args { register_t s; register_t msg; register_t len; register_t flags; }; static int linux_recv(struct thread *td, struct linux_recv_args *args) { struct recvfrom_args /* { int s; caddr_t buf; int len; int flags; struct sockaddr *from; socklen_t fromlenaddr; } */ bsd_args; bsd_args.s = args->s; bsd_args.buf = (caddr_t)PTRIN(args->msg); bsd_args.len = args->len; bsd_args.flags = linux_to_bsd_msg_flags(args->flags); bsd_args.from = NULL; bsd_args.fromlenaddr = 0; return (sys_recvfrom(td, &bsd_args)); } #endif /* __i386__ || (__amd64__ && COMPAT_LINUX32) */ int linux_sendto(struct thread *td, struct linux_sendto_args *args) { struct msghdr msg; struct iovec aiov; struct socket *so; struct file *fp; int error; if (linux_check_hdrincl(td, args->s) == 0) /* IP_HDRINCL set, tweak the packet before sending */ return (linux_sendto_hdrincl(td, args)); bzero(&msg, sizeof(msg)); error = getsock(td, args->s, &cap_send_connect_rights, &fp); if (error != 0) return (error); so = fp->f_data; if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) { msg.msg_name = PTRIN(args->to); msg.msg_namelen = args->tolen; } msg.msg_iov = &aiov; msg.msg_iovlen = 1; aiov.iov_base = PTRIN(args->msg); aiov.iov_len = args->len; fdrop(fp, td); return (linux_sendit(td, args->s, &msg, args->flags, NULL, UIO_USERSPACE)); } int linux_recvfrom(struct thread *td, struct linux_recvfrom_args *args) { struct sockaddr *sa; struct msghdr msg; struct iovec aiov; int error, fromlen; if (PTRIN(args->fromlen) != NULL) { error = copyin(PTRIN(args->fromlen), &fromlen, sizeof(fromlen)); if (error != 0) return (error); if (fromlen < 0) return (EINVAL); fromlen = min(fromlen, SOCK_MAXADDRLEN); sa = malloc(fromlen, M_SONAME, M_WAITOK); } else { fromlen = 0; sa = NULL; } msg.msg_name = sa; msg.msg_namelen = fromlen; msg.msg_iov = &aiov; msg.msg_iovlen = 1; aiov.iov_base = PTRIN(args->buf); aiov.iov_len = args->len; msg.msg_control = 0; msg.msg_flags = linux_to_bsd_msg_flags(args->flags); error = kern_recvit(td, args->s, &msg, UIO_SYSSPACE, NULL); if (error != 0) goto out; /* * XXX. Seems that FreeBSD is different from Linux here. Linux * fill source address if underlying protocol provides it, while * FreeBSD fill it if underlying protocol is not connection-oriented. * So, kern_recvit() set msg.msg_namelen to 0 if protocol pr_flags * does not contains PR_ADDR flag. */ if (PTRIN(args->from) != NULL && msg.msg_namelen != 0) error = linux_copyout_sockaddr(sa, PTRIN(args->from), msg.msg_namelen); if (error == 0 && PTRIN(args->fromlen) != NULL) error = copyout(&msg.msg_namelen, PTRIN(args->fromlen), sizeof(msg.msg_namelen)); out: free(sa, M_SONAME); return (error); } static int linux_sendmsg_common(struct thread *td, l_int s, struct l_msghdr *msghdr, l_uint flags) { struct sockaddr_storage ss = { .ss_len = sizeof(ss) }; struct cmsghdr *cmsg; struct mbuf *control; struct msghdr msg; struct l_cmsghdr linux_cmsg; struct l_cmsghdr *ptr_cmsg; struct l_msghdr linux_msghdr; struct iovec *iov; socklen_t datalen; struct socket *so; sa_family_t sa_family; struct file *fp; void *data; l_size_t len; l_size_t clen; int error; error = copyin(msghdr, &linux_msghdr, sizeof(linux_msghdr)); if (error != 0) return (error); /* * Some Linux applications (ping) define a non-NULL control data * pointer, but a msg_controllen of 0, which is not allowed in the * FreeBSD system call interface. NULL the msg_control pointer in * order to handle this case. This should be checked, but allows the * Linux ping to work. */ if (PTRIN(linux_msghdr.msg_control) != NULL && linux_msghdr.msg_controllen == 0) linux_msghdr.msg_control = PTROUT(NULL); error = linux_to_bsd_msghdr(&msg, &linux_msghdr); if (error != 0) return (error); #ifdef COMPAT_LINUX32 error = freebsd32_copyiniov(PTRIN(msg.msg_iov), msg.msg_iovlen, &iov, EMSGSIZE); #else error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); #endif if (error != 0) return (error); control = NULL; error = kern_getsockname(td, s, (struct sockaddr *)&ss); if (error != 0) goto bad; sa_family = ss.ss_family; if (flags & LINUX_MSG_OOB) { error = EOPNOTSUPP; if (sa_family == AF_UNIX) goto bad; error = getsock(td, s, &cap_send_rights, &fp); if (error != 0) goto bad; so = fp->f_data; if (so->so_type != SOCK_STREAM) error = EOPNOTSUPP; fdrop(fp, td); if (error != 0) goto bad; } if (linux_msghdr.msg_controllen >= sizeof(struct l_cmsghdr)) { error = ENOBUFS; control = m_get(M_WAITOK, MT_CONTROL); MCLGET(control, M_WAITOK); data = mtod(control, void *); datalen = 0; ptr_cmsg = PTRIN(linux_msghdr.msg_control); clen = linux_msghdr.msg_controllen; do { error = copyin(ptr_cmsg, &linux_cmsg, sizeof(struct l_cmsghdr)); if (error != 0) goto bad; error = EINVAL; if (linux_cmsg.cmsg_len < sizeof(struct l_cmsghdr) || linux_cmsg.cmsg_len > clen) goto bad; if (datalen + CMSG_HDRSZ > MCLBYTES) goto bad; /* * Now we support only SCM_RIGHTS and SCM_CRED, * so return EINVAL in any other cmsg_type */ cmsg = data; cmsg->cmsg_type = linux_to_bsd_cmsg_type(linux_cmsg.cmsg_type); cmsg->cmsg_level = linux_to_bsd_sockopt_level(linux_cmsg.cmsg_level); if (cmsg->cmsg_type == -1 || cmsg->cmsg_level != SOL_SOCKET) { linux_msg(curthread, "unsupported sendmsg cmsg level %d type %d", linux_cmsg.cmsg_level, linux_cmsg.cmsg_type); goto bad; } /* * Some applications (e.g. pulseaudio) attempt to * send ancillary data even if the underlying protocol * doesn't support it which is not allowed in the * FreeBSD system call interface. */ if (sa_family != AF_UNIX) goto next; if (cmsg->cmsg_type == SCM_CREDS) { len = sizeof(struct cmsgcred); if (datalen + CMSG_SPACE(len) > MCLBYTES) goto bad; /* * The lower levels will fill in the structure */ memset(CMSG_DATA(data), 0, len); } else { len = linux_cmsg.cmsg_len - L_CMSG_HDRSZ; if (datalen + CMSG_SPACE(len) < datalen || datalen + CMSG_SPACE(len) > MCLBYTES) goto bad; error = copyin(LINUX_CMSG_DATA(ptr_cmsg), CMSG_DATA(data), len); if (error != 0) goto bad; } cmsg->cmsg_len = CMSG_LEN(len); data = (char *)data + CMSG_SPACE(len); datalen += CMSG_SPACE(len); next: if (clen <= LINUX_CMSG_ALIGN(linux_cmsg.cmsg_len)) break; clen -= LINUX_CMSG_ALIGN(linux_cmsg.cmsg_len); ptr_cmsg = (struct l_cmsghdr *)((char *)ptr_cmsg + LINUX_CMSG_ALIGN(linux_cmsg.cmsg_len)); } while(clen >= sizeof(struct l_cmsghdr)); control->m_len = datalen; if (datalen == 0) { m_freem(control); control = NULL; } } msg.msg_iov = iov; msg.msg_flags = 0; error = linux_sendit(td, s, &msg, flags, control, UIO_USERSPACE); control = NULL; bad: m_freem(control); free(iov, M_IOV); return (error); } int linux_sendmsg(struct thread *td, struct linux_sendmsg_args *args) { return (linux_sendmsg_common(td, args->s, PTRIN(args->msg), args->flags)); } int linux_sendmmsg(struct thread *td, struct linux_sendmmsg_args *args) { struct l_mmsghdr *msg; l_uint retval; int error, datagrams; if (args->vlen > UIO_MAXIOV) args->vlen = UIO_MAXIOV; msg = PTRIN(args->msg); datagrams = 0; while (datagrams < args->vlen) { error = linux_sendmsg_common(td, args->s, &msg->msg_hdr, args->flags); if (error != 0) break; retval = td->td_retval[0]; error = copyout(&retval, &msg->msg_len, sizeof(msg->msg_len)); if (error != 0) break; ++msg; ++datagrams; } if (error == 0) td->td_retval[0] = datagrams; return (error); } static int recvmsg_scm_rights(struct thread *td, l_uint flags, socklen_t *datalen, void **data, void **udata) { int i, fd, fds, *fdp; if (flags & LINUX_MSG_CMSG_CLOEXEC) { fds = *datalen / sizeof(int); fdp = *data; for (i = 0; i < fds; i++) { fd = *fdp++; (void)kern_fcntl(td, fd, F_SETFD, FD_CLOEXEC); } } return (0); } static int recvmsg_scm_creds(socklen_t *datalen, void **data, void **udata) { struct cmsgcred *cmcred; struct l_ucred lu; cmcred = *data; lu.pid = cmcred->cmcred_pid; lu.uid = cmcred->cmcred_uid; lu.gid = cmcred->cmcred_gid; memmove(*data, &lu, sizeof(lu)); *datalen = sizeof(lu); return (0); } _Static_assert(sizeof(struct cmsgcred) >= sizeof(struct l_ucred), "scm_creds sizeof l_ucred"); static int recvmsg_scm_creds2(socklen_t *datalen, void **data, void **udata) { struct sockcred2 *scred; struct l_ucred lu; scred = *data; lu.pid = scred->sc_pid; lu.uid = scred->sc_uid; lu.gid = scred->sc_gid; memmove(*data, &lu, sizeof(lu)); *datalen = sizeof(lu); return (0); } _Static_assert(sizeof(struct sockcred2) >= sizeof(struct l_ucred), "scm_creds2 sizeof l_ucred"); #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) static int recvmsg_scm_timestamp(l_int msg_type, socklen_t *datalen, void **data, void **udata) { l_sock_timeval ltv64; l_timeval ltv; struct timeval *tv; socklen_t len; void *buf; if (*datalen != sizeof(struct timeval)) return (EMSGSIZE); tv = *data; #if defined(COMPAT_LINUX32) if (msg_type == LINUX_SCM_TIMESTAMPO && (tv->tv_sec > INT_MAX || tv->tv_sec < INT_MIN)) return (EOVERFLOW); #endif if (msg_type == LINUX_SCM_TIMESTAMPN) len = sizeof(ltv64); else len = sizeof(ltv); buf = malloc(len, M_LINUX, M_WAITOK); if (msg_type == LINUX_SCM_TIMESTAMPN) { ltv64.tv_sec = tv->tv_sec; ltv64.tv_usec = tv->tv_usec; memmove(buf, <v64, len); } else { ltv.tv_sec = tv->tv_sec; ltv.tv_usec = tv->tv_usec; memmove(buf, <v, len); } *data = *udata = buf; *datalen = len; return (0); } #else _Static_assert(sizeof(struct timeval) == sizeof(l_timeval), "scm_timestamp sizeof l_timeval"); #endif /* __i386__ || (__amd64__ && COMPAT_LINUX32) */ #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) static int recvmsg_scm_timestampns(l_int msg_type, socklen_t *datalen, void **data, void **udata) { struct l_timespec64 ts64; struct l_timespec ts32; struct timespec ts; socklen_t len; void *buf; if (msg_type == LINUX_SCM_TIMESTAMPNSO) len = sizeof(ts32); else len = sizeof(ts64); buf = malloc(len, M_LINUX, M_WAITOK); bintime2timespec(*data, &ts); if (msg_type == LINUX_SCM_TIMESTAMPNSO) { ts32.tv_sec = ts.tv_sec; ts32.tv_nsec = ts.tv_nsec; memmove(buf, &ts32, len); } else { ts64.tv_sec = ts.tv_sec; ts64.tv_nsec = ts.tv_nsec; memmove(buf, &ts64, len); } *data = *udata = buf; *datalen = len; return (0); } #else static int recvmsg_scm_timestampns(l_int msg_type, socklen_t *datalen, void **data, void **udata) { struct timespec ts; bintime2timespec(*data, &ts); memmove(*data, &ts, sizeof(struct timespec)); *datalen = sizeof(struct timespec); return (0); } _Static_assert(sizeof(struct bintime) >= sizeof(struct timespec), "scm_timestampns sizeof timespec"); #endif /* __i386__ || (__amd64__ && COMPAT_LINUX32) */ static int recvmsg_scm_sol_socket(struct thread *td, l_int msg_type, l_int lmsg_type, l_uint flags, socklen_t *datalen, void **data, void **udata) { int error; error = 0; switch (msg_type) { case SCM_RIGHTS: error = recvmsg_scm_rights(td, flags, datalen, data, udata); break; case SCM_CREDS: error = recvmsg_scm_creds(datalen, data, udata); break; case SCM_CREDS2: error = recvmsg_scm_creds2(datalen, data, udata); break; case SCM_TIMESTAMP: #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) error = recvmsg_scm_timestamp(lmsg_type, datalen, data, udata); #endif break; case SCM_BINTIME: error = recvmsg_scm_timestampns(lmsg_type, datalen, data, udata); break; } return (error); } static int recvmsg_scm_ip_origdstaddr(socklen_t *datalen, void **data, void **udata) { struct l_sockaddr *lsa; int error; error = bsd_to_linux_sockaddr(*data, &lsa, *datalen); if (error == 0) { *data = *udata = lsa; *datalen = sizeof(*lsa); } return (error); } static int recvmsg_scm_ipproto_ip(l_int msg_type, l_int lmsg_type, socklen_t *datalen, void **data, void **udata) { int error; error = 0; switch (msg_type) { case IP_ORIGDSTADDR: error = recvmsg_scm_ip_origdstaddr(datalen, data, udata); break; } return (error); } static int linux_recvmsg_common(struct thread *td, l_int s, struct l_msghdr *msghdr, l_uint flags, struct msghdr *msg) { struct proc *p = td->td_proc; struct cmsghdr *cm; struct l_cmsghdr *lcm = NULL; socklen_t datalen, maxlen, outlen; struct l_msghdr l_msghdr; struct iovec *iov, *uiov; struct mbuf *m, *control = NULL; struct mbuf **controlp; struct sockaddr *sa; caddr_t outbuf; void *data, *udata; int error, skiped; error = copyin(msghdr, &l_msghdr, sizeof(l_msghdr)); if (error != 0) return (error); /* * Pass user-supplied recvmsg() flags in msg_flags field, * following sys_recvmsg() convention. */ l_msghdr.msg_flags = flags; error = linux_to_bsd_msghdr(msg, &l_msghdr); if (error != 0) return (error); #ifdef COMPAT_LINUX32 error = freebsd32_copyiniov(PTRIN(msg->msg_iov), msg->msg_iovlen, &iov, EMSGSIZE); #else error = copyiniov(msg->msg_iov, msg->msg_iovlen, &iov, EMSGSIZE); #endif if (error != 0) return (error); if (msg->msg_name != NULL && msg->msg_namelen > 0) { msg->msg_namelen = min(msg->msg_namelen, SOCK_MAXADDRLEN); sa = malloc(msg->msg_namelen, M_SONAME, M_WAITOK); msg->msg_name = sa; } else { sa = NULL; msg->msg_name = NULL; } uiov = msg->msg_iov; msg->msg_iov = iov; controlp = (msg->msg_control != NULL) ? &control : NULL; error = kern_recvit(td, s, msg, UIO_SYSSPACE, controlp); msg->msg_iov = uiov; if (error != 0) goto bad; /* * Note that kern_recvit() updates msg->msg_namelen. */ if (msg->msg_name != NULL && msg->msg_namelen > 0) { msg->msg_name = PTRIN(l_msghdr.msg_name); error = linux_copyout_sockaddr(sa, msg->msg_name, msg->msg_namelen); if (error != 0) goto bad; } error = bsd_to_linux_msghdr(msg, &l_msghdr); if (error != 0) goto bad; skiped = outlen = 0; maxlen = l_msghdr.msg_controllen; if (control == NULL) goto out; lcm = malloc(L_CMSG_HDRSZ, M_LINUX, M_WAITOK | M_ZERO); msg->msg_control = mtod(control, struct cmsghdr *); msg->msg_controllen = control->m_len; outbuf = PTRIN(l_msghdr.msg_control); for (m = control; m != NULL; m = m->m_next) { cm = mtod(m, struct cmsghdr *); lcm->cmsg_type = bsd_to_linux_cmsg_type(p, cm->cmsg_type, cm->cmsg_level); lcm->cmsg_level = bsd_to_linux_sockopt_level(cm->cmsg_level); if (lcm->cmsg_type == -1 || cm->cmsg_level == -1) { LINUX_RATELIMIT_MSG_OPT2( "unsupported recvmsg cmsg level %d type %d", cm->cmsg_level, cm->cmsg_type); /* Skip unsupported messages */ skiped++; continue; } data = CMSG_DATA(cm); datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data; udata = NULL; error = 0; switch (cm->cmsg_level) { case IPPROTO_IP: error = recvmsg_scm_ipproto_ip(cm->cmsg_type, lcm->cmsg_type, &datalen, &data, &udata); break; case SOL_SOCKET: error = recvmsg_scm_sol_socket(td, cm->cmsg_type, lcm->cmsg_type, flags, &datalen, &data, &udata); break; } /* The recvmsg_scm_ is responsible to free udata on error. */ if (error != 0) goto bad; if (outlen + LINUX_CMSG_LEN(datalen) > maxlen) { if (outlen == 0) { error = EMSGSIZE; goto err; } else { l_msghdr.msg_flags |= LINUX_MSG_CTRUNC; m_dispose_extcontrolm(control); free(udata, M_LINUX); goto out; } } lcm->cmsg_len = LINUX_CMSG_LEN(datalen); error = copyout(lcm, outbuf, L_CMSG_HDRSZ); if (error == 0) { error = copyout(data, LINUX_CMSG_DATA(outbuf), datalen); if (error == 0) { outbuf += LINUX_CMSG_SPACE(datalen); outlen += LINUX_CMSG_SPACE(datalen); } } err: free(udata, M_LINUX); if (error != 0) goto bad; } if (outlen == 0 && skiped > 0) { error = EINVAL; goto bad; } out: l_msghdr.msg_controllen = outlen; error = copyout(&l_msghdr, msghdr, sizeof(l_msghdr)); bad: if (control != NULL) { if (error != 0) m_dispose_extcontrolm(control); m_freem(control); } free(iov, M_IOV); free(lcm, M_LINUX); free(sa, M_SONAME); return (error); } int linux_recvmsg(struct thread *td, struct linux_recvmsg_args *args) { struct msghdr bsd_msg; struct file *fp; int error; error = getsock(td, args->s, &cap_recv_rights, &fp); if (error != 0) return (error); fdrop(fp, td); return (linux_recvmsg_common(td, args->s, PTRIN(args->msg), args->flags, &bsd_msg)); } static int linux_recvmmsg_common(struct thread *td, l_int s, struct l_mmsghdr *msg, l_uint vlen, l_uint flags, struct timespec *tts) { struct msghdr bsd_msg; struct timespec ts; struct file *fp; l_uint retval; int error, datagrams; error = getsock(td, s, &cap_recv_rights, &fp); if (error != 0) return (error); datagrams = 0; while (datagrams < vlen) { error = linux_recvmsg_common(td, s, &msg->msg_hdr, flags & ~LINUX_MSG_WAITFORONE, &bsd_msg); if (error != 0) break; retval = td->td_retval[0]; error = copyout(&retval, &msg->msg_len, sizeof(msg->msg_len)); if (error != 0) break; ++msg; ++datagrams; /* * MSG_WAITFORONE turns on MSG_DONTWAIT after one packet. */ if (flags & LINUX_MSG_WAITFORONE) flags |= LINUX_MSG_DONTWAIT; /* * See BUGS section of recvmmsg(2). */ if (tts) { getnanotime(&ts); timespecsub(&ts, tts, &ts); if (!timespecisset(&ts) || ts.tv_sec > 0) break; } /* Out of band data, return right away. */ if (bsd_msg.msg_flags & MSG_OOB) break; } if (error == 0) td->td_retval[0] = datagrams; fdrop(fp, td); return (error); } int linux_recvmmsg(struct thread *td, struct linux_recvmmsg_args *args) { struct timespec ts, tts, *ptts; int error; if (args->timeout) { error = linux_get_timespec(&ts, args->timeout); if (error != 0) return (error); getnanotime(&tts); timespecadd(&tts, &ts, &tts); ptts = &tts; } else ptts = NULL; return (linux_recvmmsg_common(td, args->s, PTRIN(args->msg), args->vlen, args->flags, ptts)); } #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) int linux_recvmmsg_time64(struct thread *td, struct linux_recvmmsg_time64_args *args) { struct timespec ts, tts, *ptts; int error; if (args->timeout) { error = linux_get_timespec64(&ts, args->timeout); if (error != 0) return (error); getnanotime(&tts); timespecadd(&tts, &ts, &tts); ptts = &tts; } else ptts = NULL; return (linux_recvmmsg_common(td, args->s, PTRIN(args->msg), args->vlen, args->flags, ptts)); } #endif int linux_shutdown(struct thread *td, struct linux_shutdown_args *args) { return (kern_shutdown(td, args->s, args->how)); } int linux_setsockopt(struct thread *td, struct linux_setsockopt_args *args) { struct proc *p = td->td_proc; struct linux_pemuldata *pem; l_timeval linux_tv; struct sockaddr *sa; struct timeval tv; socklen_t len; int error, level, name, val; level = linux_to_bsd_sockopt_level(args->level); switch (level) { case SOL_SOCKET: name = linux_to_bsd_so_sockopt(args->optname); switch (name) { case LOCAL_CREDS_PERSISTENT: level = SOL_LOCAL; break; case SO_RCVTIMEO: /* FALLTHROUGH */ case SO_SNDTIMEO: error = copyin(PTRIN(args->optval), &linux_tv, sizeof(linux_tv)); if (error != 0) return (error); tv.tv_sec = linux_tv.tv_sec; tv.tv_usec = linux_tv.tv_usec; return (kern_setsockopt(td, args->s, level, name, &tv, UIO_SYSSPACE, sizeof(tv))); /* NOTREACHED */ case SO_TIMESTAMP: /* overwrite SO_BINTIME */ val = 0; error = kern_setsockopt(td, args->s, level, SO_BINTIME, &val, UIO_SYSSPACE, sizeof(val)); if (error != 0) return (error); pem = pem_find(p); pem->so_timestamp = args->optname; break; case SO_BINTIME: /* overwrite SO_TIMESTAMP */ val = 0; error = kern_setsockopt(td, args->s, level, SO_TIMESTAMP, &val, UIO_SYSSPACE, sizeof(val)); if (error != 0) return (error); pem = pem_find(p); pem->so_timestampns = args->optname; break; default: break; } break; case IPPROTO_IP: if (args->optname == LINUX_IP_RECVERR && linux_ignore_ip_recverr) { /* * XXX: This is a hack to unbreak DNS resolution * with glibc 2.30 and above. */ return (0); } name = linux_to_bsd_ip_sockopt(args->optname); break; case IPPROTO_IPV6: if (args->optname == LINUX_IPV6_RECVERR && linux_ignore_ip_recverr) { /* * XXX: This is a hack to unbreak DNS resolution * with glibc 2.30 and above. */ return (0); } name = linux_to_bsd_ip6_sockopt(args->optname); break; case IPPROTO_TCP: name = linux_to_bsd_tcp_sockopt(args->optname); break; case SOL_NETLINK: name = args->optname; break; default: name = -1; break; } if (name < 0) { if (name == -1) linux_msg(curthread, "unsupported setsockopt level %d optname %d", args->level, args->optname); return (ENOPROTOOPT); } if (name == IPV6_NEXTHOP) { len = args->optlen; error = linux_to_bsd_sockaddr(PTRIN(args->optval), &sa, &len); if (error != 0) return (error); error = kern_setsockopt(td, args->s, level, name, sa, UIO_SYSSPACE, len); free(sa, M_SONAME); } else { error = kern_setsockopt(td, args->s, level, name, PTRIN(args->optval), UIO_USERSPACE, args->optlen); } return (error); } static int linux_sockopt_copyout(struct thread *td, void *val, socklen_t len, struct linux_getsockopt_args *args) { int error; error = copyout(val, PTRIN(args->optval), len); if (error == 0) error = copyout(&len, PTRIN(args->optlen), sizeof(len)); return (error); } static int linux_getsockopt_so_peergroups(struct thread *td, struct linux_getsockopt_args *args) { struct xucred xu; socklen_t xulen, len; int error, i; xulen = sizeof(xu); error = kern_getsockopt(td, args->s, 0, LOCAL_PEERCRED, &xu, UIO_SYSSPACE, &xulen); if (error != 0) return (error); len = xu.cr_ngroups * sizeof(l_gid_t); if (args->optlen < len) { error = copyout(&len, PTRIN(args->optlen), sizeof(len)); if (error == 0) error = ERANGE; return (error); } /* * "- 1" to skip the primary group. */ for (i = 0; i < xu.cr_ngroups - 1; i++) { error = copyout(xu.cr_groups + i + 1, (void *)(args->optval + i * sizeof(l_gid_t)), sizeof(l_gid_t)); if (error != 0) return (error); } error = copyout(&len, PTRIN(args->optlen), sizeof(len)); return (error); } static int linux_getsockopt_so_peersec(struct thread *td, struct linux_getsockopt_args *args) { socklen_t len; int error; len = sizeof(SECURITY_CONTEXT_STRING); if (args->optlen < len) { error = copyout(&len, PTRIN(args->optlen), sizeof(len)); if (error == 0) error = ERANGE; return (error); } return (linux_sockopt_copyout(td, SECURITY_CONTEXT_STRING, len, args)); } static int linux_getsockopt_so_linger(struct thread *td, struct linux_getsockopt_args *args) { struct linger ling; socklen_t len; int error; len = sizeof(ling); error = kern_getsockopt(td, args->s, SOL_SOCKET, SO_LINGER, &ling, UIO_SYSSPACE, &len); if (error != 0) return (error); ling.l_onoff = ((ling.l_onoff & SO_LINGER) != 0); return (linux_sockopt_copyout(td, &ling, len, args)); } int linux_getsockopt(struct thread *td, struct linux_getsockopt_args *args) { l_timeval linux_tv; struct timeval tv; socklen_t tv_len, xulen, len; struct sockaddr *sa; struct xucred xu; struct l_ucred lxu; int error, level, name, newval; level = linux_to_bsd_sockopt_level(args->level); switch (level) { case SOL_SOCKET: switch (args->optname) { case LINUX_SO_PEERGROUPS: return (linux_getsockopt_so_peergroups(td, args)); case LINUX_SO_PEERSEC: return (linux_getsockopt_so_peersec(td, args)); default: break; } name = linux_to_bsd_so_sockopt(args->optname); switch (name) { case LOCAL_CREDS_PERSISTENT: level = SOL_LOCAL; break; case SO_RCVTIMEO: /* FALLTHROUGH */ case SO_SNDTIMEO: tv_len = sizeof(tv); error = kern_getsockopt(td, args->s, level, name, &tv, UIO_SYSSPACE, &tv_len); if (error != 0) return (error); linux_tv.tv_sec = tv.tv_sec; linux_tv.tv_usec = tv.tv_usec; return (linux_sockopt_copyout(td, &linux_tv, sizeof(linux_tv), args)); /* NOTREACHED */ case LOCAL_PEERCRED: if (args->optlen < sizeof(lxu)) return (EINVAL); /* * LOCAL_PEERCRED is not served at the SOL_SOCKET level, * but by the Unix socket's level 0. */ level = 0; xulen = sizeof(xu); error = kern_getsockopt(td, args->s, level, name, &xu, UIO_SYSSPACE, &xulen); if (error != 0) return (error); lxu.pid = xu.cr_pid; lxu.uid = xu.cr_uid; lxu.gid = xu.cr_gid; return (linux_sockopt_copyout(td, &lxu, sizeof(lxu), args)); /* NOTREACHED */ case SO_ERROR: len = sizeof(newval); error = kern_getsockopt(td, args->s, level, name, &newval, UIO_SYSSPACE, &len); if (error != 0) return (error); newval = -bsd_to_linux_errno(newval); return (linux_sockopt_copyout(td, &newval, len, args)); /* NOTREACHED */ case SO_DOMAIN: len = sizeof(newval); error = kern_getsockopt(td, args->s, level, name, &newval, UIO_SYSSPACE, &len); if (error != 0) return (error); newval = bsd_to_linux_domain((sa_family_t)newval); if (newval == AF_UNKNOWN) return (ENOPROTOOPT); return (linux_sockopt_copyout(td, &newval, len, args)); /* NOTREACHED */ case SO_LINGER: return (linux_getsockopt_so_linger(td, args)); /* NOTREACHED */ default: break; } break; case IPPROTO_IP: name = linux_to_bsd_ip_sockopt(args->optname); break; case IPPROTO_IPV6: name = linux_to_bsd_ip6_sockopt(args->optname); break; case IPPROTO_TCP: name = linux_to_bsd_tcp_sockopt(args->optname); break; default: name = -1; break; } if (name < 0) { if (name == -1) linux_msg(curthread, "unsupported getsockopt level %d optname %d", args->level, args->optname); return (EINVAL); } if (name == IPV6_NEXTHOP) { error = copyin(PTRIN(args->optlen), &len, sizeof(len)); if (error != 0) return (error); sa = malloc(len, M_SONAME, M_WAITOK); error = kern_getsockopt(td, args->s, level, name, sa, UIO_SYSSPACE, &len); if (error != 0) goto out; error = linux_copyout_sockaddr(sa, PTRIN(args->optval), len); if (error == 0) error = copyout(&len, PTRIN(args->optlen), sizeof(len)); out: free(sa, M_SONAME); } else { if (args->optval) { error = copyin(PTRIN(args->optlen), &len, sizeof(len)); if (error != 0) return (error); } error = kern_getsockopt(td, args->s, level, name, PTRIN(args->optval), UIO_USERSPACE, &len); if (error == 0) error = copyout(&len, PTRIN(args->optlen), sizeof(len)); } return (error); } /* * Based on sendfile_getsock from kern_sendfile.c * Determines whether an fd is a stream socket that can be used * with FreeBSD sendfile. */ static bool is_sendfile(struct file *fp, struct file *ofp) { struct socket *so; /* * FreeBSD sendfile() system call sends a regular file or * shared memory object out a stream socket. */ if ((fp->f_type != DTYPE_SHM && fp->f_type != DTYPE_VNODE) || (fp->f_type == DTYPE_VNODE && (fp->f_vnode == NULL || fp->f_vnode->v_type != VREG))) return (false); /* * The socket must be a stream socket and connected. */ if (ofp->f_type != DTYPE_SOCKET) return (false); so = ofp->f_data; if (so->so_type != SOCK_STREAM) return (false); /* * SCTP one-to-one style sockets currently don't work with * sendfile(). */ if (so->so_proto->pr_protocol == IPPROTO_SCTP) return (false); return (!SOLISTENING(so)); } static bool is_regular_file(struct file *fp) { return (fp->f_type == DTYPE_VNODE && fp->f_vnode != NULL && fp->f_vnode->v_type == VREG); } static int sendfile_fallback(struct thread *td, struct file *fp, l_int out, off_t *offset, l_size_t count, off_t *sbytes) { off_t current_offset, out_offset, to_send; l_size_t bytes_sent, n_read; struct file *ofp; struct iovec aiov; struct uio auio; bool seekable; size_t bufsz; void *buf; int flags, error; if (offset == NULL) { if ((error = fo_seek(fp, 0, SEEK_CUR, td)) != 0) return (error); current_offset = td->td_uretoff.tdu_off; } else { if ((fp->f_ops->fo_flags & DFLAG_SEEKABLE) == 0) return (ESPIPE); current_offset = *offset; } error = fget_write(td, out, &cap_pwrite_rights, &ofp); if (error != 0) return (error); seekable = (ofp->f_ops->fo_flags & DFLAG_SEEKABLE) != 0; if (seekable) { if ((error = fo_seek(ofp, 0, SEEK_CUR, td)) != 0) goto drop; out_offset = td->td_uretoff.tdu_off; } else out_offset = 0; flags = FOF_OFFSET | FOF_NOUPDATE; - bufsz = min(count, MAXPHYS); + bufsz = min(count, maxphys); buf = malloc(bufsz, M_LINUX, M_WAITOK); bytes_sent = 0; while (bytes_sent < count) { to_send = min(count - bytes_sent, bufsz); aiov.iov_base = buf; aiov.iov_len = bufsz; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_segflg = UIO_SYSSPACE; auio.uio_td = td; auio.uio_rw = UIO_READ; auio.uio_offset = current_offset; auio.uio_resid = to_send; error = fo_read(fp, &auio, fp->f_cred, flags, td); if (error != 0) break; n_read = to_send - auio.uio_resid; if (n_read == 0) break; aiov.iov_base = buf; aiov.iov_len = bufsz; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_segflg = UIO_SYSSPACE; auio.uio_td = td; auio.uio_rw = UIO_WRITE; auio.uio_offset = (seekable) ? out_offset : 0; auio.uio_resid = n_read; error = fo_write(ofp, &auio, ofp->f_cred, flags, td); if (error != 0) break; bytes_sent += n_read; current_offset += n_read; out_offset += n_read; } free(buf, M_LINUX); if (error == 0) { *sbytes = bytes_sent; if (offset != NULL) *offset = current_offset; else error = fo_seek(fp, current_offset, SEEK_SET, td); } if (error == 0 && seekable) error = fo_seek(ofp, out_offset, SEEK_SET, td); drop: fdrop(ofp, td); return (error); } static int sendfile_sendfile(struct thread *td, struct file *fp, l_int out, off_t *offset, l_size_t count, off_t *sbytes) { off_t current_offset; int error; if (offset == NULL) { if ((fp->f_ops->fo_flags & DFLAG_SEEKABLE) == 0) return (ESPIPE); if ((error = fo_seek(fp, 0, SEEK_CUR, td)) != 0) return (error); current_offset = td->td_uretoff.tdu_off; } else current_offset = *offset; error = fo_sendfile(fp, out, NULL, NULL, current_offset, count, sbytes, 0, td); if (error == 0) { current_offset += *sbytes; if (offset != NULL) *offset = current_offset; else error = fo_seek(fp, current_offset, SEEK_SET, td); } return (error); } static int linux_sendfile_common(struct thread *td, l_int out, l_int in, off_t *offset, l_size_t count) { struct file *fp, *ofp; off_t sbytes; int error; /* Linux cannot have 0 count. */ if (count <= 0 || (offset != NULL && *offset < 0)) return (EINVAL); AUDIT_ARG_FD(in); error = fget_read(td, in, &cap_pread_rights, &fp); if (error != 0) return (error); if ((fp->f_type != DTYPE_SHM && fp->f_type != DTYPE_VNODE) || (fp->f_type == DTYPE_VNODE && (fp->f_vnode == NULL || fp->f_vnode->v_type != VREG))) { error = EINVAL; goto drop; } error = fget_unlocked(td, out, &cap_no_rights, &ofp); if (error != 0) goto drop; if (is_regular_file(fp) && is_regular_file(ofp)) { error = kern_copy_file_range(td, in, offset, out, NULL, count, 0); } else { sbytes = 0; if (is_sendfile(fp, ofp)) error = sendfile_sendfile(td, fp, out, offset, count, &sbytes); else error = sendfile_fallback(td, fp, out, offset, count, &sbytes); if (error == ENOBUFS && (ofp->f_flag & FNONBLOCK) != 0) error = EAGAIN; if (error == 0) td->td_retval[0] = sbytes; } fdrop(ofp, td); drop: fdrop(fp, td); return (error); } int linux_sendfile(struct thread *td, struct linux_sendfile_args *arg) { /* * Differences between FreeBSD and Linux sendfile: * - Linux doesn't send anything when count is 0 (FreeBSD uses 0 to * mean send the whole file). * - Linux can send to any fd whereas FreeBSD only supports sockets. * We therefore use FreeBSD sendfile where possible for performance, * but fall back on a manual copy (sendfile_fallback). * - Linux doesn't have an equivalent for FreeBSD's flags and sf_hdtr. * - Linux takes an offset pointer and updates it to the read location. * FreeBSD takes in an offset and a 'bytes read' parameter which is * only filled if it isn't NULL. We use this parameter to update the * offset pointer if it exists. * - Linux sendfile returns bytes read on success while FreeBSD * returns 0. We use the 'bytes read' parameter to get this value. */ off_t offset64; l_off_t offset; int error; if (arg->offset != NULL) { error = copyin(arg->offset, &offset, sizeof(offset)); if (error != 0) return (error); offset64 = offset; } error = linux_sendfile_common(td, arg->out, arg->in, arg->offset != NULL ? &offset64 : NULL, arg->count); if (error == 0 && arg->offset != NULL) { #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) if (offset64 > INT32_MAX) return (EOVERFLOW); #endif offset = (l_off_t)offset64; error = copyout(&offset, arg->offset, sizeof(offset)); } return (error); } #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) int linux_sendfile64(struct thread *td, struct linux_sendfile64_args *arg) { off_t offset; int error; if (arg->offset != NULL) { error = copyin(arg->offset, &offset, sizeof(offset)); if (error != 0) return (error); } error = linux_sendfile_common(td, arg->out, arg->in, arg->offset != NULL ? &offset : NULL, arg->count); if (error == 0 && arg->offset != NULL) error = copyout(&offset, arg->offset, sizeof(offset)); return (error); } /* Argument list sizes for linux_socketcall */ static const unsigned char lxs_args_cnt[] = { 0 /* unused*/, 3 /* socket */, 3 /* bind */, 3 /* connect */, 2 /* listen */, 3 /* accept */, 3 /* getsockname */, 3 /* getpeername */, 4 /* socketpair */, 4 /* send */, 4 /* recv */, 6 /* sendto */, 6 /* recvfrom */, 2 /* shutdown */, 5 /* setsockopt */, 5 /* getsockopt */, 3 /* sendmsg */, 3 /* recvmsg */, 4 /* accept4 */, 5 /* recvmmsg */, 4 /* sendmmsg */, 4 /* sendfile */ }; #define LINUX_ARGS_CNT (nitems(lxs_args_cnt) - 1) #define LINUX_ARG_SIZE(x) (lxs_args_cnt[x] * sizeof(l_ulong)) int linux_socketcall(struct thread *td, struct linux_socketcall_args *args) { l_ulong a[6]; #if defined(__amd64__) && defined(COMPAT_LINUX32) register_t l_args[6]; #endif void *arg; int error; if (args->what < LINUX_SOCKET || args->what > LINUX_ARGS_CNT) return (EINVAL); error = copyin(PTRIN(args->args), a, LINUX_ARG_SIZE(args->what)); if (error != 0) return (error); #if defined(__amd64__) && defined(COMPAT_LINUX32) for (int i = 0; i < lxs_args_cnt[args->what]; ++i) l_args[i] = a[i]; arg = l_args; #else arg = a; #endif switch (args->what) { case LINUX_SOCKET: return (linux_socket(td, arg)); case LINUX_BIND: return (linux_bind(td, arg)); case LINUX_CONNECT: return (linux_connect(td, arg)); case LINUX_LISTEN: return (linux_listen(td, arg)); case LINUX_ACCEPT: return (linux_accept(td, arg)); case LINUX_GETSOCKNAME: return (linux_getsockname(td, arg)); case LINUX_GETPEERNAME: return (linux_getpeername(td, arg)); case LINUX_SOCKETPAIR: return (linux_socketpair(td, arg)); case LINUX_SEND: return (linux_send(td, arg)); case LINUX_RECV: return (linux_recv(td, arg)); case LINUX_SENDTO: return (linux_sendto(td, arg)); case LINUX_RECVFROM: return (linux_recvfrom(td, arg)); case LINUX_SHUTDOWN: return (linux_shutdown(td, arg)); case LINUX_SETSOCKOPT: return (linux_setsockopt(td, arg)); case LINUX_GETSOCKOPT: return (linux_getsockopt(td, arg)); case LINUX_SENDMSG: return (linux_sendmsg(td, arg)); case LINUX_RECVMSG: return (linux_recvmsg(td, arg)); case LINUX_ACCEPT4: return (linux_accept4(td, arg)); case LINUX_RECVMMSG: return (linux_recvmmsg(td, arg)); case LINUX_SENDMMSG: return (linux_sendmmsg(td, arg)); case LINUX_SENDFILE: return (linux_sendfile(td, arg)); } linux_msg(td, "socket type %d not implemented", args->what); return (ENOSYS); } #endif /* __i386__ || (__amd64__ && COMPAT_LINUX32) */ diff --git a/sys/dev/md/md.c b/sys/dev/md/md.c index 27e63363767c..241517898ad4 100644 --- a/sys/dev/md/md.c +++ b/sys/dev/md/md.c @@ -1,2181 +1,2181 @@ /*- * SPDX-License-Identifier: (Beerware AND BSD-3-Clause) * * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): * wrote this file. As long as you retain this notice you * can do whatever you want with this stuff. If we meet some day, and you think * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * */ /*- * The following functions are based on the vn(4) driver: mdstart_swap(), * mdstart_vnode(), mdcreate_swap(), mdcreate_vnode() and mddestroy(), * and as such under the following copyright: * * Copyright (c) 1988 University of Utah. * Copyright (c) 1990, 1993 * The Regents of the University of California. All rights reserved. * Copyright (c) 2013 The FreeBSD Foundation * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: Utah Hdr: vn.c 1.13 94/04/02 * From: src/sys/dev/vn/vn.c,v 1.122 2000/12/16 16:06:03 */ #include "opt_rootdevname.h" #include "opt_geom.h" #include "opt_md.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MD_MODVER 1 #define MD_SHUTDOWN 0x10000 /* Tell worker thread to terminate. */ #define MD_EXITING 0x20000 /* Worker thread is exiting. */ #define MD_PROVIDERGONE 0x40000 /* Safe to free the softc */ #ifndef MD_NSECT #define MD_NSECT (10000 * 2) #endif struct md_req { unsigned md_unit; /* unit number */ enum md_types md_type; /* type of disk */ off_t md_mediasize; /* size of disk in bytes */ unsigned md_sectorsize; /* sectorsize */ unsigned md_options; /* options */ int md_fwheads; /* firmware heads */ int md_fwsectors; /* firmware sectors */ char *md_file; /* pathname of file to mount */ enum uio_seg md_file_seg; /* location of md_file */ char *md_label; /* label of the device (userspace) */ int *md_units; /* pointer to units array (kernel) */ size_t md_units_nitems; /* items in md_units array */ }; #ifdef COMPAT_FREEBSD32 struct md_ioctl32 { unsigned md_version; unsigned md_unit; enum md_types md_type; uint32_t md_file; off_t md_mediasize; unsigned md_sectorsize; unsigned md_options; uint64_t md_base; int md_fwheads; int md_fwsectors; uint32_t md_label; int md_pad[MDNPAD]; } #ifdef __amd64__ __attribute__((__packed__)) #endif ; #ifndef __amd64__ CTASSERT((sizeof(struct md_ioctl32)) == 440); #else CTASSERT((sizeof(struct md_ioctl32)) == 436); #endif #define MDIOCATTACH_32 _IOC_NEWTYPE(MDIOCATTACH, struct md_ioctl32) #define MDIOCDETACH_32 _IOC_NEWTYPE(MDIOCDETACH, struct md_ioctl32) #define MDIOCQUERY_32 _IOC_NEWTYPE(MDIOCQUERY, struct md_ioctl32) #define MDIOCRESIZE_32 _IOC_NEWTYPE(MDIOCRESIZE, struct md_ioctl32) #endif /* COMPAT_FREEBSD32 */ static MALLOC_DEFINE(M_MD, "md_disk", "Memory Disk"); static MALLOC_DEFINE(M_MDSECT, "md_sectors", "Memory Disk Sectors"); static int md_debug; SYSCTL_INT(_debug, OID_AUTO, mddebug, CTLFLAG_RW, &md_debug, 0, "Enable md(4) debug messages"); static int md_malloc_wait; SYSCTL_INT(_vm, OID_AUTO, md_malloc_wait, CTLFLAG_RW, &md_malloc_wait, 0, "Allow malloc to wait for memory allocations"); #if defined(MD_ROOT) && !defined(MD_ROOT_FSTYPE) #define MD_ROOT_FSTYPE "ufs" #endif #if defined(MD_ROOT) /* * Preloaded image gets put here. */ #if defined(MD_ROOT_SIZE) /* * We put the mfs_root symbol into the oldmfs section of the kernel object file. * Applications that patch the object with the image can determine * the size looking at the oldmfs section size within the kernel. */ u_char mfs_root[MD_ROOT_SIZE*1024] __attribute__ ((section ("oldmfs"))); const int mfs_root_size = sizeof(mfs_root); #elif defined(MD_ROOT_MEM) /* MD region already mapped in the memory */ u_char *mfs_root; int mfs_root_size; #else extern volatile u_char __weak_symbol mfs_root; extern volatile u_char __weak_symbol mfs_root_end; #define mfs_root_size ((uintptr_t)(&mfs_root_end - &mfs_root)) #endif #endif static g_init_t g_md_init; static g_fini_t g_md_fini; static g_start_t g_md_start; static g_access_t g_md_access; static void g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, struct g_consumer *cp __unused, struct g_provider *pp); static g_provgone_t g_md_providergone; static struct cdev *status_dev = NULL; static struct sx md_sx; static struct unrhdr *md_uh; static d_ioctl_t mdctlioctl; static struct cdevsw mdctl_cdevsw = { .d_version = D_VERSION, .d_ioctl = mdctlioctl, .d_name = MD_NAME, }; struct g_class g_md_class = { .name = "MD", .version = G_VERSION, .init = g_md_init, .fini = g_md_fini, .start = g_md_start, .access = g_md_access, .dumpconf = g_md_dumpconf, .providergone = g_md_providergone, }; DECLARE_GEOM_CLASS(g_md_class, g_md); MODULE_VERSION(geom_md, 0); static LIST_HEAD(, md_s) md_softc_list = LIST_HEAD_INITIALIZER(md_softc_list); #define NINDIR (PAGE_SIZE / sizeof(uintptr_t)) #define NMASK (NINDIR-1) static int nshift; struct indir { uintptr_t *array; u_int total; u_int used; u_int shift; }; struct md_s { int unit; LIST_ENTRY(md_s) list; struct bio_queue_head bio_queue; struct mtx queue_mtx; struct cdev *dev; enum md_types type; off_t mediasize; unsigned sectorsize; unsigned opencount; unsigned fwheads; unsigned fwsectors; char ident[32]; unsigned flags; char name[20]; struct proc *procp; struct g_geom *gp; struct g_provider *pp; int (*start)(struct md_s *sc, struct bio *bp); struct devstat *devstat; bool candelete; /* MD_MALLOC related fields */ struct indir *indir; uma_zone_t uma; /* MD_PRELOAD related fields */ u_char *pl_ptr; size_t pl_len; /* MD_VNODE related fields */ struct vnode *vnode; char file[PATH_MAX]; char label[PATH_MAX]; struct ucred *cred; vm_offset_t kva; /* MD_SWAP related fields */ vm_object_t object; }; static struct indir * new_indir(u_int shift) { struct indir *ip; ip = malloc(sizeof *ip, M_MD, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); if (ip == NULL) return (NULL); ip->array = malloc(sizeof(uintptr_t) * NINDIR, M_MDSECT, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); if (ip->array == NULL) { free(ip, M_MD); return (NULL); } ip->total = NINDIR; ip->shift = shift; return (ip); } static void del_indir(struct indir *ip) { free(ip->array, M_MDSECT); free(ip, M_MD); } static void destroy_indir(struct md_s *sc, struct indir *ip) { int i; for (i = 0; i < NINDIR; i++) { if (!ip->array[i]) continue; if (ip->shift) destroy_indir(sc, (struct indir*)(ip->array[i])); else if (ip->array[i] > 255) uma_zfree(sc->uma, (void *)(ip->array[i])); } del_indir(ip); } /* * This function does the math and allocates the top level "indir" structure * for a device of "size" sectors. */ static struct indir * dimension(off_t size) { off_t rcnt; struct indir *ip; int layer; rcnt = size; layer = 0; while (rcnt > NINDIR) { rcnt /= NINDIR; layer++; } /* * XXX: the top layer is probably not fully populated, so we allocate * too much space for ip->array in here. */ ip = malloc(sizeof *ip, M_MD, M_WAITOK | M_ZERO); ip->array = malloc(sizeof(uintptr_t) * NINDIR, M_MDSECT, M_WAITOK | M_ZERO); ip->total = NINDIR; ip->shift = layer * nshift; return (ip); } /* * Read a given sector */ static uintptr_t s_read(struct indir *ip, off_t offset) { struct indir *cip; int idx; uintptr_t up; if (md_debug > 1) printf("s_read(%jd)\n", (intmax_t)offset); up = 0; for (cip = ip; cip != NULL;) { if (cip->shift) { idx = (offset >> cip->shift) & NMASK; up = cip->array[idx]; cip = (struct indir *)up; continue; } idx = offset & NMASK; return (cip->array[idx]); } return (0); } /* * Write a given sector, prune the tree if the value is 0 */ static int s_write(struct indir *ip, off_t offset, uintptr_t ptr) { struct indir *cip, *lip[10]; int idx, li; uintptr_t up; if (md_debug > 1) printf("s_write(%jd, %p)\n", (intmax_t)offset, (void *)ptr); up = 0; li = 0; cip = ip; for (;;) { lip[li++] = cip; if (cip->shift) { idx = (offset >> cip->shift) & NMASK; up = cip->array[idx]; if (up != 0) { cip = (struct indir *)up; continue; } /* Allocate branch */ cip->array[idx] = (uintptr_t)new_indir(cip->shift - nshift); if (cip->array[idx] == 0) return (ENOSPC); cip->used++; up = cip->array[idx]; cip = (struct indir *)up; continue; } /* leafnode */ idx = offset & NMASK; up = cip->array[idx]; if (up != 0) cip->used--; cip->array[idx] = ptr; if (ptr != 0) cip->used++; break; } if (cip->used != 0 || li == 1) return (0); li--; while (cip->used == 0 && cip != ip) { li--; idx = (offset >> lip[li]->shift) & NMASK; up = lip[li]->array[idx]; KASSERT(up == (uintptr_t)cip, ("md screwed up")); del_indir(cip); lip[li]->array[idx] = 0; lip[li]->used--; cip = lip[li]; } return (0); } static int g_md_access(struct g_provider *pp, int r, int w, int e) { struct md_s *sc; sc = pp->geom->softc; if (sc == NULL) { if (r <= 0 && w <= 0 && e <= 0) return (0); return (ENXIO); } r += pp->acr; w += pp->acw; e += pp->ace; if ((sc->flags & MD_READONLY) != 0 && w > 0) return (EROFS); if ((pp->acr + pp->acw + pp->ace) == 0 && (r + w + e) > 0) { sc->opencount = 1; } else if ((pp->acr + pp->acw + pp->ace) > 0 && (r + w + e) == 0) { sc->opencount = 0; } return (0); } static void g_md_start(struct bio *bp) { struct md_s *sc; sc = bp->bio_to->geom->softc; if ((bp->bio_cmd == BIO_READ) || (bp->bio_cmd == BIO_WRITE)) { devstat_start_transaction_bio(sc->devstat, bp); } mtx_lock(&sc->queue_mtx); bioq_disksort(&sc->bio_queue, bp); wakeup(sc); mtx_unlock(&sc->queue_mtx); } #define MD_MALLOC_MOVE_ZERO 1 #define MD_MALLOC_MOVE_FILL 2 #define MD_MALLOC_MOVE_READ 3 #define MD_MALLOC_MOVE_WRITE 4 #define MD_MALLOC_MOVE_CMP 5 static int md_malloc_move_ma(vm_page_t **mp, int *ma_offs, unsigned sectorsize, void *ptr, u_char fill, int op) { struct sf_buf *sf; vm_page_t m, *mp1; char *p, first; off_t *uc; unsigned n; int error, i, ma_offs1, sz, first_read; m = NULL; error = 0; sf = NULL; /* if (op == MD_MALLOC_MOVE_CMP) { gcc */ first = 0; first_read = 0; uc = ptr; mp1 = *mp; ma_offs1 = *ma_offs; /* } */ sched_pin(); for (n = sectorsize; n != 0; n -= sz) { sz = imin(PAGE_SIZE - *ma_offs, n); if (m != **mp) { if (sf != NULL) sf_buf_free(sf); m = **mp; sf = sf_buf_alloc(m, SFB_CPUPRIVATE | (md_malloc_wait ? 0 : SFB_NOWAIT)); if (sf == NULL) { error = ENOMEM; break; } } p = (char *)sf_buf_kva(sf) + *ma_offs; switch (op) { case MD_MALLOC_MOVE_ZERO: bzero(p, sz); break; case MD_MALLOC_MOVE_FILL: memset(p, fill, sz); break; case MD_MALLOC_MOVE_READ: bcopy(ptr, p, sz); cpu_flush_dcache(p, sz); break; case MD_MALLOC_MOVE_WRITE: bcopy(p, ptr, sz); break; case MD_MALLOC_MOVE_CMP: for (i = 0; i < sz; i++, p++) { if (!first_read) { *uc = (u_char)*p; first = *p; first_read = 1; } else if (*p != first) { error = EDOOFUS; break; } } break; default: KASSERT(0, ("md_malloc_move_ma unknown op %d\n", op)); break; } if (error != 0) break; *ma_offs += sz; *ma_offs %= PAGE_SIZE; if (*ma_offs == 0) (*mp)++; ptr = (char *)ptr + sz; } if (sf != NULL) sf_buf_free(sf); sched_unpin(); if (op == MD_MALLOC_MOVE_CMP && error != 0) { *mp = mp1; *ma_offs = ma_offs1; } return (error); } static int md_malloc_move_vlist(bus_dma_segment_t **pvlist, int *pma_offs, unsigned len, void *ptr, u_char fill, int op) { bus_dma_segment_t *vlist; uint8_t *p, *end, first; off_t *uc; int ma_offs, seg_len; vlist = *pvlist; ma_offs = *pma_offs; uc = ptr; for (; len != 0; len -= seg_len) { seg_len = imin(vlist->ds_len - ma_offs, len); p = (uint8_t *)(uintptr_t)vlist->ds_addr + ma_offs; switch (op) { case MD_MALLOC_MOVE_ZERO: bzero(p, seg_len); break; case MD_MALLOC_MOVE_FILL: memset(p, fill, seg_len); break; case MD_MALLOC_MOVE_READ: bcopy(ptr, p, seg_len); cpu_flush_dcache(p, seg_len); break; case MD_MALLOC_MOVE_WRITE: bcopy(p, ptr, seg_len); break; case MD_MALLOC_MOVE_CMP: end = p + seg_len; first = *uc = *p; /* Confirm all following bytes match the first */ while (++p < end) { if (*p != first) return (EDOOFUS); } break; default: KASSERT(0, ("md_malloc_move_vlist unknown op %d\n", op)); break; } ma_offs += seg_len; if (ma_offs == vlist->ds_len) { ma_offs = 0; vlist++; } ptr = (uint8_t *)ptr + seg_len; } *pvlist = vlist; *pma_offs = ma_offs; return (0); } static int mdstart_malloc(struct md_s *sc, struct bio *bp) { u_char *dst; vm_page_t *m; bus_dma_segment_t *vlist; int i, error, error1, ma_offs, notmapped; off_t secno, nsec, uc; uintptr_t sp, osp; switch (bp->bio_cmd) { case BIO_READ: case BIO_WRITE: case BIO_DELETE: break; case BIO_FLUSH: return (0); default: return (EOPNOTSUPP); } notmapped = (bp->bio_flags & BIO_UNMAPPED) != 0; vlist = (bp->bio_flags & BIO_VLIST) != 0 ? (bus_dma_segment_t *)bp->bio_data : NULL; if (notmapped) { m = bp->bio_ma; ma_offs = bp->bio_ma_offset; dst = NULL; KASSERT(vlist == NULL, ("vlists cannot be unmapped")); } else if (vlist != NULL) { ma_offs = bp->bio_ma_offset; dst = NULL; } else { dst = bp->bio_data; } nsec = bp->bio_length / sc->sectorsize; secno = bp->bio_offset / sc->sectorsize; error = 0; while (nsec--) { osp = s_read(sc->indir, secno); if (bp->bio_cmd == BIO_DELETE) { if (osp != 0) error = s_write(sc->indir, secno, 0); } else if (bp->bio_cmd == BIO_READ) { if (osp == 0) { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, NULL, 0, MD_MALLOC_MOVE_ZERO); } else if (vlist != NULL) { error = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, NULL, 0, MD_MALLOC_MOVE_ZERO); } else bzero(dst, sc->sectorsize); } else if (osp <= 255) { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, NULL, osp, MD_MALLOC_MOVE_FILL); } else if (vlist != NULL) { error = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, NULL, osp, MD_MALLOC_MOVE_FILL); } else memset(dst, osp, sc->sectorsize); } else { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_READ); } else if (vlist != NULL) { error = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_READ); } else { bcopy((void *)osp, dst, sc->sectorsize); cpu_flush_dcache(dst, sc->sectorsize); } } osp = 0; } else if (bp->bio_cmd == BIO_WRITE) { if (sc->flags & MD_COMPRESS) { if (notmapped) { error1 = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, &uc, 0, MD_MALLOC_MOVE_CMP); i = error1 == 0 ? sc->sectorsize : 0; } else if (vlist != NULL) { error1 = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, &uc, 0, MD_MALLOC_MOVE_CMP); i = error1 == 0 ? sc->sectorsize : 0; } else { uc = dst[0]; for (i = 1; i < sc->sectorsize; i++) { if (dst[i] != uc) break; } } } else { i = 0; uc = 0; } if (i == sc->sectorsize) { if (osp != uc) error = s_write(sc->indir, secno, uc); } else { if (osp <= 255) { sp = (uintptr_t)uma_zalloc(sc->uma, md_malloc_wait ? M_WAITOK : M_NOWAIT); if (sp == 0) { error = ENOSPC; break; } if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)sp, 0, MD_MALLOC_MOVE_WRITE); } else if (vlist != NULL) { error = md_malloc_move_vlist( &vlist, &ma_offs, sc->sectorsize, (void *)sp, 0, MD_MALLOC_MOVE_WRITE); } else { bcopy(dst, (void *)sp, sc->sectorsize); } error = s_write(sc->indir, secno, sp); } else { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_WRITE); } else if (vlist != NULL) { error = md_malloc_move_vlist( &vlist, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_WRITE); } else { bcopy(dst, (void *)osp, sc->sectorsize); } osp = 0; } } } else { error = EOPNOTSUPP; } if (osp > 255) uma_zfree(sc->uma, (void*)osp); if (error != 0) break; secno++; if (!notmapped && vlist == NULL) dst += sc->sectorsize; } bp->bio_resid = 0; return (error); } static void mdcopyto_vlist(void *src, bus_dma_segment_t *vlist, off_t offset, off_t len) { off_t seg_len; while (offset >= vlist->ds_len) { offset -= vlist->ds_len; vlist++; } while (len != 0) { seg_len = omin(len, vlist->ds_len - offset); bcopy(src, (void *)(uintptr_t)(vlist->ds_addr + offset), seg_len); offset = 0; src = (uint8_t *)src + seg_len; len -= seg_len; vlist++; } } static void mdcopyfrom_vlist(bus_dma_segment_t *vlist, off_t offset, void *dst, off_t len) { off_t seg_len; while (offset >= vlist->ds_len) { offset -= vlist->ds_len; vlist++; } while (len != 0) { seg_len = omin(len, vlist->ds_len - offset); bcopy((void *)(uintptr_t)(vlist->ds_addr + offset), dst, seg_len); offset = 0; dst = (uint8_t *)dst + seg_len; len -= seg_len; vlist++; } } static int mdstart_preload(struct md_s *sc, struct bio *bp) { uint8_t *p; p = sc->pl_ptr + bp->bio_offset; switch (bp->bio_cmd) { case BIO_READ: if ((bp->bio_flags & BIO_VLIST) != 0) { mdcopyto_vlist(p, (bus_dma_segment_t *)bp->bio_data, bp->bio_ma_offset, bp->bio_length); } else { bcopy(p, bp->bio_data, bp->bio_length); } cpu_flush_dcache(bp->bio_data, bp->bio_length); break; case BIO_WRITE: if ((bp->bio_flags & BIO_VLIST) != 0) { mdcopyfrom_vlist((bus_dma_segment_t *)bp->bio_data, bp->bio_ma_offset, p, bp->bio_length); } else { bcopy(bp->bio_data, p, bp->bio_length); } break; } bp->bio_resid = 0; return (0); } static int mdstart_vnode(struct md_s *sc, struct bio *bp) { int error; struct uio auio; struct iovec aiov; struct iovec *piov; struct mount *mp; struct vnode *vp; bus_dma_segment_t *vlist; struct thread *td; off_t iolen, iostart, off, len; int ma_offs, npages; bool mapped; switch (bp->bio_cmd) { case BIO_READ: auio.uio_rw = UIO_READ; break; case BIO_WRITE: auio.uio_rw = UIO_WRITE; break; case BIO_FLUSH: break; case BIO_DELETE: if (sc->candelete) break; /* FALLTHROUGH */ default: return (EOPNOTSUPP); } td = curthread; vp = sc->vnode; piov = NULL; ma_offs = bp->bio_ma_offset; off = bp->bio_offset; len = bp->bio_length; mapped = false; /* * VNODE I/O * * If an error occurs, we set BIO_ERROR but we do not set * B_INVAL because (for a write anyway), the buffer is * still valid. */ if (bp->bio_cmd == BIO_FLUSH) { do { (void)vn_start_write(vp, &mp, V_WAIT); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_FSYNC(vp, MNT_WAIT, td); VOP_UNLOCK(vp); vn_finished_write(mp); } while (error == ERELOOKUP); return (error); } else if (bp->bio_cmd == BIO_DELETE) { error = vn_deallocate(vp, &off, &len, 0, sc->flags & MD_ASYNC ? 0 : IO_SYNC, sc->cred, NOCRED); bp->bio_resid = len; return (error); } auio.uio_offset = (vm_ooffset_t)bp->bio_offset; auio.uio_resid = bp->bio_length; auio.uio_segflg = UIO_SYSSPACE; auio.uio_td = td; if ((bp->bio_flags & BIO_VLIST) != 0) { piov = malloc(sizeof(*piov) * bp->bio_ma_n, M_MD, M_WAITOK); auio.uio_iov = piov; vlist = (bus_dma_segment_t *)bp->bio_data; while (len > 0) { piov->iov_base = (void *)(uintptr_t)(vlist->ds_addr + ma_offs); piov->iov_len = vlist->ds_len - ma_offs; if (piov->iov_len > len) piov->iov_len = len; len -= piov->iov_len; ma_offs = 0; vlist++; piov++; } auio.uio_iovcnt = piov - auio.uio_iov; piov = auio.uio_iov; } else if ((bp->bio_flags & BIO_UNMAPPED) != 0) { bp->bio_resid = len; unmapped_step: npages = atop(min(maxphys, round_page(len + (ma_offs & PAGE_MASK)))); iolen = min(ptoa(npages) - (ma_offs & PAGE_MASK), len); KASSERT(iolen > 0, ("zero iolen")); - KASSERT(npages <= atop(MAXPHYS + PAGE_SIZE), + KASSERT(npages <= atop(maxphys + PAGE_SIZE), ("npages %d too large", npages)); pmap_qenter(sc->kva, &bp->bio_ma[atop(ma_offs)], npages); aiov.iov_base = (void *)(sc->kva + (ma_offs & PAGE_MASK)); aiov.iov_len = iolen; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_resid = iolen; mapped = true; } else { aiov.iov_base = bp->bio_data; aiov.iov_len = bp->bio_length; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; } iostart = auio.uio_offset; if (auio.uio_rw == UIO_READ) { vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_READ(vp, &auio, 0, sc->cred); VOP_UNLOCK(vp); } else { (void) vn_start_write(vp, &mp, V_WAIT); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_WRITE(vp, &auio, sc->flags & MD_ASYNC ? 0 : IO_SYNC, sc->cred); VOP_UNLOCK(vp); vn_finished_write(mp); if (error == 0) sc->flags &= ~MD_VERIFY; } /* When MD_CACHE is set, try to avoid double-caching the data. */ if (error == 0 && (sc->flags & MD_CACHE) == 0) VOP_ADVISE(vp, iostart, auio.uio_offset - 1, POSIX_FADV_DONTNEED); if (mapped) { pmap_qremove(sc->kva, npages); if (error == 0) { len -= iolen; bp->bio_resid -= iolen; ma_offs += iolen; if (len > 0) goto unmapped_step; } } else { bp->bio_resid = auio.uio_resid; } free(piov, M_MD); return (error); } static int mdstart_swap(struct md_s *sc, struct bio *bp) { vm_page_t m; u_char *p; vm_pindex_t i, lastp; bus_dma_segment_t *vlist; int rv, ma_offs, offs, len, lastend; switch (bp->bio_cmd) { case BIO_READ: case BIO_WRITE: case BIO_DELETE: break; case BIO_FLUSH: return (0); default: return (EOPNOTSUPP); } p = bp->bio_data; ma_offs = (bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0 ? bp->bio_ma_offset : 0; vlist = (bp->bio_flags & BIO_VLIST) != 0 ? (bus_dma_segment_t *)bp->bio_data : NULL; /* * offs is the offset at which to start operating on the * next (ie, first) page. lastp is the last page on * which we're going to operate. lastend is the ending * position within that last page (ie, PAGE_SIZE if * we're operating on complete aligned pages). */ offs = bp->bio_offset % PAGE_SIZE; lastp = (bp->bio_offset + bp->bio_length - 1) / PAGE_SIZE; lastend = (bp->bio_offset + bp->bio_length - 1) % PAGE_SIZE + 1; rv = VM_PAGER_OK; vm_object_pip_add(sc->object, 1); for (i = bp->bio_offset / PAGE_SIZE; i <= lastp; i++) { len = ((i == lastp) ? lastend : PAGE_SIZE) - offs; m = vm_page_grab_unlocked(sc->object, i, VM_ALLOC_SYSTEM); if (bp->bio_cmd == BIO_READ) { if (vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, NULL, NULL); if (rv == VM_PAGER_ERROR) { VM_OBJECT_WLOCK(sc->object); vm_page_free(m); VM_OBJECT_WUNLOCK(sc->object); break; } else if (rv == VM_PAGER_FAIL) { /* * Pager does not have the page. Zero * the allocated page, and mark it as * valid. Do not set dirty, the page * can be recreated if thrown out. */ pmap_zero_page(m); vm_page_valid(m); } if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pmap_copy_pages(&m, offs, bp->bio_ma, ma_offs, len); } else if ((bp->bio_flags & BIO_VLIST) != 0) { physcopyout_vlist(VM_PAGE_TO_PHYS(m) + offs, vlist, ma_offs, len); cpu_flush_dcache(p, len); } else { physcopyout(VM_PAGE_TO_PHYS(m) + offs, p, len); cpu_flush_dcache(p, len); } } else if (bp->bio_cmd == BIO_WRITE) { if (len == PAGE_SIZE || vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, NULL, NULL); if (rv == VM_PAGER_ERROR) { VM_OBJECT_WLOCK(sc->object); vm_page_free(m); VM_OBJECT_WUNLOCK(sc->object); break; } else if (rv == VM_PAGER_FAIL) pmap_zero_page(m); if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pmap_copy_pages(bp->bio_ma, ma_offs, &m, offs, len); } else if ((bp->bio_flags & BIO_VLIST) != 0) { physcopyin_vlist(vlist, ma_offs, VM_PAGE_TO_PHYS(m) + offs, len); } else { physcopyin(p, VM_PAGE_TO_PHYS(m) + offs, len); } vm_page_valid(m); vm_page_set_dirty(m); } else if (bp->bio_cmd == BIO_DELETE) { if (len == PAGE_SIZE || vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, NULL, NULL); VM_OBJECT_WLOCK(sc->object); if (rv == VM_PAGER_ERROR) { vm_page_free(m); VM_OBJECT_WUNLOCK(sc->object); break; } else if (rv == VM_PAGER_FAIL) { vm_page_free(m); m = NULL; } else { /* Page is valid. */ if (len != PAGE_SIZE) { pmap_zero_page_area(m, offs, len); vm_page_set_dirty(m); } else { vm_pager_page_unswapped(m); vm_page_free(m); m = NULL; } } VM_OBJECT_WUNLOCK(sc->object); } if (m != NULL) { /* * The page may be deactivated prior to setting * PGA_REFERENCED, but in this case it will be * reactivated by the page daemon. */ if (vm_page_active(m)) vm_page_reference(m); else vm_page_activate(m); vm_page_xunbusy(m); } /* Actions on further pages start at offset 0 */ p += PAGE_SIZE - offs; offs = 0; ma_offs += len; } vm_object_pip_wakeup(sc->object); return (rv != VM_PAGER_ERROR ? 0 : ENOSPC); } static int mdstart_null(struct md_s *sc, struct bio *bp) { switch (bp->bio_cmd) { case BIO_READ: bzero(bp->bio_data, bp->bio_length); cpu_flush_dcache(bp->bio_data, bp->bio_length); break; case BIO_WRITE: break; } bp->bio_resid = 0; return (0); } static void md_handleattr(struct md_s *sc, struct bio *bp) { if (sc->fwsectors && sc->fwheads && (g_handleattr_int(bp, "GEOM::fwsectors", sc->fwsectors) != 0 || g_handleattr_int(bp, "GEOM::fwheads", sc->fwheads) != 0)) return; if (g_handleattr_int(bp, "GEOM::candelete", sc->candelete) != 0) return; if (sc->ident[0] != '\0' && g_handleattr_str(bp, "GEOM::ident", sc->ident) != 0) return; if (g_handleattr_int(bp, "MNT::verified", (sc->flags & MD_VERIFY) != 0)) return; g_io_deliver(bp, EOPNOTSUPP); } static void md_kthread(void *arg) { struct md_s *sc; struct bio *bp; int error; sc = arg; thread_lock(curthread); sched_prio(curthread, PRIBIO); thread_unlock(curthread); if (sc->type == MD_VNODE) curthread->td_pflags |= TDP_NORUNNINGBUF; for (;;) { mtx_lock(&sc->queue_mtx); if (sc->flags & MD_SHUTDOWN) { sc->flags |= MD_EXITING; mtx_unlock(&sc->queue_mtx); kproc_exit(0); } bp = bioq_takefirst(&sc->bio_queue); if (!bp) { msleep(sc, &sc->queue_mtx, PRIBIO | PDROP, "mdwait", 0); continue; } mtx_unlock(&sc->queue_mtx); if (bp->bio_cmd == BIO_GETATTR) { md_handleattr(sc, bp); } else { error = sc->start(sc, bp); if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { /* * Devstat uses (bio_bcount, bio_resid) for * determining the length of the completed part * of the i/o. g_io_deliver() will translate * from bio_completed to that, but it also * destroys the bio so we must do our own * translation. */ bp->bio_bcount = bp->bio_length; devstat_end_transaction_bio(sc->devstat, bp); } bp->bio_completed = bp->bio_length - bp->bio_resid; g_io_deliver(bp, error); } } } static struct md_s * mdfind(int unit) { struct md_s *sc; LIST_FOREACH(sc, &md_softc_list, list) { if (sc->unit == unit) break; } return (sc); } static struct md_s * mdnew(int unit, int *errp, enum md_types type) { struct md_s *sc; int error; *errp = 0; if (unit == -1) unit = alloc_unr(md_uh); else unit = alloc_unr_specific(md_uh, unit); if (unit == -1) { *errp = EBUSY; return (NULL); } sc = malloc(sizeof(*sc), M_MD, M_WAITOK | M_ZERO); sc->type = type; bioq_init(&sc->bio_queue); mtx_init(&sc->queue_mtx, "md bio queue", NULL, MTX_DEF); sc->unit = unit; sprintf(sc->name, "md%d", unit); LIST_INSERT_HEAD(&md_softc_list, sc, list); error = kproc_create(md_kthread, sc, &sc->procp, 0, 0,"%s", sc->name); if (error == 0) return (sc); LIST_REMOVE(sc, list); mtx_destroy(&sc->queue_mtx); free_unr(md_uh, sc->unit); free(sc, M_MD); *errp = error; return (NULL); } static void mdinit(struct md_s *sc) { struct g_geom *gp; struct g_provider *pp; g_topology_lock(); gp = g_new_geomf(&g_md_class, "md%d", sc->unit); gp->softc = sc; pp = g_new_providerf(gp, "md%d", sc->unit); devstat_remove_entry(pp->stat); pp->stat = NULL; pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE; pp->mediasize = sc->mediasize; pp->sectorsize = sc->sectorsize; switch (sc->type) { case MD_MALLOC: case MD_VNODE: case MD_SWAP: pp->flags |= G_PF_ACCEPT_UNMAPPED; break; case MD_PRELOAD: case MD_NULL: break; } sc->gp = gp; sc->pp = pp; sc->devstat = devstat_new_entry("md", sc->unit, sc->sectorsize, DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT, DEVSTAT_PRIORITY_MAX); sc->devstat->id = pp; g_error_provider(pp, 0); g_topology_unlock(); } static int mdcreate_malloc(struct md_s *sc, struct md_req *mdr) { uintptr_t sp; int error; off_t u; error = 0; if (mdr->md_options & ~(MD_AUTOUNIT | MD_COMPRESS | MD_RESERVE)) return (EINVAL); if (mdr->md_sectorsize != 0 && !powerof2(mdr->md_sectorsize)) return (EINVAL); /* Compression doesn't make sense if we have reserved space */ if (mdr->md_options & MD_RESERVE) mdr->md_options &= ~MD_COMPRESS; if (mdr->md_fwsectors != 0) sc->fwsectors = mdr->md_fwsectors; if (mdr->md_fwheads != 0) sc->fwheads = mdr->md_fwheads; sc->flags = mdr->md_options & (MD_COMPRESS | MD_FORCE); sc->indir = dimension(sc->mediasize / sc->sectorsize); sc->uma = uma_zcreate(sc->name, sc->sectorsize, NULL, NULL, NULL, NULL, 0x1ff, 0); if (mdr->md_options & MD_RESERVE) { off_t nsectors; nsectors = sc->mediasize / sc->sectorsize; for (u = 0; u < nsectors; u++) { sp = (uintptr_t)uma_zalloc(sc->uma, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); if (sp != 0) error = s_write(sc->indir, u, sp); else error = ENOMEM; if (error != 0) break; } } return (error); } static int mdsetcred(struct md_s *sc, struct ucred *cred) { char *tmpbuf; int error = 0; /* * Set credits in our softc */ if (sc->cred) crfree(sc->cred); sc->cred = crhold(cred); /* * Horrible kludge to establish credentials for NFS XXX. */ if (sc->vnode) { struct uio auio; struct iovec aiov; tmpbuf = malloc(sc->sectorsize, M_TEMP, M_WAITOK); bzero(&auio, sizeof(auio)); aiov.iov_base = tmpbuf; aiov.iov_len = sc->sectorsize; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = 0; auio.uio_rw = UIO_READ; auio.uio_segflg = UIO_SYSSPACE; auio.uio_resid = aiov.iov_len; vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); error = VOP_READ(sc->vnode, &auio, 0, sc->cred); VOP_UNLOCK(sc->vnode); free(tmpbuf, M_TEMP); } return (error); } static int mdcreate_vnode(struct md_s *sc, struct md_req *mdr, struct thread *td) { struct vattr vattr; struct nameidata nd; char *fname; int error, flags; long v; fname = mdr->md_file; if (mdr->md_file_seg == UIO_USERSPACE) { error = copyinstr(fname, sc->file, sizeof(sc->file), NULL); if (error != 0) return (error); } else if (mdr->md_file_seg == UIO_SYSSPACE) strlcpy(sc->file, fname, sizeof(sc->file)); else return (EDOOFUS); /* * If the user specified that this is a read only device, don't * set the FWRITE mask before trying to open the backing store. */ flags = FREAD | ((mdr->md_options & MD_READONLY) ? 0 : FWRITE) \ | ((mdr->md_options & MD_VERIFY) ? O_VERIFY : 0); NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->file); error = vn_open(&nd, &flags, 0, NULL); if (error != 0) return (error); NDFREE_PNBUF(&nd); if (nd.ni_vp->v_type != VREG) { error = EINVAL; goto bad; } error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); if (error != 0) goto bad; if ((mdr->md_options & MD_MUSTDEALLOC) != 0) { error = VOP_PATHCONF(nd.ni_vp, _PC_DEALLOC_PRESENT, &v); if (error != 0) goto bad; if (v == 0) sc->candelete = false; } if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); if (VN_IS_DOOMED(nd.ni_vp)) { /* Forced unmount. */ error = EBADF; goto bad; } } nd.ni_vp->v_vflag |= VV_MD; VOP_UNLOCK(nd.ni_vp); if (mdr->md_fwsectors != 0) sc->fwsectors = mdr->md_fwsectors; if (mdr->md_fwheads != 0) sc->fwheads = mdr->md_fwheads; snprintf(sc->ident, sizeof(sc->ident), "MD-DEV%ju-INO%ju", (uintmax_t)vattr.va_fsid, (uintmax_t)vattr.va_fileid); sc->flags = mdr->md_options & (MD_ASYNC | MD_CACHE | MD_FORCE | MD_VERIFY); if (!(flags & FWRITE)) sc->flags |= MD_READONLY; sc->vnode = nd.ni_vp; error = mdsetcred(sc, td->td_ucred); if (error != 0) { sc->vnode = NULL; vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); nd.ni_vp->v_vflag &= ~VV_MD; goto bad; } - sc->kva = kva_alloc(MAXPHYS + PAGE_SIZE); + sc->kva = kva_alloc(maxphys + PAGE_SIZE); return (0); bad: VOP_UNLOCK(nd.ni_vp); (void)vn_close(nd.ni_vp, flags, td->td_ucred, td); return (error); } static void g_md_providergone(struct g_provider *pp) { struct md_s *sc = pp->geom->softc; mtx_lock(&sc->queue_mtx); sc->flags |= MD_PROVIDERGONE; wakeup(&sc->flags); mtx_unlock(&sc->queue_mtx); } static int mddestroy(struct md_s *sc, struct thread *td) { if (sc->gp) { g_topology_lock(); g_wither_geom(sc->gp, ENXIO); g_topology_unlock(); mtx_lock(&sc->queue_mtx); while (!(sc->flags & MD_PROVIDERGONE)) msleep(&sc->flags, &sc->queue_mtx, PRIBIO, "mddestroy", 0); mtx_unlock(&sc->queue_mtx); } if (sc->devstat) { devstat_remove_entry(sc->devstat); sc->devstat = NULL; } mtx_lock(&sc->queue_mtx); sc->flags |= MD_SHUTDOWN; wakeup(sc); while (!(sc->flags & MD_EXITING)) msleep(sc->procp, &sc->queue_mtx, PRIBIO, "mddestroy", hz / 10); mtx_unlock(&sc->queue_mtx); mtx_destroy(&sc->queue_mtx); if (sc->vnode != NULL) { vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); sc->vnode->v_vflag &= ~VV_MD; VOP_UNLOCK(sc->vnode); (void)vn_close(sc->vnode, sc->flags & MD_READONLY ? FREAD : (FREAD|FWRITE), sc->cred, td); } if (sc->cred != NULL) crfree(sc->cred); if (sc->object != NULL) vm_object_deallocate(sc->object); if (sc->indir) destroy_indir(sc, sc->indir); if (sc->uma) uma_zdestroy(sc->uma); if (sc->kva) - kva_free(sc->kva, MAXPHYS + PAGE_SIZE); + kva_free(sc->kva, maxphys + PAGE_SIZE); LIST_REMOVE(sc, list); free_unr(md_uh, sc->unit); free(sc, M_MD); return (0); } static int mdresize(struct md_s *sc, struct md_req *mdr) { int error, res; vm_pindex_t oldpages, newpages; switch (sc->type) { case MD_VNODE: case MD_NULL: break; case MD_SWAP: if (mdr->md_mediasize <= 0 || (mdr->md_mediasize % PAGE_SIZE) != 0) return (EDOM); oldpages = OFF_TO_IDX(sc->mediasize); newpages = OFF_TO_IDX(mdr->md_mediasize); if (newpages < oldpages) { VM_OBJECT_WLOCK(sc->object); vm_object_page_remove(sc->object, newpages, 0, 0); swap_release_by_cred(IDX_TO_OFF(oldpages - newpages), sc->cred); sc->object->charge = IDX_TO_OFF(newpages); sc->object->size = newpages; VM_OBJECT_WUNLOCK(sc->object); } else if (newpages > oldpages) { res = swap_reserve_by_cred(IDX_TO_OFF(newpages - oldpages), sc->cred); if (!res) return (ENOMEM); if ((mdr->md_options & MD_RESERVE) || (sc->flags & MD_RESERVE)) { error = swap_pager_reserve(sc->object, oldpages, newpages - oldpages); if (error < 0) { swap_release_by_cred( IDX_TO_OFF(newpages - oldpages), sc->cred); return (EDOM); } } VM_OBJECT_WLOCK(sc->object); sc->object->charge = IDX_TO_OFF(newpages); sc->object->size = newpages; VM_OBJECT_WUNLOCK(sc->object); } break; default: return (EOPNOTSUPP); } sc->mediasize = mdr->md_mediasize; g_topology_lock(); g_resize_provider(sc->pp, sc->mediasize); g_topology_unlock(); return (0); } static int mdcreate_swap(struct md_s *sc, struct md_req *mdr, struct thread *td) { vm_ooffset_t npage; int error; /* * Range check. Disallow negative sizes and sizes not being * multiple of page size. */ if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) return (EDOM); /* * Allocate an OBJT_SWAP object. * * Note the truncation. */ if ((mdr->md_options & MD_VERIFY) != 0) return (EINVAL); npage = mdr->md_mediasize / PAGE_SIZE; if (mdr->md_fwsectors != 0) sc->fwsectors = mdr->md_fwsectors; if (mdr->md_fwheads != 0) sc->fwheads = mdr->md_fwheads; sc->object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * npage, VM_PROT_DEFAULT, 0, td->td_ucred); if (sc->object == NULL) return (ENOMEM); sc->flags = mdr->md_options & (MD_FORCE | MD_RESERVE); if (mdr->md_options & MD_RESERVE) { if (swap_pager_reserve(sc->object, 0, npage) < 0) { error = EDOM; goto finish; } } error = mdsetcred(sc, td->td_ucred); finish: if (error != 0) { vm_object_deallocate(sc->object); sc->object = NULL; } return (error); } static int mdcreate_null(struct md_s *sc, struct md_req *mdr, struct thread *td) { /* * Range check. Disallow negative sizes and sizes not being * multiple of page size. */ if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) return (EDOM); return (0); } static int kern_mdattach_locked(struct thread *td, struct md_req *mdr) { struct md_s *sc; unsigned sectsize; int error, i; sx_assert(&md_sx, SA_XLOCKED); switch (mdr->md_type) { case MD_MALLOC: case MD_PRELOAD: case MD_VNODE: case MD_SWAP: case MD_NULL: break; default: return (EINVAL); } if (mdr->md_sectorsize == 0) sectsize = DEV_BSIZE; else sectsize = mdr->md_sectorsize; if (sectsize > maxphys || mdr->md_mediasize < sectsize) return (EINVAL); if (mdr->md_options & MD_AUTOUNIT) sc = mdnew(-1, &error, mdr->md_type); else { if (mdr->md_unit > INT_MAX) return (EINVAL); sc = mdnew(mdr->md_unit, &error, mdr->md_type); } if (sc == NULL) return (error); if (mdr->md_label != NULL) error = copyinstr(mdr->md_label, sc->label, sizeof(sc->label), NULL); if (error != 0) goto err_after_new; if (mdr->md_options & MD_AUTOUNIT) mdr->md_unit = sc->unit; sc->mediasize = mdr->md_mediasize; sc->sectorsize = sectsize; sc->candelete = true; error = EDOOFUS; switch (sc->type) { case MD_MALLOC: sc->start = mdstart_malloc; error = mdcreate_malloc(sc, mdr); break; case MD_PRELOAD: /* * We disallow attaching preloaded memory disks via * ioctl. Preloaded memory disks are automatically * attached in g_md_init(). */ error = EOPNOTSUPP; break; case MD_VNODE: sc->start = mdstart_vnode; error = mdcreate_vnode(sc, mdr, td); break; case MD_SWAP: sc->start = mdstart_swap; error = mdcreate_swap(sc, mdr, td); break; case MD_NULL: sc->start = mdstart_null; error = mdcreate_null(sc, mdr, td); break; } err_after_new: if (error != 0) { mddestroy(sc, td); return (error); } /* Prune off any residual fractional sector */ i = sc->mediasize % sc->sectorsize; sc->mediasize -= i; mdinit(sc); return (0); } static int kern_mdattach(struct thread *td, struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mdattach_locked(td, mdr); sx_xunlock(&md_sx); return (error); } static int kern_mddetach_locked(struct thread *td, struct md_req *mdr) { struct md_s *sc; sx_assert(&md_sx, SA_XLOCKED); if (mdr->md_mediasize != 0 || (mdr->md_options & ~MD_FORCE) != 0) return (EINVAL); sc = mdfind(mdr->md_unit); if (sc == NULL) return (ENOENT); if (sc->opencount != 0 && !(sc->flags & MD_FORCE) && !(mdr->md_options & MD_FORCE)) return (EBUSY); return (mddestroy(sc, td)); } static int kern_mddetach(struct thread *td, struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mddetach_locked(td, mdr); sx_xunlock(&md_sx); return (error); } static int kern_mdresize_locked(struct md_req *mdr) { struct md_s *sc; sx_assert(&md_sx, SA_XLOCKED); if ((mdr->md_options & ~(MD_FORCE | MD_RESERVE)) != 0) return (EINVAL); sc = mdfind(mdr->md_unit); if (sc == NULL) return (ENOENT); if (mdr->md_mediasize < sc->sectorsize) return (EINVAL); mdr->md_mediasize -= mdr->md_mediasize % sc->sectorsize; if (mdr->md_mediasize < sc->mediasize && !(sc->flags & MD_FORCE) && !(mdr->md_options & MD_FORCE)) return (EBUSY); return (mdresize(sc, mdr)); } static int kern_mdresize(struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mdresize_locked(mdr); sx_xunlock(&md_sx); return (error); } static int kern_mdquery_locked(struct md_req *mdr) { struct md_s *sc; int error; sx_assert(&md_sx, SA_XLOCKED); sc = mdfind(mdr->md_unit); if (sc == NULL) return (ENOENT); mdr->md_type = sc->type; mdr->md_options = sc->flags; mdr->md_mediasize = sc->mediasize; mdr->md_sectorsize = sc->sectorsize; error = 0; if (mdr->md_label != NULL) { error = copyout(sc->label, mdr->md_label, strlen(sc->label) + 1); if (error != 0) return (error); } if (sc->type == MD_VNODE || (sc->type == MD_PRELOAD && mdr->md_file != NULL)) error = copyout(sc->file, mdr->md_file, strlen(sc->file) + 1); return (error); } static int kern_mdquery(struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mdquery_locked(mdr); sx_xunlock(&md_sx); return (error); } /* Copy members that are not userspace pointers. */ #define MD_IOCTL2REQ(mdio, mdr) do { \ (mdr)->md_unit = (mdio)->md_unit; \ (mdr)->md_type = (mdio)->md_type; \ (mdr)->md_mediasize = (mdio)->md_mediasize; \ (mdr)->md_sectorsize = (mdio)->md_sectorsize; \ (mdr)->md_options = (mdio)->md_options; \ (mdr)->md_fwheads = (mdio)->md_fwheads; \ (mdr)->md_fwsectors = (mdio)->md_fwsectors; \ (mdr)->md_units = &(mdio)->md_pad[0]; \ (mdr)->md_units_nitems = nitems((mdio)->md_pad); \ } while(0) /* Copy members that might have been updated */ #define MD_REQ2IOCTL(mdr, mdio) do { \ (mdio)->md_unit = (mdr)->md_unit; \ (mdio)->md_type = (mdr)->md_type; \ (mdio)->md_mediasize = (mdr)->md_mediasize; \ (mdio)->md_sectorsize = (mdr)->md_sectorsize; \ (mdio)->md_options = (mdr)->md_options; \ (mdio)->md_fwheads = (mdr)->md_fwheads; \ (mdio)->md_fwsectors = (mdr)->md_fwsectors; \ } while(0) static int mdctlioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) { struct md_req mdr; int error; if (md_debug) printf("mdctlioctl(%s %lx %p %x %p)\n", devtoname(dev), cmd, addr, flags, td); bzero(&mdr, sizeof(mdr)); switch (cmd) { case MDIOCATTACH: case MDIOCDETACH: case MDIOCRESIZE: case MDIOCQUERY: { struct md_ioctl *mdio = (struct md_ioctl *)addr; if (mdio->md_version != MDIOVERSION) return (EINVAL); MD_IOCTL2REQ(mdio, &mdr); mdr.md_file = mdio->md_file; mdr.md_file_seg = UIO_USERSPACE; /* If the file is adjacent to the md_ioctl it's in kernel. */ if ((void *)mdio->md_file == (void *)(mdio + 1)) mdr.md_file_seg = UIO_SYSSPACE; mdr.md_label = mdio->md_label; break; } #ifdef COMPAT_FREEBSD32 case MDIOCATTACH_32: case MDIOCDETACH_32: case MDIOCRESIZE_32: case MDIOCQUERY_32: { struct md_ioctl32 *mdio = (struct md_ioctl32 *)addr; if (mdio->md_version != MDIOVERSION) return (EINVAL); MD_IOCTL2REQ(mdio, &mdr); mdr.md_file = (void *)(uintptr_t)mdio->md_file; mdr.md_file_seg = UIO_USERSPACE; mdr.md_label = (void *)(uintptr_t)mdio->md_label; break; } #endif default: /* Fall through to handler switch. */ break; } error = 0; switch (cmd) { case MDIOCATTACH: #ifdef COMPAT_FREEBSD32 case MDIOCATTACH_32: #endif error = kern_mdattach(td, &mdr); break; case MDIOCDETACH: #ifdef COMPAT_FREEBSD32 case MDIOCDETACH_32: #endif error = kern_mddetach(td, &mdr); break; case MDIOCRESIZE: #ifdef COMPAT_FREEBSD32 case MDIOCRESIZE_32: #endif error = kern_mdresize(&mdr); break; case MDIOCQUERY: #ifdef COMPAT_FREEBSD32 case MDIOCQUERY_32: #endif error = kern_mdquery(&mdr); break; default: error = ENOIOCTL; } switch (cmd) { case MDIOCATTACH: case MDIOCQUERY: { struct md_ioctl *mdio = (struct md_ioctl *)addr; MD_REQ2IOCTL(&mdr, mdio); break; } #ifdef COMPAT_FREEBSD32 case MDIOCATTACH_32: case MDIOCQUERY_32: { struct md_ioctl32 *mdio = (struct md_ioctl32 *)addr; MD_REQ2IOCTL(&mdr, mdio); break; } #endif default: /* Other commands to not alter mdr. */ break; } return (error); } static void md_preloaded(u_char *image, size_t length, const char *name) { struct md_s *sc; int error; sc = mdnew(-1, &error, MD_PRELOAD); if (sc == NULL) return; sc->mediasize = length; sc->sectorsize = DEV_BSIZE; sc->pl_ptr = image; sc->pl_len = length; sc->start = mdstart_preload; if (name != NULL) strlcpy(sc->file, name, sizeof(sc->file)); #ifdef MD_ROOT if (sc->unit == 0) { #ifndef ROOTDEVNAME rootdevnames[0] = MD_ROOT_FSTYPE ":/dev/md0"; #endif #ifdef MD_ROOT_READONLY sc->flags |= MD_READONLY; #endif } #endif mdinit(sc); if (name != NULL) { printf("%s%d: Preloaded image <%s> %zd bytes at %p\n", MD_NAME, sc->unit, name, length, image); } else { printf("%s%d: Embedded image %zd bytes at %p\n", MD_NAME, sc->unit, length, image); } } static void g_md_init(struct g_class *mp __unused) { caddr_t mod; u_char *ptr, *name, *type; unsigned len; int i; /* figure out log2(NINDIR) */ for (i = NINDIR, nshift = -1; i; nshift++) i >>= 1; mod = NULL; sx_init(&md_sx, "MD config lock"); g_topology_unlock(); md_uh = new_unrhdr(0, INT_MAX, NULL); #ifdef MD_ROOT if (mfs_root_size != 0) { sx_xlock(&md_sx); #ifdef MD_ROOT_MEM md_preloaded(mfs_root, mfs_root_size, NULL); #else md_preloaded(__DEVOLATILE(u_char *, &mfs_root), mfs_root_size, NULL); #endif sx_xunlock(&md_sx); } #endif /* XXX: are preload_* static or do they need Giant ? */ while ((mod = preload_search_next_name(mod)) != NULL) { name = (char *)preload_search_info(mod, MODINFO_NAME); if (name == NULL) continue; type = (char *)preload_search_info(mod, MODINFO_TYPE); if (type == NULL) continue; if (strcmp(type, "md_image") && strcmp(type, "mfs_root")) continue; ptr = preload_fetch_addr(mod); len = preload_fetch_size(mod); if (ptr != NULL && len != 0) { sx_xlock(&md_sx); md_preloaded(ptr, len, name); sx_xunlock(&md_sx); } } status_dev = make_dev(&mdctl_cdevsw, INT_MAX, UID_ROOT, GID_WHEEL, 0600, MDCTL_NAME); g_topology_lock(); } static void g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, struct g_consumer *cp __unused, struct g_provider *pp) { struct md_s *mp; char *type; mp = gp->softc; if (mp == NULL) return; switch (mp->type) { case MD_MALLOC: type = "malloc"; break; case MD_PRELOAD: type = "preload"; break; case MD_VNODE: type = "vnode"; break; case MD_SWAP: type = "swap"; break; case MD_NULL: type = "null"; break; default: type = "unknown"; break; } if (pp != NULL) { if (indent == NULL) { sbuf_printf(sb, " u %d", mp->unit); sbuf_printf(sb, " s %ju", (uintmax_t) mp->sectorsize); sbuf_printf(sb, " f %ju", (uintmax_t) mp->fwheads); sbuf_printf(sb, " fs %ju", (uintmax_t) mp->fwsectors); sbuf_printf(sb, " l %ju", (uintmax_t) mp->mediasize); sbuf_printf(sb, " t %s", type); if ((mp->type == MD_VNODE && mp->vnode != NULL) || (mp->type == MD_PRELOAD && mp->file[0] != '\0')) sbuf_printf(sb, " file %s", mp->file); sbuf_printf(sb, " label %s", mp->label); } else { sbuf_printf(sb, "%s%d\n", indent, mp->unit); sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->sectorsize); sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->fwheads); sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->fwsectors); if (mp->ident[0] != '\0') { sbuf_printf(sb, "%s", indent); g_conf_printf_escaped(sb, "%s", mp->ident); sbuf_printf(sb, "\n"); } sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->mediasize); sbuf_printf(sb, "%s%s\n", indent, (mp->flags & MD_COMPRESS) == 0 ? "off": "on"); sbuf_printf(sb, "%s%s\n", indent, (mp->flags & MD_READONLY) == 0 ? "read-write": "read-only"); sbuf_printf(sb, "%s%s\n", indent, type); if ((mp->type == MD_VNODE && mp->vnode != NULL) || (mp->type == MD_PRELOAD && mp->file[0] != '\0')) { sbuf_printf(sb, "%s", indent); g_conf_printf_escaped(sb, "%s", mp->file); sbuf_printf(sb, "\n"); } if (mp->type == MD_VNODE) sbuf_printf(sb, "%s%s\n", indent, (mp->flags & MD_CACHE) == 0 ? "off": "on"); sbuf_printf(sb, "%s\n"); } } } static void g_md_fini(struct g_class *mp __unused) { sx_destroy(&md_sx); if (status_dev != NULL) destroy_dev(status_dev); delete_unrhdr(md_uh); } diff --git a/sys/dev/rtsx/rtsx.c b/sys/dev/rtsx/rtsx.c index 464a155e66c2..a2f124f6c30d 100644 --- a/sys/dev/rtsx/rtsx.c +++ b/sys/dev/rtsx/rtsx.c @@ -1,3910 +1,3910 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006 Uwe Stuehler * Copyright (c) 2012 Stefan Sperling * Copyright (c) 2020 Henri Hennebert * Copyright (c) 2020 Gary Jennejohn * Copyright (c) 2020 Jesper Schmitz Mouridsen * All rights reserved. * * Patch from: * - Lutz Bichler * * Base on OpenBSD /sys/dev/pci/rtsx_pci.c & /dev/ic/rtsx.c * on Linux /drivers/mmc/host/rtsx_pci_sdmmc.c, * /include/linux/rtsx_pci.h & * /drivers/misc/cardreader/rtsx_pcr.c * on NetBSD /sys/dev/ic/rtsx.c * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include /* For FreeBSD 11 */ #include /* For FreeBSD 11 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "opt_mmccam.h" #ifdef MMCCAM #include #include #include #include #include #include #include "mmc_sim_if.h" #endif /* MMCCAM */ #include "rtsxreg.h" /* The softc holds our per-instance data. */ struct rtsx_softc { struct mtx rtsx_mtx; /* device mutex */ device_t rtsx_dev; /* device */ uint16_t rtsx_flags; /* device flags */ uint16_t rtsx_device_id; /* device ID */ device_t rtsx_mmc_dev; /* device of mmc bus */ uint32_t rtsx_intr_enabled; /* enabled interrupts */ uint32_t rtsx_intr_status; /* soft interrupt status */ int rtsx_irq_res_id; /* bus IRQ resource id */ struct resource *rtsx_irq_res; /* bus IRQ resource */ void *rtsx_irq_cookie; /* bus IRQ resource cookie */ struct callout rtsx_timeout_callout; /* callout for timeout */ int rtsx_timeout_cmd; /* interrupt timeout for setup commands */ int rtsx_timeout_io; /* interrupt timeout for I/O commands */ void (*rtsx_intr_trans_ok)(struct rtsx_softc *sc); /* function to call if transfer succeed */ void (*rtsx_intr_trans_ko)(struct rtsx_softc *sc); /* function to call if transfer fail */ struct timeout_task rtsx_card_insert_task; /* card insert delayed task */ struct task rtsx_card_remove_task; /* card remove task */ int rtsx_mem_res_id; /* bus memory resource id */ struct resource *rtsx_mem_res; /* bus memory resource */ bus_space_tag_t rtsx_mem_btag; /* host register set tag */ bus_space_handle_t rtsx_mem_bhandle; /* host register set handle */ bus_dma_tag_t rtsx_cmd_dma_tag; /* DMA tag for command transfer */ bus_dmamap_t rtsx_cmd_dmamap; /* DMA map for command transfer */ void *rtsx_cmd_dmamem; /* DMA mem for command transfer */ bus_addr_t rtsx_cmd_buffer; /* device visible address of the DMA segment */ int rtsx_cmd_index; /* index in rtsx_cmd_buffer */ bus_dma_tag_t rtsx_data_dma_tag; /* DMA tag for data transfer */ bus_dmamap_t rtsx_data_dmamap; /* DMA map for data transfer */ void *rtsx_data_dmamem; /* DMA mem for data transfer */ bus_addr_t rtsx_data_buffer; /* device visible address of the DMA segment */ #ifdef MMCCAM union ccb *rtsx_ccb; /* CAM control block */ struct mmc_sim rtsx_mmc_sim; /* CAM generic sim */ struct mmc_request rtsx_cam_req; /* CAM MMC request */ #endif /* MMCCAM */ struct mmc_request *rtsx_req; /* MMC request */ struct mmc_host rtsx_host; /* host parameters */ int rtsx_pcie_cap; /* PCIe capability offset */ int8_t rtsx_bus_busy; /* bus busy status */ int8_t rtsx_ios_bus_width; /* current host.ios.bus_width */ int32_t rtsx_ios_clock; /* current host.ios.clock */ int8_t rtsx_ios_power_mode; /* current host.ios.power mode */ int8_t rtsx_ios_timing; /* current host.ios.timing */ int8_t rtsx_ios_vccq; /* current host.ios.vccq */ uint8_t rtsx_read_only; /* card read only status */ uint8_t rtsx_inversion; /* inversion of card detection and read only status */ uint8_t rtsx_force_timing; /* force bus_timing_uhs_sdr50 */ uint8_t rtsx_debug_mask; /* debugging mask */ #define RTSX_DEBUG_BASIC 0x01 /* debug basic flow */ #define RTSX_TRACE_SD_CMD 0x02 /* trace SD commands */ #define RTSX_DEBUG_TUNING 0x04 /* debug tuning */ #ifdef MMCCAM uint8_t rtsx_cam_status; /* CAM status - 1 if card in use */ #endif /* MMCCAM */ uint64_t rtsx_read_count; /* count of read operations */ uint64_t rtsx_write_count; /* count of write operations */ bool rtsx_discovery_mode; /* are we in discovery mode? */ bool rtsx_tuning_mode; /* are we tuning */ bool rtsx_double_clk; /* double clock freqency */ bool rtsx_vpclk; /* voltage at Pulse-width Modulation(PWM) clock? */ uint8_t rtsx_ssc_depth; /* Spread spectrum clocking depth */ uint8_t rtsx_card_drive_sel; /* value for RTSX_CARD_DRIVE_SEL */ uint8_t rtsx_sd30_drive_sel_3v3;/* value for RTSX_SD30_DRIVE_SEL */ }; /* rtsx_flags values */ #define RTSX_F_DEFAULT 0x0000 #define RTSX_F_CARD_PRESENT 0x0001 #define RTSX_F_SDIO_SUPPORT 0x0002 #define RTSX_F_VERSION_A 0x0004 #define RTSX_F_VERSION_B 0x0008 #define RTSX_F_VERSION_C 0x0010 #define RTSX_F_VERSION_D 0x0020 #define RTSX_F_8411B_QFN48 0x0040 #define RTSX_F_REVERSE_SOCKET 0x0080 #define RTSX_REALTEK 0x10ec #define RTSX_RTS5209 0x5209 #define RTSX_RTS5227 0x5227 #define RTSX_RTS5229 0x5229 #define RTSX_RTS522A 0x522a #define RTSX_RTS525A 0x525a #define RTSX_RTS5249 0x5249 #define RTSX_RTS5260 0x5260 #define RTSX_RTL8402 0x5286 #define RTSX_RTL8411 0x5289 #define RTSX_RTL8411B 0x5287 #define RTSX_VERSION "2.1g" static const struct rtsx_pciids { uint16_t device_id; const char *desc; } rtsx_ids[] = { { RTSX_RTS5209, RTSX_VERSION " Realtek RTS5209 PCIe SD Card Reader" }, { RTSX_RTS5227, RTSX_VERSION " Realtek RTS5227 PCIe SD Card Reader" }, { RTSX_RTS5229, RTSX_VERSION " Realtek RTS5229 PCIe SD Card Reader" }, { RTSX_RTS522A, RTSX_VERSION " Realtek RTS522A PCIe SD Card Reader" }, { RTSX_RTS525A, RTSX_VERSION " Realtek RTS525A PCIe SD Card Reader" }, { RTSX_RTS5249, RTSX_VERSION " Realtek RTS5249 PCIe SD Card Reader" }, { RTSX_RTS5260, RTSX_VERSION " Realtek RTS5260 PCIe SD Card Reader" }, { RTSX_RTL8402, RTSX_VERSION " Realtek RTL8402 PCIe SD Card Reader" }, { RTSX_RTL8411, RTSX_VERSION " Realtek RTL8411 PCIe SD Card Reader" }, { RTSX_RTL8411B, RTSX_VERSION " Realtek RTL8411B PCIe SD Card Reader" }, }; /* See `kenv | grep smbios.system` */ static const struct rtsx_inversion_model { char *maker; char *family; char *product; } rtsx_inversion_models[] = { { "LENOVO", "ThinkPad T470p", "20J7S0PM00"}, { "LENOVO", "ThinkPad X13 Gen 1", "20UF000QRT"}, { NULL, NULL, NULL} }; static int rtsx_dma_alloc(struct rtsx_softc *sc); static void rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error); static void rtsx_dma_free(struct rtsx_softc *sc); static void rtsx_intr(void *arg); static void rtsx_handle_card_present(struct rtsx_softc *sc); static void rtsx_card_task(void *arg, int pending __unused); static bool rtsx_is_card_present(struct rtsx_softc *sc); static int rtsx_init(struct rtsx_softc *sc); static int rtsx_map_sd_drive(int index); static int rtsx_rts5227_fill_driving(struct rtsx_softc *sc); static int rtsx_rts5249_fill_driving(struct rtsx_softc *sc); static int rtsx_rts5260_fill_driving(struct rtsx_softc *sc); static int rtsx_read(struct rtsx_softc *, uint16_t, uint8_t *); static int rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val); static int rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val); static int rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val); static int rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val); static int rtsx_bus_power_off(struct rtsx_softc *sc); static int rtsx_bus_power_on(struct rtsx_softc *sc); static int rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width); static int rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing); static int rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq); static int rtsx_stop_sd_clock(struct rtsx_softc *sc); static int rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu); #ifndef MMCCAM static void rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point); static void rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point); static void rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map); static int rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point); static int rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc); static void rtsx_sd_wait_data_idle(struct rtsx_softc *sc); static uint8_t rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map); static int rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit); #endif /* !MMCCAM */ #if 0 /* For led */ static int rtsx_led_enable(struct rtsx_softc *sc); static int rtsx_led_disable(struct rtsx_softc *sc); #endif /* For led */ static uint8_t rtsx_response_type(uint16_t mmc_rsp); static void rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg, uint8_t mask, uint8_t data); static void rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt); static void rtsx_send_cmd(struct rtsx_softc *sc); static void rtsx_ret_resp(struct rtsx_softc *sc); static void rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_stop_cmd(struct rtsx_softc *sc); static void rtsx_clear_error(struct rtsx_softc *sc); static void rtsx_req_done(struct rtsx_softc *sc); static int rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd); static int rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_ask_ppbuf_part1(struct rtsx_softc *sc); static void rtsx_get_ppbuf_part1(struct rtsx_softc *sc); static void rtsx_get_ppbuf_part2(struct rtsx_softc *sc); static void rtsx_put_ppbuf_part1(struct rtsx_softc *sc); static void rtsx_put_ppbuf_part2(struct rtsx_softc *sc); static void rtsx_write_ppbuf(struct rtsx_softc *sc); static int rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_xfer_begin(struct rtsx_softc *sc); static void rtsx_xfer_start(struct rtsx_softc *sc); static void rtsx_xfer_finish(struct rtsx_softc *sc); static void rtsx_timeout(void *arg); #ifdef MMCCAM static int rtsx_get_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts); static int rtsx_set_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts); static int rtsx_cam_request(device_t dev, union ccb *ccb); #endif /* MMCCAM */ static int rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result); static int rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value); static int rtsx_mmcbr_update_ios(device_t bus, device_t child __unused); static int rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused); static int rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req); #ifndef MMCCAM static int rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400 __unused); static int rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused); static int rtsx_mmcbr_get_ro(device_t bus, device_t child __unused); static int rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused); static int rtsx_mmcbr_release_host(device_t bus, device_t child __unused); #endif /* !MMCCAM */ static int rtsx_probe(device_t dev); static int rtsx_attach(device_t dev); static int rtsx_detach(device_t dev); static int rtsx_shutdown(device_t dev); static int rtsx_suspend(device_t dev); static int rtsx_resume(device_t dev); #define RTSX_LOCK_INIT(_sc) mtx_init(&(_sc)->rtsx_mtx, \ device_get_nameunit(sc->rtsx_dev), "rtsx", MTX_DEF) #define RTSX_LOCK(_sc) mtx_lock(&(_sc)->rtsx_mtx) #define RTSX_UNLOCK(_sc) mtx_unlock(&(_sc)->rtsx_mtx) #define RTSX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rtsx_mtx) #define RTSX_SDCLK_OFF 0 #define RTSX_SDCLK_250KHZ 250000 #define RTSX_SDCLK_400KHZ 400000 #define RTSX_SDCLK_25MHZ 25000000 #define RTSX_SDCLK_50MHZ 50000000 #define RTSX_SDCLK_100MHZ 100000000 #define RTSX_SDCLK_208MHZ 208000000 #define RTSX_MIN_DIV_N 80 #define RTSX_MAX_DIV_N 208 #define RTSX_MAX_DATA_BLKLEN 512 #define RTSX_DMA_ALIGN 4 #define RTSX_HOSTCMD_MAX 256 #define RTSX_DMA_CMD_BIFSIZE (sizeof(uint32_t) * RTSX_HOSTCMD_MAX) -#define RTSX_DMA_DATA_BUFSIZE MAXPHYS +#define RTSX_DMA_DATA_BUFSIZE maxphys #define ISSET(t, f) ((t) & (f)) #define READ4(sc, reg) \ (bus_space_read_4((sc)->rtsx_mem_btag, (sc)->rtsx_mem_bhandle, (reg))) #define WRITE4(sc, reg, val) \ (bus_space_write_4((sc)->rtsx_mem_btag, (sc)->rtsx_mem_bhandle, (reg), (val))) #define RTSX_READ(sc, reg, val) \ do { \ int err = rtsx_read((sc), (reg), (val)); \ if (err) \ return (err); \ } while (0) #define RTSX_WRITE(sc, reg, val) \ do { \ int err = rtsx_write((sc), (reg), 0xff, (val)); \ if (err) \ return (err); \ } while (0) #define RTSX_CLR(sc, reg, bits) \ do { \ int err = rtsx_write((sc), (reg), (bits), 0); \ if (err) \ return (err); \ } while (0) #define RTSX_SET(sc, reg, bits) \ do { \ int err = rtsx_write((sc), (reg), (bits), 0xff);\ if (err) \ return (err); \ } while (0) #define RTSX_BITOP(sc, reg, mask, bits) \ do { \ int err = rtsx_write((sc), (reg), (mask), (bits)); \ if (err) \ return (err); \ } while (0) /* * We use two DMA buffers: a command buffer and a data buffer. * * The command buffer contains a command queue for the host controller, * which describes SD/MMC commands to run, and other parameters. The chip * runs the command queue when a special bit in the RTSX_HCBAR register is * set and signals completion with the RTSX_TRANS_OK_INT interrupt. * Each command is encoded as a 4 byte sequence containing command number * (read, write, or check a host controller register), a register address, * and a data bit-mask and value. * SD/MMC commands which do not transfer any data from/to the card only use * the command buffer. * * The data buffer is used for transfer longer than 512. Data transfer is * controlled via the RTSX_HDBAR register and completion is signalled by * the RTSX_TRANS_OK_INT interrupt. * * The chip is unable to perform DMA above 4GB. */ /* * Main commands in the usual seqence used: * * CMD0 Go idle state * CMD8 Send interface condition * CMD55 Application Command for next ACMD * ACMD41 Send Operation Conditions Register (OCR: voltage profile of the card) * CMD2 Send Card Identification (CID) Register * CMD3 Send relative address * CMD9 Send Card Specific Data (CSD) * CMD13 Send status (32 bits - bit 25: card password protected) * CMD7 Select card (before Get card SCR) * ACMD51 Send SCR (SD CARD Configuration Register - [51:48]: Bus widths supported) * CMD6 SD switch function * ACMD13 Send SD status (512 bits) * ACMD42 Set/Clear card detect * ACMD6 Set bus width * CMD19 Send tuning block * CMD12 Stop transmission * * CMD17 Read single block (<=512) * CMD18 Read multiple blocks (>512) * CMD24 Write single block (<=512) * CMD25 Write multiple blocks (>512) * * CMD52 IO R/W direct * CMD5 Send Operation Conditions */ static int rtsx_dma_alloc(struct rtsx_softc *sc) { int error = 0; error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev), /* inherit from parent */ RTSX_DMA_ALIGN, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ RTSX_DMA_CMD_BIFSIZE, 1, /* maxsize, nsegments */ RTSX_DMA_CMD_BIFSIZE, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->rtsx_cmd_dma_tag); if (error) { device_printf(sc->rtsx_dev, "Can't create cmd parent DMA tag\n"); return (error); } error = bus_dmamem_alloc(sc->rtsx_cmd_dma_tag, /* DMA tag */ &sc->rtsx_cmd_dmamem, /* will hold the KVA pointer */ BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, /* flags */ &sc->rtsx_cmd_dmamap); /* DMA map */ if (error) { device_printf(sc->rtsx_dev, "Can't create DMA map for command transfer\n"); goto destroy_cmd_dma_tag; } error = bus_dmamap_load(sc->rtsx_cmd_dma_tag, /* DMA tag */ sc->rtsx_cmd_dmamap, /* DMA map */ sc->rtsx_cmd_dmamem, /* KVA pointer to be mapped */ RTSX_DMA_CMD_BIFSIZE, /* size of buffer */ rtsx_dmamap_cb, /* callback */ &sc->rtsx_cmd_buffer, /* first arg of callback */ 0); /* flags */ if (error || sc->rtsx_cmd_buffer == 0) { device_printf(sc->rtsx_dev, "Can't load DMA memory for command transfer\n"); error = (error) ? error : EFAULT; goto destroy_cmd_dmamem_alloc; } error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev), /* inherit from parent */ RTSX_DMA_DATA_BUFSIZE, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ RTSX_DMA_DATA_BUFSIZE, 1, /* maxsize, nsegments */ RTSX_DMA_DATA_BUFSIZE, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->rtsx_data_dma_tag); if (error) { device_printf(sc->rtsx_dev, "Can't create data parent DMA tag\n"); goto destroy_cmd_dmamap_load; } error = bus_dmamem_alloc(sc->rtsx_data_dma_tag, /* DMA tag */ &sc->rtsx_data_dmamem, /* will hold the KVA pointer */ BUS_DMA_WAITOK | BUS_DMA_ZERO, /* flags */ &sc->rtsx_data_dmamap); /* DMA map */ if (error) { device_printf(sc->rtsx_dev, "Can't create DMA map for data transfer\n"); goto destroy_data_dma_tag; } error = bus_dmamap_load(sc->rtsx_data_dma_tag, /* DMA tag */ sc->rtsx_data_dmamap, /* DMA map */ sc->rtsx_data_dmamem, /* KVA pointer to be mapped */ RTSX_DMA_DATA_BUFSIZE, /* size of buffer */ rtsx_dmamap_cb, /* callback */ &sc->rtsx_data_buffer, /* first arg of callback */ 0); /* flags */ if (error || sc->rtsx_data_buffer == 0) { device_printf(sc->rtsx_dev, "Can't load DMA memory for data transfer\n"); error = (error) ? error : EFAULT; goto destroy_data_dmamem_alloc; } return (error); destroy_data_dmamem_alloc: bus_dmamem_free(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamem, sc->rtsx_data_dmamap); destroy_data_dma_tag: bus_dma_tag_destroy(sc->rtsx_data_dma_tag); destroy_cmd_dmamap_load: bus_dmamap_unload(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap); destroy_cmd_dmamem_alloc: bus_dmamem_free(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamem, sc->rtsx_cmd_dmamap); destroy_cmd_dma_tag: bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag); return (error); } static void rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error) { printf("rtsx_dmamap_cb: error %d\n", error); return; } *(bus_addr_t *)arg = segs[0].ds_addr; } static void rtsx_dma_free(struct rtsx_softc *sc) { if (sc->rtsx_cmd_dma_tag != NULL) { if (sc->rtsx_cmd_dmamap != NULL) bus_dmamap_unload(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap); if (sc->rtsx_cmd_dmamem != NULL) bus_dmamem_free(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamem, sc->rtsx_cmd_dmamap); sc->rtsx_cmd_dmamap = NULL; sc->rtsx_cmd_dmamem = NULL; sc->rtsx_cmd_buffer = 0; bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag); sc->rtsx_cmd_dma_tag = NULL; } if (sc->rtsx_data_dma_tag != NULL) { if (sc->rtsx_data_dmamap != NULL) bus_dmamap_unload(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap); if (sc->rtsx_data_dmamem != NULL) bus_dmamem_free(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamem, sc->rtsx_data_dmamap); sc->rtsx_data_dmamap = NULL; sc->rtsx_data_dmamem = NULL; sc->rtsx_data_buffer = 0; bus_dma_tag_destroy(sc->rtsx_data_dma_tag); sc->rtsx_data_dma_tag = NULL; } } static void rtsx_intr(void *arg) { struct rtsx_softc *sc = arg; uint32_t enabled; uint32_t status; RTSX_LOCK(sc); enabled = sc->rtsx_intr_enabled; status = READ4(sc, RTSX_BIPR); /* read Bus Interrupt Pending Register */ sc->rtsx_intr_status = status; if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "Interrupt handler - enabled: 0x%08x, status: 0x%08x\n", enabled, status); /* Ack interrupts. */ WRITE4(sc, RTSX_BIPR, status); if (((enabled & status) == 0) || status == 0xffffffff) { device_printf(sc->rtsx_dev, "Spurious interrupt - enabled: 0x%08x, status: 0x%08x\n", enabled, status); RTSX_UNLOCK(sc); return; } /* Detect write protect. */ if (status & RTSX_SD_WRITE_PROTECT) sc->rtsx_read_only = 1; else sc->rtsx_read_only = 0; /* Start task to handle SD card status change (from dwmmc.c). */ if (status & RTSX_SD_INT) { device_printf(sc->rtsx_dev, "Interrupt card inserted/removed\n"); rtsx_handle_card_present(sc); } if (sc->rtsx_req == NULL) { RTSX_UNLOCK(sc); return; } if (status & RTSX_TRANS_OK_INT) { sc->rtsx_req->cmd->error = MMC_ERR_NONE; if (sc->rtsx_intr_trans_ok != NULL) sc->rtsx_intr_trans_ok(sc); } else if (status & RTSX_TRANS_FAIL_INT) { uint8_t stat1; sc->rtsx_req->cmd->error = MMC_ERR_FAILED; if (rtsx_read(sc, RTSX_SD_STAT1, &stat1) == 0 && (stat1 & RTSX_SD_CRC_ERR)) { device_printf(sc->rtsx_dev, "CRC error\n"); sc->rtsx_req->cmd->error = MMC_ERR_BADCRC; } if (!sc->rtsx_tuning_mode) device_printf(sc->rtsx_dev, "Transfer fail - status: 0x%08x\n", status); rtsx_stop_cmd(sc); if (sc->rtsx_intr_trans_ko != NULL) sc->rtsx_intr_trans_ko(sc); } RTSX_UNLOCK(sc); } /* * Function called from the IRQ handler (from dwmmc.c). */ static void rtsx_handle_card_present(struct rtsx_softc *sc) { bool was_present; bool is_present; #ifdef MMCCAM was_present = sc->rtsx_cam_status; #else /* !MMCCAM */ was_present = sc->rtsx_mmc_dev != NULL; #endif /* MMCCAM */ is_present = rtsx_is_card_present(sc); if (is_present) device_printf(sc->rtsx_dev, "Card present\n"); else device_printf(sc->rtsx_dev, "Card absent\n"); if (!was_present && is_present) { /* * The delay is to debounce the card insert * (sometimes the card detect pin stabilizes * before the other pins have made good contact). */ taskqueue_enqueue_timeout(taskqueue_swi_giant, &sc->rtsx_card_insert_task, -hz); } else if (was_present && !is_present) { taskqueue_enqueue(taskqueue_swi_giant, &sc->rtsx_card_remove_task); } } /* * This function is called at startup. */ static void rtsx_card_task(void *arg, int pending __unused) { struct rtsx_softc *sc = arg; if (rtsx_is_card_present(sc)) { sc->rtsx_flags |= RTSX_F_CARD_PRESENT; /* Card is present, attach if necessary. */ #ifdef MMCCAM if (sc->rtsx_cam_status == 0) { #else /* !MMCCAM */ if (sc->rtsx_mmc_dev == NULL) { #endif /* MMCCAM */ if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "Card inserted\n"); sc->rtsx_read_count = sc->rtsx_write_count = 0; #ifdef MMCCAM sc->rtsx_cam_status = 1; mmc_cam_sim_discover(&sc->rtsx_mmc_sim); #else /* !MMCCAM */ RTSX_LOCK(sc); sc->rtsx_mmc_dev = device_add_child(sc->rtsx_dev, "mmc", -1); RTSX_UNLOCK(sc); if (sc->rtsx_mmc_dev == NULL) { device_printf(sc->rtsx_dev, "Adding MMC bus failed\n"); } else { device_set_ivars(sc->rtsx_mmc_dev, sc); device_probe_and_attach(sc->rtsx_mmc_dev); } #endif /* MMCCAM */ } } else { sc->rtsx_flags &= ~RTSX_F_CARD_PRESENT; /* Card isn't present, detach if necessary. */ #ifdef MMCCAM if (sc->rtsx_cam_status != 0) { #else /* !MMCCAM */ if (sc->rtsx_mmc_dev != NULL) { #endif /* MMCCAM */ if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "Card removed\n"); if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "Read count: %" PRIu64 ", write count: %" PRIu64 "\n", sc->rtsx_read_count, sc->rtsx_write_count); #ifdef MMCCAM sc->rtsx_cam_status = 0; mmc_cam_sim_discover(&sc->rtsx_mmc_sim); #else /* !MMCCAM */ if (device_delete_child(sc->rtsx_dev, sc->rtsx_mmc_dev)) device_printf(sc->rtsx_dev, "Detaching MMC bus failed\n"); sc->rtsx_mmc_dev = NULL; #endif /* MMCCAM */ } } } static bool rtsx_is_card_present(struct rtsx_softc *sc) { uint32_t status; status = READ4(sc, RTSX_BIPR); if (sc->rtsx_inversion == 0) return (status & RTSX_SD_EXIST); else return !(status & RTSX_SD_EXIST); } static int rtsx_init(struct rtsx_softc *sc) { uint8_t version; uint8_t val; int error; sc->rtsx_host.host_ocr = RTSX_SUPPORTED_VOLTAGE; sc->rtsx_host.f_min = RTSX_SDCLK_250KHZ; sc->rtsx_host.f_max = RTSX_SDCLK_208MHZ; sc->rtsx_host.caps = MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED | MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25; sc->rtsx_host.caps |= MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104; if (sc->rtsx_device_id == RTSX_RTS5209) sc->rtsx_host.caps |= MMC_CAP_8_BIT_DATA; pci_find_cap(sc->rtsx_dev, PCIY_EXPRESS, &(sc->rtsx_pcie_cap)); /* * Check IC version. */ switch (sc->rtsx_device_id) { case RTSX_RTS5229: /* Read IC version from dummy register. */ RTSX_READ(sc, RTSX_DUMMY_REG, &version); if ((version & 0x0F) == RTSX_IC_VERSION_C) sc->rtsx_flags |= RTSX_F_VERSION_C; break; case RTSX_RTS522A: /* Read IC version from dummy register. */ RTSX_READ(sc, RTSX_DUMMY_REG, &version); if ((version & 0x0F) == RTSX_IC_VERSION_A) sc->rtsx_flags |= RTSX_F_VERSION_A; break; case RTSX_RTS525A: /* Read IC version from dummy register. */ RTSX_READ(sc, RTSX_DUMMY_REG, &version); if ((version & 0x0F) == RTSX_IC_VERSION_A) sc->rtsx_flags |= RTSX_F_VERSION_A; break; case RTSX_RTL8411B: RTSX_READ(sc, RTSX_RTL8411B_PACKAGE, &version); if (version & RTSX_RTL8411B_QFN48) sc->rtsx_flags |= RTSX_F_8411B_QFN48; break; } /* * Fetch vendor settings. */ /* * Normally OEMs will set vendor setting to the config space * of Realtek card reader in BIOS stage. This statement reads * the setting and configure the internal registers according * to it, to improve card reader's compatibility condition. */ sc->rtsx_card_drive_sel = RTSX_CARD_DRIVE_DEFAULT; switch (sc->rtsx_device_id) { uint32_t reg; uint32_t reg1; uint8_t reg3; case RTSX_RTS5209: sc->rtsx_card_drive_sel = RTSX_RTS5209_CARD_DRIVE_DEFAULT; sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); if (!(reg & 0x80)) { sc->rtsx_card_drive_sel = (reg >> 8) & 0x3F; sc->rtsx_sd30_drive_sel_3v3 = reg & 0x07; } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS5227: case RTSX_RTS522A: sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (!(reg & 0x1000000)) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03; if (reg & 0x4000) sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET; } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3, (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false"); break; case RTSX_RTS5229: sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (!(reg & 0x1000000)) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive((reg >> 5) & 0x03); } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS525A: case RTSX_RTS5249: case RTSX_RTS5260: sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if ((reg & 0x1000000)) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03; if (reg & 0x4000) sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET; } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "card_drive_sel = 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3, (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false"); break; case RTSX_RTL8402: case RTSX_RTL8411: sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT; sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg1 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (reg1 & 0x1000000) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg1 >> 25) & 0x01) << 6; reg3 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG3, 1); sc->rtsx_sd30_drive_sel_3v3 = (reg3 >> 5) & 0x07; } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { device_printf(sc->rtsx_dev, "pci_read_config() error - reg1: 0x%08x\n", reg1); } if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTL8411B: sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT; sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (!(reg & 0x1000000)) { sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive(reg & 0x03); } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; } if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_init() rtsx_flags: 0x%04x\n", sc->rtsx_flags); /* Enable interrupts. */ sc->rtsx_intr_enabled = RTSX_TRANS_OK_INT_EN | RTSX_TRANS_FAIL_INT_EN | RTSX_SD_INT_EN; WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled); /* Power on SSC clock. */ RTSX_CLR(sc, RTSX_FPDCTL, RTSX_SSC_POWER_DOWN); /* Wait SSC power stable. */ DELAY(200); /* Disable ASPM */ val = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, 1); pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, val & 0xfc, 1); /* * Optimize phy. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: /* Some magic numbers from Linux driver. */ if ((error = rtsx_write_phy(sc, 0x00, 0xB966))) return (error); break; case RTSX_RTS5227: RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); /* Optimize RX sensitivity. */ if ((error = rtsx_write_phy(sc, 0x00, 0xBA42))) return (error); break; case RTSX_RTS5229: /* Optimize RX sensitivity. */ if ((error = rtsx_write_phy(sc, 0x00, 0xBA42))) return (error); break; case RTSX_RTS522A: RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); if (sc->rtsx_flags & RTSX_F_VERSION_A) { if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2, RTSX_PHY_RCR2_INIT_27S))) return (error); } if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1, RTSX_PHY_RCR1_INIT_27S))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD0, RTSX_PHY_FLD0_INIT_27S))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3, RTSX_PHY_FLD3_INIT_27S))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4, RTSX_PHY_FLD4_INIT_27S))) return (error); break; case RTSX_RTS525A: if ((error = rtsx_write_phy(sc, RTSX__PHY_FLD0, RTSX__PHY_FLD0_CLK_REQ_20C | RTSX__PHY_FLD0_RX_IDLE_EN | RTSX__PHY_FLD0_BIT_ERR_RSTN | RTSX__PHY_FLD0_BER_COUNT | RTSX__PHY_FLD0_BER_TIMER | RTSX__PHY_FLD0_CHECK_EN))) return (error); if ((error = rtsx_write_phy(sc, RTSX__PHY_ANA03, RTSX__PHY_ANA03_TIMER_MAX | RTSX__PHY_ANA03_OOBS_DEB_EN | RTSX__PHY_CMU_DEBUG_EN))) return (error); if (sc->rtsx_flags & RTSX_F_VERSION_A) if ((error = rtsx_write_phy(sc, RTSX__PHY_REV0, RTSX__PHY_REV0_FILTER_OUT | RTSX__PHY_REV0_CDR_BYPASS_PFD | RTSX__PHY_REV0_CDR_RX_IDLE_BYPASS))) return (error); break; case RTSX_RTS5249: RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); if ((error = rtsx_write_phy(sc, RTSX_PHY_REV, RTSX_PHY_REV_RESV | RTSX_PHY_REV_RXIDLE_LATCHED | RTSX_PHY_REV_P1_EN | RTSX_PHY_REV_RXIDLE_EN | RTSX_PHY_REV_CLKREQ_TX_EN | RTSX_PHY_REV_RX_PWST | RTSX_PHY_REV_CLKREQ_DT_1_0 | RTSX_PHY_REV_STOP_CLKRD | RTSX_PHY_REV_STOP_CLKWR))) return (error); DELAY(1000); if ((error = rtsx_write_phy(sc, RTSX_PHY_BPCR, RTSX_PHY_BPCR_IBRXSEL | RTSX_PHY_BPCR_IBTXSEL | RTSX_PHY_BPCR_IB_FILTER | RTSX_PHY_BPCR_CMIRROR_EN))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_PCR, RTSX_PHY_PCR_FORCE_CODE | RTSX_PHY_PCR_OOBS_CALI_50 | RTSX_PHY_PCR_OOBS_VCM_08 | RTSX_PHY_PCR_OOBS_SEN_90 | RTSX_PHY_PCR_RSSI_EN | RTSX_PHY_PCR_RX10K))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2, RTSX_PHY_RCR2_EMPHASE_EN | RTSX_PHY_RCR2_NADJR | RTSX_PHY_RCR2_CDR_SR_2 | RTSX_PHY_RCR2_FREQSEL_12 | RTSX_PHY_RCR2_CDR_SC_12P | RTSX_PHY_RCR2_CALIB_LATE))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4, RTSX_PHY_FLD4_FLDEN_SEL | RTSX_PHY_FLD4_REQ_REF | RTSX_PHY_FLD4_RXAMP_OFF | RTSX_PHY_FLD4_REQ_ADDA | RTSX_PHY_FLD4_BER_COUNT | RTSX_PHY_FLD4_BER_TIMER | RTSX_PHY_FLD4_BER_CHK_EN))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_RDR, RTSX_PHY_RDR_RXDSEL_1_9 | RTSX_PHY_SSC_AUTO_PWD))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1, RTSX_PHY_RCR1_ADP_TIME_4 | RTSX_PHY_RCR1_VCO_COARSE))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3, RTSX_PHY_FLD3_TIMER_4 | RTSX_PHY_FLD3_TIMER_6 | RTSX_PHY_FLD3_RXDELINK))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE, RTSX_PHY_TUNE_TUNEREF_1_0 | RTSX_PHY_TUNE_VBGSEL_1252 | RTSX_PHY_TUNE_SDBUS_33 | RTSX_PHY_TUNE_TUNED18 | RTSX_PHY_TUNE_TUNED12 | RTSX_PHY_TUNE_TUNEA12))) return (error); break; } /* Set mcu_cnt to 7 to ensure data can be sampled properly. */ RTSX_BITOP(sc, RTSX_CLK_DIV, 0x07, 0x07); /* Disable sleep mode. */ RTSX_CLR(sc, RTSX_HOST_SLEEP_STATE, RTSX_HOST_ENTER_S1 | RTSX_HOST_ENTER_S3); /* Disable card clock. */ RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL); /* Reset delink mode. */ RTSX_CLR(sc, RTSX_CHANGE_LINK_STATE, RTSX_FORCE_RST_CORE_EN | RTSX_NON_STICKY_RST_N_DBG); /* Card driving select. */ RTSX_WRITE(sc, RTSX_CARD_DRIVE_SEL, sc->rtsx_card_drive_sel); /* Enable SSC clock. */ RTSX_WRITE(sc, RTSX_SSC_CTL1, RTSX_SSC_8X_EN | RTSX_SSC_SEL_4M); RTSX_WRITE(sc, RTSX_SSC_CTL2, 0x12); /* Disable cd_pwr_save. */ RTSX_BITOP(sc, RTSX_CHANGE_LINK_STATE, 0x16, RTSX_MAC_PHY_RST_N_DBG); /* Clear Link Ready Interrupt. */ RTSX_BITOP(sc, RTSX_IRQSTAT0, RTSX_LINK_READY_INT, RTSX_LINK_READY_INT); /* Enlarge the estimation window of PERST# glitch * to reduce the chance of invalid card interrupt. */ RTSX_WRITE(sc, RTSX_PERST_GLITCH_WIDTH, 0x80); /* Set RC oscillator to 400K. */ RTSX_CLR(sc, RTSX_RCCTL, RTSX_RCCTL_F_2M); /* Enable interrupt write-clear (default is read-clear). */ RTSX_CLR(sc, RTSX_NFTS_TX_CTRL, RTSX_INT_READ_CLR); switch (sc->rtsx_device_id) { case RTSX_RTS525A: case RTSX_RTS5260: RTSX_BITOP(sc, RTSX_PM_CLK_FORCE_CTL, 1, 1); break; } /* OC power down. */ RTSX_BITOP(sc, RTSX_FPDCTL, RTSX_SD_OC_POWER_DOWN, RTSX_SD_OC_POWER_DOWN); /* Enable clk_request_n to enable clock power management */ pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL + 1, 1, 1); /* Enter L1 when host tx idle */ pci_write_config(sc->rtsx_dev, 0x70F, 0x5B, 1); /* * Specific extra init. */ switch (sc->rtsx_device_id) { uint16_t cap; case RTSX_RTS5209: /* Turn off LED. */ RTSX_WRITE(sc, RTSX_CARD_GPIO, 0x03); /* Reset ASPM state to default value. */ RTSX_CLR(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK); /* Force CLKREQ# PIN to drive 0 to request clock. */ RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08); /* Configure GPIO as output. */ RTSX_WRITE(sc, RTSX_CARD_GPIO_DIR, 0x03); /* Configure driving. */ RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS5227: /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure LTR. */ cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2); if (cap & PCIEM_CTL2_LTR_ENABLE) RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3); /* Configure OBFF. */ RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE); /* Configure driving. */ if ((error = rtsx_rts5227_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88); RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); /*!!! Added for reboot after Windows. */ RTSX_BITOP(sc, RTSX_PM_CTRL3, RTSX_PM_WAKE_EN, RTSX_PM_WAKE_EN); break; case RTSX_RTS5229: /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ /* With this reset: dd if=/dev/random of=/dev/mmcsd0 encounter a timeout. */ //!!! RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Force CLKREQ# PIN to drive 0 to request clock. */ RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08); /* Switch LDO3318 source from DV33 to card_3v3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure driving. */ RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS522A: /* Add specific init from RTS5227. */ /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure LTR. */ cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2); if (cap & PCIEM_CTL2_LTR_ENABLE) RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3); /* Configure OBFF. */ RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE); /* Configure driving. */ if ((error = rtsx_rts5227_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88); RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3, 0x10); /* specific for RTS522A. */ RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL, RTSX_FUNC_FORCE_UPME_XMT_DBG, RTSX_FUNC_FORCE_UPME_XMT_DBG); RTSX_BITOP(sc, RTSX_PCLK_CTL, 0x04, 0x04); RTSX_BITOP(sc, RTSX_PM_EVENT_DEBUG, RTSX_PME_DEBUG_0, RTSX_PME_DEBUG_0); RTSX_WRITE(sc, RTSX_PM_CLK_FORCE_CTL, 0x11); break; case RTSX_RTS525A: /* Add specific init from RTS5249. */ /* Rest L1SUB Config. */ RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff); /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure driving. */ if ((error = rtsx_rts5249_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80); /* Specifc for RTS525A. */ RTSX_BITOP(sc, RTSX_PCLK_CTL, RTSX_PCLK_MODE_SEL, RTSX_PCLK_MODE_SEL); if (sc->rtsx_flags & RTSX_F_VERSION_A) { RTSX_WRITE(sc, RTSX_L1SUB_CONFIG2, RTSX_L1SUB_AUTO_CFG); RTSX_BITOP(sc, RTSX_RREF_CFG, RTSX_RREF_VBGSEL_MASK, RTSX_RREF_VBGSEL_1V25); RTSX_BITOP(sc, RTSX_LDO_VIO_CFG, RTSX_LDO_VIO_TUNE_MASK, RTSX_LDO_VIO_1V7); RTSX_BITOP(sc, RTSX_LDO_DV12S_CFG, RTSX_LDO_D12_TUNE_MASK, RTSX_LDO_D12_TUNE_DF); RTSX_BITOP(sc, RTSX_LDO_AV12S_CFG, RTSX_LDO_AV12S_TUNE_MASK, RTSX_LDO_AV12S_TUNE_DF); RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0, RTSX_LDO_VCC_LMTVTH_MASK, RTSX_LDO_VCC_LMTVTH_2A); RTSX_BITOP(sc, RTSX_OOBS_CONFIG, RTSX_OOBS_AUTOK_DIS | RTSX_OOBS_VAL_MASK, 0x89); } break; case RTSX_RTS5249: /* Rest L1SUB Config. */ RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff); /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure driving. */ if ((error = rtsx_rts5249_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80); break; case RTSX_RTS5260: /* Set mcu_cnt to 7 to ensure data can be sampled properly. */ RTSX_BITOP(sc, RTSX_CLK_DIV, 0x07, 0x07); RTSX_WRITE(sc, RTSX_SSC_DIV_N_0, 0x5D); /* Force no MDIO */ RTSX_WRITE(sc, RTSX_RTS5260_AUTOLOAD_CFG4, RTSX_RTS5260_MIMO_DISABLE); /* Modify SDVCC Tune Default Parameters! */ RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0, RTSX_RTS5260_DVCC_TUNE_MASK, RTSX_RTS5260_DVCC_33); RTSX_BITOP(sc, RTSX_PCLK_CTL, RTSX_PCLK_MODE_SEL, RTSX_PCLK_MODE_SEL); RTSX_BITOP(sc, RTSX_L1SUB_CONFIG1, RTSX_AUX_CLK_ACTIVE_SEL_MASK, RTSX_MAC_CKSW_DONE); /* Rest L1SUB Config */ RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xFF); RTSX_BITOP(sc, RTSX_PM_CLK_FORCE_CTL, RTSX_CLK_PM_EN, RTSX_CLK_PM_EN); RTSX_WRITE(sc, RTSX_PWD_SUSPEND_EN, 0xFF); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_PWR_GATE_EN, RTSX_PWR_GATE_EN); RTSX_BITOP(sc, RTSX_REG_VREF, RTSX_PWD_SUSPND_EN, RTSX_PWD_SUSPND_EN); RTSX_BITOP(sc, RTSX_RBCTL, RTSX_U_AUTO_DMA_EN_MASK, RTSX_U_AUTO_DMA_DISABLE); if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80); RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_DISABLE); RTSX_CLR(sc, RTSX_RTS5260_DVCC_CTRL, RTSX_RTS5260_DVCC_OCP_EN | RTSX_RTS5260_DVCC_OCP_CL_EN); /* CLKREQ# PIN will be forced to drive low. */ RTSX_BITOP(sc, RTSX_PETXCFG, RTSX_FORCE_CLKREQ_DELINK_MASK, RTSX_FORCE_CLKREQ_LOW); RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3, 0x10); break; case RTSX_RTL8402: case RTSX_RTL8411: RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE, RTSX_CD_ENABLE); break; case RTSX_RTL8411B: if (sc->rtsx_flags & RTSX_F_8411B_QFN48) RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5); RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); /* Enable SD interrupt. */ RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE, RTSX_CD_ENABLE); /* Clear hw_pfm_en to disable hardware PFM mode. */ RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL, 0x06, 0x00); break; } /*!!! Added for reboot after Windows. */ rtsx_bus_power_off(sc); rtsx_set_sd_timing(sc, bus_timing_normal); rtsx_set_sd_clock(sc, 0); /*!!! Added for reboot after Windows. */ return (0); } static int rtsx_map_sd_drive(int index) { uint8_t sd_drive[4] = { 0x01, /* Type D */ 0x02, /* Type C */ 0x05, /* Type A */ 0x03 /* Type B */ }; return (sd_drive[index]); } /* For voltage 3v3. */ static int rtsx_rts5227_fill_driving(struct rtsx_softc *sc) { u_char driving_3v3[4][3] = { {0x13, 0x13, 0x13}, {0x96, 0x96, 0x96}, {0x7F, 0x7F, 0x7F}, {0x96, 0x96, 0x96}, }; RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]); RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]); RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]); return (0); } /* For voltage 3v3. */ static int rtsx_rts5249_fill_driving(struct rtsx_softc *sc) { u_char driving_3v3[4][3] = { {0x11, 0x11, 0x18}, {0x55, 0x55, 0x5C}, {0xFF, 0xFF, 0xFF}, {0x96, 0x96, 0x96}, }; RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]); RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]); RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]); return (0); } static int rtsx_rts5260_fill_driving(struct rtsx_softc *sc) { u_char driving_3v3[4][3] = { {0x11, 0x11, 0x11}, {0x22, 0x22, 0x22}, {0x55, 0x55, 0x55}, {0x33, 0x33, 0x33}, }; RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]); RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]); RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]); return (0); } static int rtsx_read(struct rtsx_softc *sc, uint16_t addr, uint8_t *val) { int tries = 1024; uint32_t arg; uint32_t reg; arg = RTSX_HAIMR_BUSY | (uint32_t)((addr & 0x3FFF) << 16); WRITE4(sc, RTSX_HAIMR, arg); while (tries--) { reg = READ4(sc, RTSX_HAIMR); if (!(reg & RTSX_HAIMR_BUSY)) break; } *val = (reg & 0xff); if (tries > 0) { return (0); } else { device_printf(sc->rtsx_dev, "rtsx_read(0x%x) timeout\n", arg); return (ETIMEDOUT); } } static int rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val) { int tries = 1024; uint8_t data0, data1, data2, data3, rwctl; RTSX_WRITE(sc, RTSX_CFGADDR0, addr); RTSX_WRITE(sc, RTSX_CFGADDR1, addr >> 8); RTSX_WRITE(sc, RTSX_CFGRWCTL, RTSX_CFG_BUSY | (func & 0x03 << 4)); while (tries--) { RTSX_READ(sc, RTSX_CFGRWCTL, &rwctl); if (!(rwctl & RTSX_CFG_BUSY)) break; } if (tries == 0) return (ETIMEDOUT); RTSX_READ(sc, RTSX_CFGDATA0, &data0); RTSX_READ(sc, RTSX_CFGDATA1, &data1); RTSX_READ(sc, RTSX_CFGDATA2, &data2); RTSX_READ(sc, RTSX_CFGDATA3, &data3); *val = (data3 << 24) | (data2 << 16) | (data1 << 8) | data0; return (0); } static int rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val) { int tries = 1024; uint32_t arg; uint32_t reg; arg = RTSX_HAIMR_BUSY | RTSX_HAIMR_WRITE | (uint32_t)(((addr & 0x3FFF) << 16) | (mask << 8) | val); WRITE4(sc, RTSX_HAIMR, arg); while (tries--) { reg = READ4(sc, RTSX_HAIMR); if (!(reg & RTSX_HAIMR_BUSY)) { if (val != (reg & 0xff)) { device_printf(sc->rtsx_dev, "rtsx_write(0x%x) error reg=0x%x\n", arg, reg); return (EIO); } return (0); } } device_printf(sc->rtsx_dev, "rtsx_write(0x%x) timeout\n", arg); return (ETIMEDOUT); } static int rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val) { int tries = 100000; uint8_t data0, data1, rwctl; RTSX_WRITE(sc, RTSX_PHY_ADDR, addr); RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_READ); while (tries--) { RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl); if (!(rwctl & RTSX_PHY_BUSY)) break; } if (tries == 0) return (ETIMEDOUT); RTSX_READ(sc, RTSX_PHY_DATA0, &data0); RTSX_READ(sc, RTSX_PHY_DATA1, &data1); *val = data1 << 8 | data0; return (0); } static int rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val) { int tries = 100000; uint8_t rwctl; RTSX_WRITE(sc, RTSX_PHY_DATA0, val); RTSX_WRITE(sc, RTSX_PHY_DATA1, val >> 8); RTSX_WRITE(sc, RTSX_PHY_ADDR, addr); RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_WRITE); while (tries--) { RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl); if (!(rwctl & RTSX_PHY_BUSY)) break; } return ((tries == 0) ? ETIMEDOUT : 0); } /* * Notice that the meaning of RTSX_PWR_GATE_CTRL changes between RTS5209 and * RTS5229. In RTS5209 it is a mask of disabled power gates, while in RTS5229 * it is a mask of *enabled* gates. */ static int rtsx_bus_power_off(struct rtsx_softc *sc) { if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_bus_power_off()\n"); /* Disable SD clock. */ RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN); /* Disable SD output. */ RTSX_CLR(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN); /* Turn off power. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK, RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA); RTSX_SET(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_OFF); break; case RTSX_RTS5227: case RTSX_RTS5229: case RTSX_RTS522A: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK, RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA); RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK); break; case RTSX_RTS5260: rtsx_stop_cmd(sc); /* Switch vccq to 330 */ RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_VDD1, RTSX_DV331812_VDD1); RTSX_BITOP(sc, RTSX_LDO_DV18_CFG, RTSX_DV331812_MASK, RTSX_DV331812_33); RTSX_CLR(sc, RTSX_SD_PAD_CTL, RTSX_SD_IO_USING_1V8); rtsx_rts5260_fill_driving(sc); RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_POW_SDVDD1_MASK, RTSX_LDO_POW_SDVDD1_OFF); RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_POWERON, RTSX_DV331812_POWEROFF); break; case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_OFF); RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB, RTSX_BPP_LDO_SUSPEND); break; default: RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK); RTSX_SET(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_OFF); RTSX_CLR(sc, RTSX_CARD_PWR_CTL, RTSX_PMOS_STRG_800mA); break; } /* Disable pull control. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); break; case RTSX_RTS5227: case RTSX_RTS522A: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); break; case RTSX_RTS5229: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); if (sc->rtsx_flags & RTSX_F_VERSION_C) RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3_TYPE_C); else RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); break; case RTSX_RTS525A: case RTSX_RTS5249: case RTSX_RTS5260: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x55); break; case RTSX_RTL8402: case RTSX_RTL8411: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0x95); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x05); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04); break; case RTSX_RTL8411B: if (sc->rtsx_flags & RTSX_F_8411B_QFN48) { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15); } else { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd5); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15); } break; } return (0); } static int rtsx_bus_power_on(struct rtsx_softc *sc) { if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_bus_power_on()\n"); /* Select SD card. */ RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL); RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD); /* Enable SD clock. */ RTSX_BITOP(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN, RTSX_SD_CLK_EN); /* Enable pull control. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); break; case RTSX_RTS5227: case RTSX_RTS522A: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); break; case RTSX_RTS5229: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); if (sc->rtsx_flags & RTSX_F_VERSION_C) RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3_TYPE_C); else RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); break; case RTSX_RTS525A: case RTSX_RTS5249: case RTSX_RTS5260: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0xaa); break; case RTSX_RTL8402: case RTSX_RTL8411: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xa9); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x09); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04); break; case RTSX_RTL8411B: if (sc->rtsx_flags & RTSX_F_8411B_QFN48) { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf9); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x19); } else { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd9); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15); } break; } /* * To avoid a current peak, enable card power in two phases * with a delay in between. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC2); DELAY(200); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_ON); break; case RTSX_RTS5227: case RTSX_RTS522A: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1); DELAY(20000); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2); RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN, RTSX_SD_OUTPUT_EN); RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_MS_OUTPUT_EN, RTSX_MS_OUTPUT_EN); break; case RTSX_RTS5229: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1); DELAY(200); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2); break; case RTSX_RTS525A: RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_VCC_TUNE_MASK, RTSX_LDO_VCC_3V3); case RTSX_RTS5249: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1); DELAY(5000); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2); break; case RTSX_RTS5260: RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_VDD1, RTSX_DV331812_VDD1); RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0, RTSX_RTS5260_DVCC_TUNE_MASK, RTSX_RTS5260_DVCC_33); RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_POW_SDVDD1_MASK, RTSX_LDO_POW_SDVDD1_ON); RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_POWERON, RTSX_DV331812_POWERON); DELAY(20000); RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK | RTSX_SD_ASYNC_FIFO_NOT_RST, RTSX_SD30_MODE | RTSX_SD_ASYNC_FIFO_NOT_RST); RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE, RTSX_CRC_VAR_CLK0 | RTSX_SD30_FIX_CLK | RTSX_SAMPLE_VAR_CLK1); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); /* Initialize SD_CFG1 register */ RTSX_WRITE(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_128 | RTSX_SD20_MODE); RTSX_WRITE(sc, RTSX_SD_SAMPLE_POINT_CTL, RTSX_SD20_RX_POS_EDGE); RTSX_CLR(sc, RTSX_SD_PUSH_POINT_CTL, 0xff); RTSX_BITOP(sc, RTSX_CARD_STOP, RTSX_SD_STOP | RTSX_SD_CLR_ERR, RTSX_SD_STOP | RTSX_SD_CLR_ERR); /* Reset SD_CFG3 register */ RTSX_CLR(sc, RTSX_SD_CFG3, RTSX_SD30_CLK_END_EN); RTSX_CLR(sc, RTSX_REG_SD_STOP_SDCLK_CFG, RTSX_SD30_CLK_STOP_CFG_EN | RTSX_SD30_CLK_STOP_CFG0 | RTSX_SD30_CLK_STOP_CFG1); RTSX_CLR(sc, RTSX_REG_PRE_RW_MODE, RTSX_EN_INFINITE_MODE); break; case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_5_PERCENT_ON); RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB, RTSX_BPP_LDO_SUSPEND); DELAY(150); RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_10_PERCENT_ON); DELAY(150); RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_15_PERCENT_ON); DELAY(150); RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_ON); RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB, RTSX_BPP_LDO_ON); break; } /* Enable SD card output. */ RTSX_WRITE(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN); DELAY(200); return (0); } /* * Set but width. */ static int rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width) { uint32_t bus_width; switch (width) { case bus_width_1: bus_width = RTSX_BUS_WIDTH_1; break; case bus_width_4: bus_width = RTSX_BUS_WIDTH_4; break; case bus_width_8: bus_width = RTSX_BUS_WIDTH_8; break; default: return (MMC_ERR_INVALID); } RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_BUS_WIDTH_MASK, bus_width); if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { char *busw[] = { "1 bit", "4 bits", "8 bits" }; device_printf(sc->rtsx_dev, "Setting bus width to %s\n", busw[bus_width]); } return (0); } static int rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing) { if (timing == bus_timing_hs && sc->rtsx_force_timing) { timing = bus_timing_uhs_sdr50; sc->rtsx_ios_timing = timing; } if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_sd_timing(%u)\n", timing); switch (timing) { case bus_timing_uhs_sdr50: case bus_timing_uhs_sdr104: sc->rtsx_double_clk = false; sc->rtsx_vpclk = true; RTSX_BITOP(sc, RTSX_SD_CFG1, 0x0c | RTSX_SD_ASYNC_FIFO_NOT_RST, RTSX_SD30_MODE | RTSX_SD_ASYNC_FIFO_NOT_RST); RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE, RTSX_CRC_VAR_CLK0 | RTSX_SD30_FIX_CLK | RTSX_SAMPLE_VAR_CLK1); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); break; case bus_timing_hs: RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE); RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE, RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); RTSX_BITOP(sc, RTSX_SD_PUSH_POINT_CTL, RTSX_SD20_TX_SEL_MASK, RTSX_SD20_TX_14_AHEAD); RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL, RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_14_DELAY); break; default: RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE); RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE, RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_SD_PUSH_POINT_CTL, RTSX_SD20_TX_NEG_EDGE); RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL, RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_POS_EDGE); break; } return (0); } /* * Set or change SDCLK frequency or disable the SD clock. * Return zero on success. */ static int rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq) { uint8_t clk; uint8_t clk_divider, n, div, mcu; int error = 0; if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_sd_clock(%u)\n", freq); if (freq == RTSX_SDCLK_OFF) { error = rtsx_stop_sd_clock(sc); return error; } sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_500K; sc->rtsx_discovery_mode = (freq <= 1000000) ? true : false; if (sc->rtsx_discovery_mode) { /* We use 250k(around) here, in discovery stage. */ clk_divider = RTSX_CLK_DIVIDE_128; freq = 30000000; } else { clk_divider = RTSX_CLK_DIVIDE_0; } RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, clk_divider); freq /= 1000000; if (sc->rtsx_discovery_mode || !sc->rtsx_double_clk) clk = freq; else clk = freq * 2; switch (sc->rtsx_device_id) { case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: n = clk * 4 / 5 - 2; break; default: n = clk - 2; break; } if ((clk <= 2) || (n > RTSX_MAX_DIV_N)) return (MMC_ERR_INVALID); mcu = 125 / clk + 3; if (mcu > 15) mcu = 15; /* Make sure that the SSC clock div_n is not less than RTSX_MIN_DIV_N. */ div = RTSX_CLK_DIV_1; while ((n < RTSX_MIN_DIV_N) && (div < RTSX_CLK_DIV_8)) { switch (sc->rtsx_device_id) { case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: n = (((n + 2) * 5 / 4) * 2) * 4 / 5 - 2; break; default: n = (n + 2) * 2 - 2; break; } div++; } if (sc->rtsx_double_clk && sc->rtsx_ssc_depth > 1) sc->rtsx_ssc_depth -= 1; if (div > RTSX_CLK_DIV_1) { if (sc->rtsx_ssc_depth > (div - 1)) sc->rtsx_ssc_depth -= (div - 1); else sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_4M; } /* Enable SD clock. */ error = rtsx_switch_sd_clock(sc, clk, n, div, mcu); return (error); } static int rtsx_stop_sd_clock(struct rtsx_softc *sc) { RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL); RTSX_SET(sc, RTSX_SD_BUS_STAT, RTSX_SD_CLK_FORCE_STOP); return (0); } static int rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu) { if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) { device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - discovery-mode is %s, ssc_depth: %d\n", (sc->rtsx_discovery_mode) ? "true" : "false", sc->rtsx_ssc_depth); device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - clk: %d, n: %d, div: %d, mcu: %d\n", clk, n, div, mcu); } RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CLK_DIV, (div << 4) | mcu); RTSX_CLR(sc, RTSX_SSC_CTL1, RTSX_RSTB); RTSX_BITOP(sc, RTSX_SSC_CTL2, RTSX_SSC_DEPTH_MASK, sc->rtsx_ssc_depth); RTSX_WRITE(sc, RTSX_SSC_DIV_N_0, n); RTSX_BITOP(sc, RTSX_SSC_CTL1, RTSX_RSTB, RTSX_RSTB); if (sc->rtsx_vpclk) { RTSX_CLR(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET); RTSX_BITOP(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET); } /* Wait SSC clock stable. */ DELAY(200); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); return (0); } #ifndef MMCCAM static void rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point) { if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_sd_change_tx_phase() - sample_point: %d\n", sample_point); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK); rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_SELECT_MASK, sample_point); rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, 0); rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0); rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0); } static void rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point) { if (sc->rtsx_debug_mask & RTSX_DEBUG_TUNING) device_printf(sc->rtsx_dev, "rtsx_sd_change_rx_phase() - sample_point: %d\n", sample_point); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK); rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_SELECT_MASK, sample_point); rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, 0); rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0); rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0); } static void rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map) { uint32_t raw_phase_map = 0; int i; int error; for (i = 0; i < RTSX_RX_PHASE_MAX; i++) { error = rtsx_sd_tuning_rx_cmd(sc, (uint8_t)i); if (error == 0) raw_phase_map |= 1 << i; } if (phase_map != NULL) *phase_map = raw_phase_map; } static int rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point) { struct mmc_request req = {}; struct mmc_command cmd = {}; int error = 0; cmd.opcode = MMC_SEND_TUNING_BLOCK; cmd.arg = 0; req.cmd = &cmd; RTSX_LOCK(sc); sc->rtsx_req = &req; rtsx_sd_change_rx_phase(sc, sample_point); rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN, RTSX_SD_RSP_80CLK_TIMEOUT_EN); rtsx_init_cmd(sc, &cmd); rtsx_set_cmd_data_len(sc, 1, 0x40); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_AUTO_TUNING | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Set interrupt post processing */ sc->rtsx_intr_trans_ok = rtsx_sd_tuning_rx_cmd_wakeup; sc->rtsx_intr_trans_ko = rtsx_sd_tuning_rx_cmd_wakeup; /* Run the command queue. */ rtsx_send_cmd(sc); error = rtsx_sd_tuning_rx_cmd_wait(sc, &cmd); if (error) { if (sc->rtsx_debug_mask & RTSX_DEBUG_TUNING) device_printf(sc->rtsx_dev, "rtsx_sd_tuning_rx_cmd() - error: %d\n", error); rtsx_sd_wait_data_idle(sc); rtsx_clear_error(sc); } rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN, 0); sc->rtsx_req = NULL; RTSX_UNLOCK(sc); return (error); } static int rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd) { int status; int mask = RTSX_TRANS_OK_INT | RTSX_TRANS_FAIL_INT; status = sc->rtsx_intr_status & mask; while (status == 0) { if (msleep(&sc->rtsx_intr_status, &sc->rtsx_mtx, 0, "rtsxintr", sc->rtsx_timeout_cmd) == EWOULDBLOCK) { cmd->error = MMC_ERR_TIMEOUT; return (MMC_ERR_TIMEOUT); } status = sc->rtsx_intr_status & mask; } return (cmd->error); } static void rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc) { wakeup(&sc->rtsx_intr_status); } static void rtsx_sd_wait_data_idle(struct rtsx_softc *sc) { int i; uint8_t val; for (i = 0; i < 100; i++) { rtsx_read(sc, RTSX_SD_DATA_STATE, &val); if (val & RTSX_SD_DATA_IDLE) return; DELAY(100); } } static uint8_t rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map) { int start = 0, len = 0; int start_final = 0, len_final = 0; uint8_t final_phase = 0xff; while (start < RTSX_RX_PHASE_MAX) { len = rtsx_sd_get_rx_phase_len(phase_map, start); if (len_final < len) { start_final = start; len_final = len; } start += len ? len : 1; } final_phase = (start_final + len_final / 2) % RTSX_RX_PHASE_MAX; if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_sd_search_final_rx_phase() - phase_map: %x, start_final: %d, len_final: %d, final_phase: %d\n", phase_map, start_final, len_final, final_phase); return final_phase; } static int rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit) { int i; for (i = 0; i < RTSX_RX_PHASE_MAX; i++) { if ((phase_map & (1 << (start_bit + i) % RTSX_RX_PHASE_MAX)) == 0) return i; } return RTSX_RX_PHASE_MAX; } #endif /* !MMCCAM */ #if 0 /* For led */ static int rtsx_led_enable(struct rtsx_softc *sc) { switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_CLR(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF); RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED); break; case RTSX_RTL8411B: RTSX_CLR(sc, RTSX_GPIO_CTL, 0x01); RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED); break; default: RTSX_SET(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON); RTSX_SET(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK); break; } return (0); } static int rtsx_led_disable(struct rtsx_softc *sc) { switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN); RTSX_WRITE(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF); break; case RTSX_RTL8411B: RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN); RTSX_SET(sc, RTSX_GPIO_CTL, 0x01); break; default: RTSX_CLR(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK); RTSX_CLR(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON); break; } return (0); } #endif /* For led */ static uint8_t rtsx_response_type(uint16_t mmc_rsp) { int i; struct rsp_type { uint16_t mmc_rsp; uint8_t rtsx_rsp; } rsp_types[] = { { MMC_RSP_NONE, RTSX_SD_RSP_TYPE_R0 }, { MMC_RSP_R1, RTSX_SD_RSP_TYPE_R1 }, { MMC_RSP_R1B, RTSX_SD_RSP_TYPE_R1B }, { MMC_RSP_R2, RTSX_SD_RSP_TYPE_R2 }, { MMC_RSP_R3, RTSX_SD_RSP_TYPE_R3 }, { MMC_RSP_R4, RTSX_SD_RSP_TYPE_R4 }, { MMC_RSP_R5, RTSX_SD_RSP_TYPE_R5 }, { MMC_RSP_R6, RTSX_SD_RSP_TYPE_R6 }, { MMC_RSP_R7, RTSX_SD_RSP_TYPE_R7 } }; for (i = 0; i < nitems(rsp_types); i++) { if (mmc_rsp == rsp_types[i].mmc_rsp) return (rsp_types[i].rtsx_rsp); } return (0); } /* * Init command buffer with SD command index and argument. */ static void rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd) { sc->rtsx_cmd_index = 0; rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD0, 0xff, RTSX_SD_CMD_START | cmd->opcode); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD1, 0xff, cmd->arg >> 24); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD2, 0xff, cmd->arg >> 16); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD3, 0xff, cmd->arg >> 8); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD4, 0xff, cmd->arg); } /* * Append a properly encoded host command to the host command buffer. */ static void rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg, uint8_t mask, uint8_t data) { KASSERT(sc->rtsx_cmd_index < RTSX_HOSTCMD_MAX, ("rtsx: Too many host commands (%d)\n", sc->rtsx_cmd_index)); uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem); cmd_buffer[sc->rtsx_cmd_index++] = htole32((uint32_t)(cmd & 0x3) << 30) | ((uint32_t)(reg & 0x3fff) << 16) | ((uint32_t)(mask) << 8) | ((uint32_t)data); } /* * Queue commands to configure data transfer size. */ static void rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt) { rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_L, 0xff, block_cnt & 0xff); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_H, 0xff, block_cnt >> 8); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_L, 0xff, byte_cnt & 0xff); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_H, 0xff, byte_cnt >> 8); } /* * Run the command queue. */ static void rtsx_send_cmd(struct rtsx_softc *sc) { if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_send_cmd()\n"); sc->rtsx_intr_status = 0; /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREWRITE); /* Tell the chip where the command buffer is and run the commands. */ WRITE4(sc, RTSX_HCBAR, (uint32_t)sc->rtsx_cmd_buffer); WRITE4(sc, RTSX_HCBCTLR, ((sc->rtsx_cmd_index * 4) & 0x00ffffff) | RTSX_START_CMD | RTSX_HW_AUTO_RSP); } /* * Stop previous command. */ static void rtsx_stop_cmd(struct rtsx_softc *sc) { /* Stop command transfer. */ WRITE4(sc, RTSX_HCBCTLR, RTSX_STOP_CMD); /* Stop DMA transfer. */ WRITE4(sc, RTSX_HDBCTLR, RTSX_STOP_DMA); switch (sc->rtsx_device_id) { case RTSX_RTS5260: rtsx_write(sc, RTSX_RTS5260_DMA_RST_CTL_0, RTSX_RTS5260_DMA_RST | RTSX_RTS5260_ADMA3_RST, RTSX_RTS5260_DMA_RST | RTSX_RTS5260_ADMA3_RST); rtsx_write(sc, RTSX_RBCTL, RTSX_RB_FLUSH, RTSX_RB_FLUSH); break; default: rtsx_write(sc, RTSX_DMACTL, RTSX_DMA_RST, RTSX_DMA_RST); rtsx_write(sc, RTSX_RBCTL, RTSX_RB_FLUSH, RTSX_RB_FLUSH); break; } } /* * Clear error. */ static void rtsx_clear_error(struct rtsx_softc *sc) { /* Clear error. */ rtsx_write(sc, RTSX_CARD_STOP, RTSX_SD_STOP | RTSX_SD_CLR_ERR, RTSX_SD_STOP | RTSX_SD_CLR_ERR); } /* * Signal end of request to mmc/mmcsd. */ static void rtsx_req_done(struct rtsx_softc *sc) { #ifdef MMCCAM union ccb *ccb; #endif /* MMCCAM */ struct mmc_request *req; req = sc->rtsx_req; if (req->cmd->error == MMC_ERR_NONE) { if (req->cmd->opcode == MMC_READ_SINGLE_BLOCK || req->cmd->opcode == MMC_READ_MULTIPLE_BLOCK) sc->rtsx_read_count++; else if (req->cmd->opcode == MMC_WRITE_BLOCK || req->cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK) sc->rtsx_write_count++; } else { rtsx_clear_error(sc); } callout_stop(&sc->rtsx_timeout_callout); sc->rtsx_req = NULL; #ifdef MMCCAM ccb = sc->rtsx_ccb; sc->rtsx_ccb = NULL; ccb->ccb_h.status = (req->cmd->error == 0 ? CAM_REQ_CMP : CAM_REQ_CMP_ERR); xpt_done(ccb); #else /* !MMCCAM */ req->done(req); #endif /* MMCCAM */ } /* * Send request. */ static int rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd) { uint8_t rsp_type; uint16_t reg; if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_send_req() - CMD%d\n", cmd->opcode); /* Convert response type. */ rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK); if (rsp_type == 0) { device_printf(sc->rtsx_dev, "Unknown rsp_type: 0x%lx\n", (cmd->flags & MMC_RSP_MASK)); cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } rtsx_init_cmd(sc, cmd); /* Queue command to set response type. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, rsp_type); /* Use the ping-pong buffer (cmd buffer) for commands which do not transfer data. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE, 0x01, RTSX_PINGPONG_BUFFER); /* Queue commands to perform SD transfer. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_CMD_RSP | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE, RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE); /* If needed queue commands to read back card status response. */ if (rsp_type == RTSX_SD_RSP_TYPE_R2) { /* Read data from ping-pong buffer. */ for (reg = RTSX_PPBUF_BASE2; reg < RTSX_PPBUF_BASE2 + 16; reg++) rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0); } else if (rsp_type != RTSX_SD_RSP_TYPE_R0) { /* Read data from SD_CMDx registers. */ for (reg = RTSX_SD_CMD0; reg <= RTSX_SD_CMD4; reg++) rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0); } rtsx_push_cmd(sc, RTSX_READ_REG_CMD, RTSX_SD_STAT1, 0, 0); /* Set transfer OK function. */ if (sc->rtsx_intr_trans_ok == NULL) sc->rtsx_intr_trans_ok = rtsx_ret_resp; /* Run the command queue. */ rtsx_send_cmd(sc); return (0); } /* * Return response of previous command (case cmd->data == NULL) and complete resquest. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_ret_resp(struct rtsx_softc *sc) { struct mmc_command *cmd; cmd = sc->rtsx_req->cmd; rtsx_set_resp(sc, cmd); rtsx_req_done(sc); } /* * Set response of previous command. */ static void rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd) { uint8_t rsp_type; rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK); /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE); /* Copy card response into mmc response buffer. */ if (ISSET(cmd->flags, MMC_RSP_PRESENT)) { uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem); if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) { device_printf(sc->rtsx_dev, "cmd_buffer: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", cmd_buffer[0], cmd_buffer[1], cmd_buffer[2], cmd_buffer[3], cmd_buffer[4]); } if (rsp_type == RTSX_SD_RSP_TYPE_R2) { /* First byte is CHECK_REG_CMD return value, skip it. */ unsigned char *ptr = (unsigned char *)cmd_buffer + 1; int i; /* * The controller offloads the last byte {CRC-7, end bit 1} * of response type R2. Assign dummy CRC, 0, and end bit to this * byte (ptr[16], goes into the LSB of resp[3] later). */ ptr[16] = 0x01; /* The second byte is the status of response, skip it. */ for (i = 0; i < 4; i++) cmd->resp[i] = be32dec(ptr + 1 + i * 4); } else { /* * First byte is CHECK_REG_CMD return value, second * one is the command op code -- we skip those. */ cmd->resp[0] = ((be32toh(cmd_buffer[0]) & 0x0000ffff) << 16) | ((be32toh(cmd_buffer[1]) & 0xffff0000) >> 16); } if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "cmd->resp: 0x%08x 0x%08x 0x%08x 0x%08x\n", cmd->resp[0], cmd->resp[1], cmd->resp[2], cmd->resp[3]); } } /* * Use the ping-pong buffer (cmd buffer) for transfer <= 512 bytes. */ static int rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd) { int read; if (cmd->data == NULL || cmd->data->len == 0) { cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ? RTSX_MAX_DATA_BLKLEN : cmd->data->len; read = ISSET(cmd->data->flags, MMC_DATA_READ); if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_xfer_short() - %s xfer: %ld bytes with block size %ld\n", read ? "Read" : "Write", (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len); if (cmd->data->len > 512) { device_printf(sc->rtsx_dev, "rtsx_xfer_short() - length too large: %ld > 512\n", (unsigned long)cmd->data->len); cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } if (read) { if (sc->rtsx_discovery_mode) rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_0); rtsx_init_cmd(sc, cmd); /* Queue commands to configure data transfer size. */ rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len); /* From Linux: rtsx_pci_sdmmc.c sd_read_data(). */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6); /* Use the ping-pong buffer (cmd buffer). */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE, 0x01, RTSX_PINGPONG_BUFFER); /* Queue commands to perform SD transfer. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_NORMAL_READ | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_ask_ppbuf_part1; /* Run the command queue. */ rtsx_send_cmd(sc); } else { /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part1; /* Run the command queue. */ rtsx_send_req(sc, cmd); } return (0); } /* * Use the ping-pong buffer (cmd buffer) for the transfer - first part <= 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_ask_ppbuf_part1(struct rtsx_softc *sc) { struct mmc_command *cmd; uint16_t reg = RTSX_PPBUF_BASE2; int len; int i; cmd = sc->rtsx_req->cmd; len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len; sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0); } /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part1; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Get the data from the ping-pong buffer (cmd buffer) - first part <= 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_get_ppbuf_part1(struct rtsx_softc *sc) { struct mmc_command *cmd; uint8_t *ptr; int len; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len; /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE); memcpy(ptr, sc->rtsx_cmd_dmamem, len); len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? cmd->data->len - RTSX_HOSTCMD_MAX : 0; /* Use the ping-pong buffer (cmd buffer) for the transfer - second part > 256 bytes. */ if (len > 0) { uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX; int i; sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0); } /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part2; /* Run the command queue. */ rtsx_send_cmd(sc); } else { if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD && cmd->opcode == ACMD_SEND_SCR) { uint8_t *ptr = cmd->data->data; device_printf(sc->rtsx_dev, "SCR: 0x%02x%02x%02x%02x%02x%02x%02x%02x\n", ptr[0], ptr[1], ptr[2], ptr[3], ptr[4], ptr[5], ptr[6], ptr[7]); } if (sc->rtsx_discovery_mode) rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128); rtsx_req_done(sc); } } /* * Get the data from the ping-pong buffer (cmd buffer) - second part > 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_get_ppbuf_part2(struct rtsx_softc *sc) { struct mmc_command *cmd; uint8_t *ptr; int len; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; ptr += RTSX_HOSTCMD_MAX; len = cmd->data->len - RTSX_HOSTCMD_MAX; /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE); memcpy(ptr, sc->rtsx_cmd_dmamem, len); if (sc->rtsx_discovery_mode) rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128); rtsx_req_done(sc); } /* * Use the ping-pong buffer (cmd buffer) for transfer - first part <= 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_put_ppbuf_part1(struct rtsx_softc *sc) { struct mmc_command *cmd; uint16_t reg = RTSX_PPBUF_BASE2; uint8_t *ptr; int len; int i; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len; rtsx_set_resp(sc, cmd); sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr); ptr++; } /* Set transfer OK function. */ if (cmd->data->len > RTSX_HOSTCMD_MAX) sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part2; else sc->rtsx_intr_trans_ok = rtsx_write_ppbuf; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Use the ping-pong buffer (cmd buffer) for transfer - second part > 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_put_ppbuf_part2(struct rtsx_softc *sc) { struct mmc_command *cmd; uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX; uint8_t *ptr; int len; int i; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; ptr += RTSX_HOSTCMD_MAX; len = cmd->data->len - RTSX_HOSTCMD_MAX; sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr); ptr++; } /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_write_ppbuf; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Write the data previously given via the ping-pong buffer on the card. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_write_ppbuf(struct rtsx_softc *sc) { struct mmc_command *cmd; cmd = sc->rtsx_req->cmd; sc->rtsx_cmd_index = 0; /* Queue commands to configure data transfer size. */ rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len); /* From Linux: rtsx_pci_sdmmc.c sd_write_data(). */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_0); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_AUTO_WRITE3 | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_req_done; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Use the data buffer for transfer > 512 bytes. */ static int rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd) { int read = ISSET(cmd->data->flags, MMC_DATA_READ); cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ? RTSX_MAX_DATA_BLKLEN : cmd->data->len; if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_xfer() - %s xfer: %ld bytes with block size %ld\n", read ? "Read" : "Write", (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len); if (cmd->data->len > RTSX_DMA_DATA_BUFSIZE) { - device_printf(sc->rtsx_dev, "rtsx_xfer() length too large: %ld > %d\n", + device_printf(sc->rtsx_dev, "rtsx_xfer() length too large: %ld > %ld\n", (unsigned long)cmd->data->len, RTSX_DMA_DATA_BUFSIZE); cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } if (!read) { /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_xfer_begin; /* Run the command queue. */ rtsx_send_req(sc, cmd); } else { rtsx_xfer_start(sc); } return (0); } /* * Get request response and start dma data transfer (write command). * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_xfer_begin(struct rtsx_softc *sc) { struct mmc_command *cmd; cmd = sc->rtsx_req->cmd; if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_xfer_begin() - CMD%d\n", cmd->opcode); rtsx_set_resp(sc, cmd); rtsx_xfer_start(sc); } /* * Start dma data transfer. */ static void rtsx_xfer_start(struct rtsx_softc *sc) { struct mmc_command *cmd; int read; uint8_t cfg2; int dma_dir; int tmode; cmd = sc->rtsx_req->cmd; read = ISSET(cmd->data->flags, MMC_DATA_READ); /* Configure DMA transfer mode parameters. */ if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK) cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_6; else cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_0; if (read) { dma_dir = RTSX_DMA_DIR_FROM_CARD; /* * Use transfer mode AUTO_READ1, which assume we not * already send the read command and don't need to send * CMD 12 manually after read. */ tmode = RTSX_TM_AUTO_READ1; cfg2 |= RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC7; rtsx_init_cmd(sc, cmd); } else { dma_dir = RTSX_DMA_DIR_TO_CARD; /* * Use transfer mode AUTO_WRITE3, wich assumes we've already * sent the write command and gotten the response, and will * send CMD 12 manually after writing. */ tmode = RTSX_TM_AUTO_WRITE3; cfg2 |= RTSX_SD_NO_CALCULATE_CRC7 | RTSX_SD_NO_CHECK_CRC7; sc->rtsx_cmd_index = 0; } /* Queue commands to configure data transfer size. */ rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len); /* Configure DMA controller. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_IRQSTAT0, RTSX_DMA_DONE_INT, RTSX_DMA_DONE_INT); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC3, 0xff, cmd->data->len >> 24); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC2, 0xff, cmd->data->len >> 16); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC1, 0xff, cmd->data->len >> 8); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC0, 0xff, cmd->data->len); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMACTL, RTSX_DMA_EN | RTSX_DMA_DIR | RTSX_DMA_PACK_SIZE_MASK, RTSX_DMA_EN | dma_dir | RTSX_DMA_512); /* Use the DMA ring buffer for commands which transfer data. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE, 0x01, RTSX_RING_BUFFER); /* Queue command to set response type. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, cfg2); /* Queue commands to perform SD transfer. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, tmode | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Run the command queue. */ rtsx_send_cmd(sc); if (!read) memcpy(sc->rtsx_data_dmamem, cmd->data->data, cmd->data->len); /* Sync data DMA buffer. */ bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREREAD); bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREWRITE); /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_xfer_finish; /* Tell the chip where the data buffer is and run the transfer. */ WRITE4(sc, RTSX_HDBAR, sc->rtsx_data_buffer); WRITE4(sc, RTSX_HDBCTLR, RTSX_TRIG_DMA | (read ? RTSX_DMA_READ : 0) | (cmd->data->len & 0x00ffffff)); } /* * Finish dma data transfer. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_xfer_finish(struct rtsx_softc *sc) { struct mmc_command *cmd; int read; cmd = sc->rtsx_req->cmd; if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_xfer_finish() - CMD%d\n", cmd->opcode); read = ISSET(cmd->data->flags, MMC_DATA_READ); /* Sync data DMA buffer. */ bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTWRITE); if (read) { memcpy(cmd->data->data, sc->rtsx_data_dmamem, cmd->data->len); rtsx_req_done(sc); } else { /* Send CMD12 after AUTO_WRITE3 (see mmcsd_rw() in mmcsd.c) */ /* and complete request. */ sc->rtsx_intr_trans_ok = NULL; rtsx_send_req(sc, sc->rtsx_req->stop); } } /* * Manage request timeout. */ static void rtsx_timeout(void *arg) { struct rtsx_softc *sc; sc = (struct rtsx_softc *)arg; if (sc->rtsx_req != NULL) { device_printf(sc->rtsx_dev, "Controller timeout for CMD%u\n", sc->rtsx_req->cmd->opcode); sc->rtsx_req->cmd->error = MMC_ERR_TIMEOUT; rtsx_stop_cmd(sc); rtsx_req_done(sc); } else { device_printf(sc->rtsx_dev, "Controller timeout!\n"); } } #ifdef MMCCAM static int rtsx_get_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts) { struct rtsx_softc *sc; sc = device_get_softc(dev); cts->host_ocr = sc->rtsx_host.host_ocr; cts->host_f_min = sc->rtsx_host.f_min; cts->host_f_max = sc->rtsx_host.f_max; cts->host_caps = sc->rtsx_host.caps; cts->host_max_data = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE; memcpy(&cts->ios, &sc->rtsx_host.ios, sizeof(struct mmc_ios)); return (0); } /* * Apply settings and return status accordingly. */ static int rtsx_set_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts) { struct rtsx_softc *sc; struct mmc_ios *ios; struct mmc_ios *new_ios; sc = device_get_softc(dev); ios = &sc->rtsx_host.ios; new_ios = &cts->ios; /* Update only requested fields */ if (cts->ios_valid & MMC_CLK) { ios->clock = new_ios->clock; sc->rtsx_ios_clock = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - clock: %u\n", ios->clock); } if (cts->ios_valid & MMC_VDD) { ios->vdd = new_ios->vdd; if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - vdd: %d\n", ios->vdd); } if (cts->ios_valid & MMC_CS) { ios->chip_select = new_ios->chip_select; if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - chip_select: %d\n", ios->chip_select); } if (cts->ios_valid & MMC_BW) { ios->bus_width = new_ios->bus_width; sc->rtsx_ios_bus_width = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - bus width: %d\n", ios->bus_width); } if (cts->ios_valid & MMC_PM) { ios->power_mode = new_ios->power_mode; sc->rtsx_ios_power_mode = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - power mode: %d\n", ios->power_mode); } if (cts->ios_valid & MMC_BT) { ios->timing = new_ios->timing; sc->rtsx_ios_timing = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - timing: %d\n", ios->timing); } if (cts->ios_valid & MMC_BM) { ios->bus_mode = new_ios->bus_mode; if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - bus mode: %d\n", ios->bus_mode); } #if __FreeBSD_version >= 1300000 if (cts->ios_valid & MMC_VCCQ) { ios->vccq = new_ios->vccq; sc->rtsx_ios_vccq = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - vccq: %d\n", ios->vccq); } #endif /* __FreeBSD_version >= 1300000 */ if (rtsx_mmcbr_update_ios(sc->rtsx_dev, NULL) == 0) return (CAM_REQ_CMP); else return (CAM_REQ_CMP_ERR); } /* * Build a request and run it. */ static int rtsx_cam_request(device_t dev, union ccb *ccb) { struct rtsx_softc *sc; sc = device_get_softc(dev); RTSX_LOCK(sc); if (sc->rtsx_ccb != NULL) { RTSX_UNLOCK(sc); return (CAM_BUSY); } sc->rtsx_ccb = ccb; sc->rtsx_cam_req.cmd = &ccb->mmcio.cmd; sc->rtsx_cam_req.stop = &ccb->mmcio.stop; RTSX_UNLOCK(sc); rtsx_mmcbr_request(sc->rtsx_dev, NULL, &sc->rtsx_cam_req); return (0); } #endif /* MMCCAM */ static int rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct rtsx_softc *sc; sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: /* ivar 0 - 1 = opendrain, 2 = pushpull */ *result = sc->rtsx_host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: /* ivar 1 - 0 = 1b 2 = 4b, 3 = 8b */ *result = sc->rtsx_host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: /* ivar 2 - O = dontcare, 1 = cs_high, 2 = cs_low */ *result = sc->rtsx_host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: /* ivar 3 - clock in Hz */ *result = sc->rtsx_host.ios.clock; break; case MMCBR_IVAR_F_MIN: /* ivar 4 */ *result = sc->rtsx_host.f_min; break; case MMCBR_IVAR_F_MAX: /* ivar 5 */ *result = sc->rtsx_host.f_max; break; case MMCBR_IVAR_HOST_OCR: /* ivar 6 - host operation conditions register */ *result = sc->rtsx_host.host_ocr; break; case MMCBR_IVAR_MODE: /* ivar 7 - 0 = mode_mmc, 1 = mode_sd */ *result = sc->rtsx_host.mode; break; case MMCBR_IVAR_OCR: /* ivar 8 - operation conditions register */ *result = sc->rtsx_host.ocr; break; case MMCBR_IVAR_POWER_MODE: /* ivar 9 - 0 = off, 1 = up, 2 = on */ *result = sc->rtsx_host.ios.power_mode; break; case MMCBR_IVAR_VDD: /* ivar 11 - voltage power pin */ *result = sc->rtsx_host.ios.vdd; break; case MMCBR_IVAR_VCCQ: /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */ *result = sc->rtsx_host.ios.vccq; break; case MMCBR_IVAR_CAPS: /* ivar 13 */ *result = sc->rtsx_host.caps; break; case MMCBR_IVAR_TIMING: /* ivar 14 - 0 = normal, 1 = timing_hs, ... */ *result = sc->rtsx_host.ios.timing; break; case MMCBR_IVAR_MAX_DATA: /* ivar 15 */ *result = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE; break; case MMCBR_IVAR_RETUNE_REQ: /* ivar 10 */ case MMCBR_IVAR_MAX_BUSY_TIMEOUT: /* ivar 16 */ default: return (EINVAL); } if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(bus, "Read ivar #%d, value %#x / #%d\n", which, *(int *)result, *(int *)result); return (0); } static int rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct rtsx_softc *sc; sc = device_get_softc(bus); if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(bus, "Write ivar #%d, value %#x / #%d\n", which, (int)value, (int)value); switch (which) { case MMCBR_IVAR_BUS_MODE: /* ivar 0 - 1 = opendrain, 2 = pushpull */ sc->rtsx_host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: /* ivar 1 - 0 = 1b 2 = 4b, 3 = 8b */ sc->rtsx_host.ios.bus_width = value; sc->rtsx_ios_bus_width = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; case MMCBR_IVAR_CHIP_SELECT: /* ivar 2 - O = dontcare, 1 = cs_high, 2 = cs_low */ sc->rtsx_host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: /* ivar 3 - clock in Hz */ sc->rtsx_host.ios.clock = value; sc->rtsx_ios_clock = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; case MMCBR_IVAR_MODE: /* ivar 7 - 0 = mode_mmc, 1 = mode_sd */ sc->rtsx_host.mode = value; break; case MMCBR_IVAR_OCR: /* ivar 8 - operation conditions register */ sc->rtsx_host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: /* ivar 9 - 0 = off, 1 = up, 2 = on */ sc->rtsx_host.ios.power_mode = value; sc->rtsx_ios_power_mode = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; case MMCBR_IVAR_VDD: /* ivar 11 - voltage power pin */ sc->rtsx_host.ios.vdd = value; break; case MMCBR_IVAR_VCCQ: /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */ sc->rtsx_host.ios.vccq = value; sc->rtsx_ios_vccq = value; /* rtsx_mmcbr_switch_vccq() will be called by mmc.c (MMCCAM undef). */ break; case MMCBR_IVAR_TIMING: /* ivar 14 - 0 = normal, 1 = timing_hs, ... */ sc->rtsx_host.ios.timing = value; sc->rtsx_ios_timing = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; /* These are read-only. */ case MMCBR_IVAR_F_MIN: /* ivar 4 */ case MMCBR_IVAR_F_MAX: /* ivar 5 */ case MMCBR_IVAR_HOST_OCR: /* ivar 6 - host operation conditions register */ case MMCBR_IVAR_RETUNE_REQ: /* ivar 10 */ case MMCBR_IVAR_CAPS: /* ivar 13 */ case MMCBR_IVAR_MAX_DATA: /* ivar 15 */ case MMCBR_IVAR_MAX_BUSY_TIMEOUT: /* ivar 16 */ default: return (EINVAL); } return (0); } static int rtsx_mmcbr_update_ios(device_t bus, device_t child__unused) { struct rtsx_softc *sc; struct mmc_ios *ios; int error; sc = device_get_softc(bus); ios = &sc->rtsx_host.ios; if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(bus, "rtsx_mmcbr_update_ios()\n"); /* if MMCBR_IVAR_BUS_WIDTH updated. */ if (sc->rtsx_ios_bus_width < 0) { sc->rtsx_ios_bus_width = ios->bus_width; if ((error = rtsx_set_bus_width(sc, ios->bus_width))) return (error); } /* if MMCBR_IVAR_POWER_MODE updated. */ if (sc->rtsx_ios_power_mode < 0) { sc->rtsx_ios_power_mode = ios->power_mode; switch (ios->power_mode) { case power_off: if ((error = rtsx_bus_power_off(sc))) return (error); break; case power_up: if ((error = rtsx_bus_power_on(sc))) return (error); break; case power_on: if ((error = rtsx_bus_power_on(sc))) return (error); break; } } sc->rtsx_double_clk = true; sc->rtsx_vpclk = false; /* if MMCBR_IVAR_TIMING updated. */ if (sc->rtsx_ios_timing < 0) { sc->rtsx_ios_timing = ios->timing; if ((error = rtsx_set_sd_timing(sc, ios->timing))) return (error); } /* if MMCBR_IVAR_CLOCK updated, must be after rtsx_set_sd_timing() */ if (sc->rtsx_ios_clock < 0) { sc->rtsx_ios_clock = ios->clock; if ((error = rtsx_set_sd_clock(sc, ios->clock))) return (error); } /* if MMCCAM and vccq updated */ if (sc->rtsx_ios_vccq < 0) { sc->rtsx_ios_vccq = ios->vccq; if ((error = rtsx_mmcbr_switch_vccq(sc->rtsx_dev, NULL))) return (error); } return (0); } /* * Set output stage logic power voltage. */ static int rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused) { struct rtsx_softc *sc; int vccq = 0; int error; sc = device_get_softc(bus); switch (sc->rtsx_host.ios.vccq) { case vccq_120: vccq = 120; break; case vccq_180: vccq = 180; break; case vccq_330: vccq = 330; break; }; /* It seems it is always vccq_330. */ if (vccq == 330) { switch (sc->rtsx_device_id) { uint16_t val; case RTSX_RTS5227: if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4))) return (error); if ((error = rtsx_rts5227_fill_driving(sc))) return (error); break; case RTSX_RTS5209: case RTSX_RTS5229: RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3); if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4))) return (error); break; case RTSX_RTS522A: if ((error = rtsx_write_phy(sc, 0x08, 0x57E4))) return (error); if ((error = rtsx_rts5227_fill_driving(sc))) return (error); break; case RTSX_RTS525A: RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_LDO_D3318_MASK, RTSX_LDO_D3318_33V); RTSX_BITOP(sc, RTSX_SD_PAD_CTL, RTSX_SD_IO_USING_1V8, 0); if ((error = rtsx_rts5249_fill_driving(sc))) return (error); break; case RTSX_RTS5249: if ((error = rtsx_read_phy(sc, RTSX_PHY_TUNE, &val))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE, (val & RTSX_PHY_TUNE_VOLTAGE_MASK) | RTSX_PHY_TUNE_VOLTAGE_3V3))) return (error); if ((error = rtsx_rts5249_fill_driving(sc))) return (error); break; case RTSX_RTS5260: RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_VDD1, RTSX_DV331812_VDD1); RTSX_BITOP(sc, RTSX_LDO_DV18_CFG, RTSX_DV331812_MASK, RTSX_DV331812_33); RTSX_CLR(sc, RTSX_SD_PAD_CTL, RTSX_SD_IO_USING_1V8); if ((error = rtsx_rts5260_fill_driving(sc))) return (error); break; case RTSX_RTL8402: RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3); RTSX_BITOP(sc, RTSX_LDO_CTL, (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_MASK, (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_3V3); break; case RTSX_RTL8411: case RTSX_RTL8411B: RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3); RTSX_BITOP(sc, RTSX_LDO_CTL, (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_MASK, (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_3V3); break; } DELAY(300); } if (sc->rtsx_debug_mask & (RTSX_DEBUG_BASIC | RTSX_TRACE_SD_CMD)) device_printf(sc->rtsx_dev, "rtsx_mmcbr_switch_vccq(%d)\n", vccq); return (0); } #ifndef MMCCAM /* * Tune card if bus_timing_uhs_sdr50. */ static int rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400) { struct rtsx_softc *sc; uint32_t raw_phase_map[RTSX_RX_TUNING_CNT] = {0}; uint32_t phase_map; uint8_t final_phase; int i; sc = device_get_softc(bus); if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - hs400 is %s\n", (hs400) ? "true" : "false"); if (sc->rtsx_ios_timing != bus_timing_uhs_sdr50) return (0); sc->rtsx_tuning_mode = true; switch (sc->rtsx_device_id) { case RTSX_RTS5209: case RTSX_RTS5227: rtsx_sd_change_tx_phase(sc, 27); break; case RTSX_RTS522A: rtsx_sd_change_tx_phase(sc, 20); break; case RTSX_RTS5229: rtsx_sd_change_tx_phase(sc, 27); break; case RTSX_RTS525A: case RTSX_RTS5249: rtsx_sd_change_tx_phase(sc, 29); break; case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: rtsx_sd_change_tx_phase(sc, 7); break; } /* trying rx tuning for bus_timing_uhs_sdr50. */ for (i = 0; i < RTSX_RX_TUNING_CNT; i++) { rtsx_sd_tuning_rx_phase(sc, &(raw_phase_map[i])); if (raw_phase_map[i] == 0) break; } phase_map = 0xffffffff; for (i = 0; i < RTSX_RX_TUNING_CNT; i++) { if (sc->rtsx_debug_mask & (RTSX_DEBUG_BASIC | RTSX_DEBUG_TUNING)) device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX raw_phase_map[%d]: 0x%08x\n", i, raw_phase_map[i]); phase_map &= raw_phase_map[i]; } if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX phase_map: 0x%08x\n", phase_map); if (phase_map) { final_phase = rtsx_sd_search_final_rx_phase(sc, phase_map); if (final_phase != 0xff) { rtsx_sd_change_rx_phase(sc, final_phase); } } sc->rtsx_tuning_mode = false; return (0); } static int rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused) { struct rtsx_softc *sc; sc = device_get_softc(bus); if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_mmcbr_retune()\n"); return (0); } #endif /* !MMCCAM */ static int rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req) { struct rtsx_softc *sc; struct mmc_command *cmd; int timeout; int error; sc = device_get_softc(bus); RTSX_LOCK(sc); if (sc->rtsx_req != NULL) { RTSX_UNLOCK(sc); return (EBUSY); } sc->rtsx_req = req; cmd = req->cmd; cmd->error = error = MMC_ERR_NONE; sc->rtsx_intr_status = 0; sc->rtsx_intr_trans_ok = NULL; sc->rtsx_intr_trans_ko = rtsx_req_done; if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(sc->rtsx_dev, "rtsx_mmcbr_request(CMD%u arg %#x, flags %#x, dlen %u, dflags %#x)\n", cmd->opcode, cmd->arg, cmd->flags, cmd->data != NULL ? (unsigned int)cmd->data->len : 0, cmd->data != NULL ? cmd->data->flags : 0); /* Check if card present. */ if (!ISSET(sc->rtsx_flags, RTSX_F_CARD_PRESENT)) { cmd->error = error = MMC_ERR_FAILED; goto end; } /* Refuse SDIO probe if the chip doesn't support SDIO. */ if (cmd->opcode == IO_SEND_OP_COND && !ISSET(sc->rtsx_flags, RTSX_F_SDIO_SUPPORT)) { cmd->error = error = MMC_ERR_INVALID; goto end; } /* Return MMC_ERR_TIMEOUT for SD_IO_RW_DIRECT and IO_SEND_OP_COND. */ if (cmd->opcode == SD_IO_RW_DIRECT || cmd->opcode == IO_SEND_OP_COND) { cmd->error = error = MMC_ERR_TIMEOUT; goto end; } /* Select SD card. */ RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL); RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD); if (cmd->data == NULL) { DELAY(200); timeout = sc->rtsx_timeout_cmd; error = rtsx_send_req(sc, cmd); } else if (cmd->data->len <= 512) { timeout = sc->rtsx_timeout_io; error = rtsx_xfer_short(sc, cmd); } else { timeout = sc->rtsx_timeout_io; error = rtsx_xfer(sc, cmd); } end: if (error == MMC_ERR_NONE) { callout_reset(&sc->rtsx_timeout_callout, timeout * hz, rtsx_timeout, sc); } else { rtsx_req_done(sc); } RTSX_UNLOCK(sc); return (error); } #ifndef MMCCAM static int rtsx_mmcbr_get_ro(device_t bus, device_t child __unused) { struct rtsx_softc *sc; sc = device_get_softc(bus); if (sc->rtsx_inversion == 0) return (sc->rtsx_read_only); else return !(sc->rtsx_read_only); } static int rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused) { struct rtsx_softc *sc; sc = device_get_softc(bus); if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(bus, "rtsx_mmcbr_acquire_host()\n"); RTSX_LOCK(sc); while (sc->rtsx_bus_busy) msleep(&sc->rtsx_bus_busy, &sc->rtsx_mtx, 0, "rtsxah", 0); sc->rtsx_bus_busy++; RTSX_UNLOCK(sc); return (0); } static int rtsx_mmcbr_release_host(device_t bus, device_t child __unused) { struct rtsx_softc *sc; sc = device_get_softc(bus); if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) device_printf(bus, "rtsx_mmcbr_release_host()\n"); RTSX_LOCK(sc); sc->rtsx_bus_busy--; wakeup(&sc->rtsx_bus_busy); RTSX_UNLOCK(sc); return (0); } #endif /* !MMCCAM */ /* * * PCI Support Functions * */ /* * Compare the device ID (chip) of this device against the IDs that this driver * supports. If there is a match, set the description and return success. */ static int rtsx_probe(device_t dev) { uint16_t vendor_id; uint16_t device_id; int i; vendor_id = pci_get_vendor(dev); device_id = pci_get_device(dev); if (vendor_id != RTSX_REALTEK) return (ENXIO); for (i = 0; i < nitems(rtsx_ids); i++) { if (rtsx_ids[i].device_id == device_id) { device_set_desc(dev, rtsx_ids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } /* * Attach function is only called if the probe is successful. */ static int rtsx_attach(device_t dev) { struct rtsx_softc *sc = device_get_softc(dev); uint16_t vendor_id; uint16_t device_id; struct sysctl_ctx_list *ctx; struct sysctl_oid_list *tree; int msi_count = 1; uint32_t sdio_cfg; int error; char *maker; char *family; char *product; int i; vendor_id = pci_get_vendor(dev); device_id = pci_get_device(dev); if (bootverbose) device_printf(dev, "Attach - Vendor ID: 0x%x - Device ID: 0x%x\n", vendor_id, device_id); sc->rtsx_dev = dev; sc->rtsx_device_id = device_id; sc->rtsx_req = NULL; sc->rtsx_timeout_cmd = 1; sc->rtsx_timeout_io = 10; sc->rtsx_read_only = 0; sc->rtsx_inversion = 0; sc->rtsx_force_timing = 0; sc->rtsx_debug_mask = 0; sc->rtsx_read_count = 0; sc->rtsx_write_count = 0; maker = kern_getenv("smbios.system.maker"); family = kern_getenv("smbios.system.family"); product = kern_getenv("smbios.system.product"); for (i = 0; rtsx_inversion_models[i].maker != NULL; i++) { if (strcmp(rtsx_inversion_models[i].maker, maker) == 0 && strcmp(rtsx_inversion_models[i].family, family) == 0 && strcmp(rtsx_inversion_models[i].product, product) == 0) { device_printf(dev, "Inversion activated for %s/%s/%s, see BUG in rtsx(4)\n", maker, family, product); device_printf(dev, "If a card is detected without an SD card present," " add dev.rtsx.0.inversion=0 in loader.conf(5)\n"); sc->rtsx_inversion = 1; } } RTSX_LOCK_INIT(sc); ctx = device_get_sysctl_ctx(dev); tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); SYSCTL_ADD_INT(ctx, tree, OID_AUTO, "timeout_io", CTLFLAG_RW, &sc->rtsx_timeout_io, 0, "Request timeout for I/O commands in seconds"); SYSCTL_ADD_INT(ctx, tree, OID_AUTO, "timeout_cmd", CTLFLAG_RW, &sc->rtsx_timeout_cmd, 0, "Request timeout for setup commands in seconds"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "read_only", CTLFLAG_RD, &sc->rtsx_read_only, 0, "Card is write protected"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "inversion", CTLFLAG_RWTUN, &sc->rtsx_inversion, 0, "Inversion of card detection and read only status"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "force_timing", CTLFLAG_RW, &sc->rtsx_force_timing, 0, "Force bus_timing_uhs_sdr50"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "debug_mask", CTLFLAG_RWTUN, &sc->rtsx_debug_mask, 0, "debugging mask, see rtsx(4)"); SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "read_count", CTLFLAG_RD | CTLFLAG_STATS, &sc->rtsx_read_count, 0, "Count of read operations"); SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "write_count", CTLFLAG_RD | CTLFLAG_STATS, &sc->rtsx_write_count, 0, "Count of write operations"); if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) device_printf(dev, "We are running with inversion: %d\n", sc->rtsx_inversion); /* Allocate IRQ. */ sc->rtsx_irq_res_id = 0; if (pci_alloc_msi(dev, &msi_count) == 0) sc->rtsx_irq_res_id = 1; sc->rtsx_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->rtsx_irq_res_id, RF_ACTIVE | (sc->rtsx_irq_res_id != 0 ? 0 : RF_SHAREABLE)); if (sc->rtsx_irq_res == NULL) { device_printf(dev, "Can't allocate IRQ resources for %d\n", sc->rtsx_irq_res_id); pci_release_msi(dev); return (ENXIO); } callout_init_mtx(&sc->rtsx_timeout_callout, &sc->rtsx_mtx, 0); /* Allocate memory resource. */ if (sc->rtsx_device_id == RTSX_RTS525A) sc->rtsx_mem_res_id = PCIR_BAR(1); else sc->rtsx_mem_res_id = PCIR_BAR(0); sc->rtsx_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->rtsx_mem_res_id, RF_ACTIVE); if (sc->rtsx_mem_res == NULL) { device_printf(dev, "Can't allocate memory resource for %d\n", sc->rtsx_mem_res_id); goto destroy_rtsx_irq_res; } if (bootverbose) device_printf(dev, "rtsx_irq_res_id: %d, rtsx_mem_res_id: %d\n", sc->rtsx_irq_res_id, sc->rtsx_mem_res_id); sc->rtsx_mem_btag = rman_get_bustag(sc->rtsx_mem_res); sc->rtsx_mem_bhandle = rman_get_bushandle(sc->rtsx_mem_res); TIMEOUT_TASK_INIT(taskqueue_swi_giant, &sc->rtsx_card_insert_task, 0, rtsx_card_task, sc); TASK_INIT(&sc->rtsx_card_remove_task, 0, rtsx_card_task, sc); /* Allocate two DMA buffers: a command buffer and a data buffer. */ error = rtsx_dma_alloc(sc); if (error) goto destroy_rtsx_irq_res; /* Activate the interrupt. */ error = bus_setup_intr(dev, sc->rtsx_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, rtsx_intr, sc, &sc->rtsx_irq_cookie); if (error) { device_printf(dev, "Can't set up irq [0x%x]!\n", error); goto destroy_rtsx_mem_res; } pci_enable_busmaster(dev); if (rtsx_read_cfg(sc, 0, RTSX_SDIOCFG_REG, &sdio_cfg) == 0) { if ((sdio_cfg & RTSX_SDIOCFG_SDIO_ONLY) || (sdio_cfg & RTSX_SDIOCFG_HAVE_SDIO)) sc->rtsx_flags |= RTSX_F_SDIO_SUPPORT; } #ifdef MMCCAM sc->rtsx_ccb = NULL; sc->rtsx_cam_status = 0; SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "cam_status", CTLFLAG_RD, &sc->rtsx_cam_status, 0, "driver cam card present"); if (mmc_cam_sim_alloc(dev, "rtsx_mmc", &sc->rtsx_mmc_sim) != 0) { device_printf(dev, "Can't allocate CAM SIM\n"); goto destroy_rtsx_irq; } #endif /* MMCCAM */ /* Initialize device. */ error = rtsx_init(sc); if (error) { device_printf(dev, "Error %d during rtsx_init()\n", error); goto destroy_rtsx_irq; } /* * Schedule a card detection as we won't get an interrupt * if the card is inserted when we attach. We wait a quarter * of a second to allow for a "spontaneous" interrupt which may * change the card presence state. This delay avoid a panic * on some configuration (e.g. Lenovo T540p). */ DELAY(250000); if (rtsx_is_card_present(sc)) device_printf(sc->rtsx_dev, "A card is detected\n"); else device_printf(sc->rtsx_dev, "No card is detected\n"); rtsx_card_task(sc, 0); if (bootverbose) device_printf(dev, "Device attached\n"); return (0); destroy_rtsx_irq: bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie); destroy_rtsx_mem_res: bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_mem_res_id, sc->rtsx_mem_res); rtsx_dma_free(sc); destroy_rtsx_irq_res: callout_drain(&sc->rtsx_timeout_callout); bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id, sc->rtsx_irq_res); pci_release_msi(dev); RTSX_LOCK_DESTROY(sc); return (ENXIO); } static int rtsx_detach(device_t dev) { struct rtsx_softc *sc = device_get_softc(dev); int error; if (bootverbose) device_printf(dev, "Detach - Vendor ID: 0x%x - Device ID: 0x%x\n", pci_get_vendor(dev), sc->rtsx_device_id); /* Disable interrupts. */ sc->rtsx_intr_enabled = 0; WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled); /* Stop device. */ error = device_delete_children(sc->rtsx_dev); sc->rtsx_mmc_dev = NULL; if (error) return (error); taskqueue_drain_timeout(taskqueue_swi_giant, &sc->rtsx_card_insert_task); taskqueue_drain(taskqueue_swi_giant, &sc->rtsx_card_remove_task); /* Teardown the state in our softc created in our attach routine. */ rtsx_dma_free(sc); if (sc->rtsx_mem_res != NULL) bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_mem_res_id, sc->rtsx_mem_res); if (sc->rtsx_irq_cookie != NULL) bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie); if (sc->rtsx_irq_res != NULL) { callout_drain(&sc->rtsx_timeout_callout); bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id, sc->rtsx_irq_res); pci_release_msi(dev); } RTSX_LOCK_DESTROY(sc); #ifdef MMCCAM mmc_cam_sim_free(&sc->rtsx_mmc_sim); #endif /* MMCCAM */ return (0); } static int rtsx_shutdown(device_t dev) { if (bootverbose) device_printf(dev, "Shutdown\n"); return (0); } /* * Device suspend routine. */ static int rtsx_suspend(device_t dev) { struct rtsx_softc *sc = device_get_softc(dev); device_printf(dev, "Suspend\n"); #ifdef MMCCAM if (sc->rtsx_ccb != NULL) { device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n", sc->rtsx_ccb->mmcio.cmd.opcode, sc->rtsx_intr_status); } #else /* !MMCCAM */ if (sc->rtsx_req != NULL) { device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n", sc->rtsx_req->cmd->opcode, sc->rtsx_intr_status); } #endif /* MMCCAM */ bus_generic_suspend(dev); return (0); } /* * Device resume routine. */ static int rtsx_resume(device_t dev) { device_printf(dev, "Resume\n"); rtsx_init(device_get_softc(dev)); bus_generic_resume(dev); return (0); } static device_method_t rtsx_methods[] = { /* Device interface */ DEVMETHOD(device_probe, rtsx_probe), DEVMETHOD(device_attach, rtsx_attach), DEVMETHOD(device_detach, rtsx_detach), DEVMETHOD(device_shutdown, rtsx_shutdown), DEVMETHOD(device_suspend, rtsx_suspend), DEVMETHOD(device_resume, rtsx_resume), /* Bus interface */ DEVMETHOD(bus_read_ivar, rtsx_read_ivar), DEVMETHOD(bus_write_ivar, rtsx_write_ivar), #ifndef MMCCAM /* MMC bridge interface */ DEVMETHOD(mmcbr_update_ios, rtsx_mmcbr_update_ios), DEVMETHOD(mmcbr_switch_vccq, rtsx_mmcbr_switch_vccq), DEVMETHOD(mmcbr_tune, rtsx_mmcbr_tune), DEVMETHOD(mmcbr_retune, rtsx_mmcbr_retune), DEVMETHOD(mmcbr_request, rtsx_mmcbr_request), DEVMETHOD(mmcbr_get_ro, rtsx_mmcbr_get_ro), DEVMETHOD(mmcbr_acquire_host, rtsx_mmcbr_acquire_host), DEVMETHOD(mmcbr_release_host, rtsx_mmcbr_release_host), #endif /* !MMCCAM */ #ifdef MMCCAM /* MMCCAM interface */ DEVMETHOD(mmc_sim_get_tran_settings, rtsx_get_tran_settings), DEVMETHOD(mmc_sim_set_tran_settings, rtsx_set_tran_settings), DEVMETHOD(mmc_sim_cam_request, rtsx_cam_request), #endif /* MMCCAM */ DEVMETHOD_END }; DEFINE_CLASS_0(rtsx, rtsx_driver, rtsx_methods, sizeof(struct rtsx_softc)); DRIVER_MODULE(rtsx, pci, rtsx_driver, NULL, NULL); /* For Plug and Play */ MODULE_PNP_INFO("U16:device;D:#;T:vendor=0x10ec", pci, rtsx, rtsx_ids, nitems(rtsx_ids)); #ifndef MMCCAM MMC_DECLARE_BRIDGE(rtsx); #endif /* !MMCCAM */ diff --git a/sys/kern/subr_param.c b/sys/kern/subr_param.c index 2a721c3c113f..ea5e8d02be2b 100644 --- a/sys/kern/subr_param.c +++ b/sys/kern/subr_param.c @@ -1,351 +1,352 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1980, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_param.h" #include "opt_msgbuf.h" #include "opt_maxphys.h" #include "opt_maxusers.h" #include +#include #include #include #include #include #include #include #include #include #include #include #include /* * System parameter formulae. */ #ifndef HZ # define HZ 1000 # ifndef HZ_VM # define HZ_VM 100 # endif #else # ifndef HZ_VM # define HZ_VM HZ # endif #endif #define NPROC (20 + 16 * maxusers) #ifndef NBUF #define NBUF 0 #endif #ifndef MAXFILES #define MAXFILES (40 + 32 * maxusers) #endif static int sysctl_kern_vm_guest(SYSCTL_HANDLER_ARGS); int hz; /* system clock's frequency */ int tick; /* usec per tick (1000000 / hz) */ time_t tick_seconds_max; /* max hz * seconds an integer can hold */ struct bintime tick_bt; /* bintime per tick (1s / hz) */ sbintime_t tick_sbt; int maxusers; /* base tunable */ int maxproc; /* maximum # of processes */ int maxprocperuid; /* max # of procs per user */ int maxfiles; /* sys. wide open files limit */ int maxfilesperproc; /* per-proc open files limit */ int msgbufsize; /* size of kernel message buffer */ int nbuf; /* number of bcache bufs */ int bio_transient_maxcnt; int ngroups_max; /* max # groups per process */ int nswbuf; pid_t pid_max = PID_MAX; u_long maxswzone; /* max swmeta KVA storage */ u_long maxbcache; /* max buffer cache KVA storage */ u_long maxpipekva; /* Limit on pipe KVA */ u_long maxphys; /* max raw I/O transfer size */ int vm_guest = VM_GUEST_NO; /* Running as virtual machine guest? */ u_long maxtsiz; /* max text size */ u_long dfldsiz; /* initial data size limit */ u_long maxdsiz; /* max data size */ u_long dflssiz; /* initial stack size limit */ u_long maxssiz; /* max stack size */ u_long sgrowsiz; /* amount to grow stack */ SYSCTL_INT(_kern, OID_AUTO, hz, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &hz, 0, "Number of clock ticks per second"); SYSCTL_INT(_kern, OID_AUTO, hz_max, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, HZ_MAXIMUM, "Maximum hz value supported"); SYSCTL_INT(_kern, OID_AUTO, hz_min, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, HZ_MINIMUM, "Minimum hz value supported"); SYSCTL_INT(_kern, OID_AUTO, nbuf, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &nbuf, 0, "Number of buffers in the buffer cache"); SYSCTL_INT(_kern, OID_AUTO, nswbuf, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &nswbuf, 0, "Number of swap buffers"); SYSCTL_INT(_kern, OID_AUTO, msgbufsize, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &msgbufsize, 0, "Size of the kernel message buffer"); SYSCTL_LONG(_kern, OID_AUTO, maxswzone, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxswzone, 0, "Maximum memory for swap metadata"); SYSCTL_LONG(_kern, OID_AUTO, maxbcache, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxbcache, 0, "Maximum value of vfs.maxbufspace"); SYSCTL_INT(_kern, OID_AUTO, bio_transient_maxcnt, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &bio_transient_maxcnt, 0, "Maximum number of transient BIOs mappings"); SYSCTL_ULONG(_kern, OID_AUTO, maxtsiz, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, &maxtsiz, 0, "Maximum text size"); SYSCTL_ULONG(_kern, OID_AUTO, dfldsiz, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, &dfldsiz, 0, "Initial data size limit"); SYSCTL_ULONG(_kern, OID_AUTO, maxdsiz, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, &maxdsiz, 0, "Maximum data size"); SYSCTL_ULONG(_kern, OID_AUTO, dflssiz, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, &dflssiz, 0, "Initial stack size limit"); SYSCTL_ULONG(_kern, OID_AUTO, maxssiz, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, &maxssiz, 0, "Maximum stack size"); SYSCTL_ULONG(_kern, OID_AUTO, sgrowsiz, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, &sgrowsiz, 0, "Amount to grow stack on a stack fault"); SYSCTL_PROC(_kern, OID_AUTO, vm_guest, CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_MPSAFE, NULL, 0, sysctl_kern_vm_guest, "A", "Virtual machine guest detected?"); /* * The elements of this array are ordered based upon the values of the * corresponding enum VM_GUEST members. */ static const char *const vm_guest_sysctl_names[] = { [VM_GUEST_NO] = "none", [VM_GUEST_VM] = "generic", [VM_GUEST_XEN] = "xen", [VM_GUEST_HV] = "hv", [VM_GUEST_VMWARE] = "vmware", [VM_GUEST_KVM] = "kvm", [VM_GUEST_BHYVE] = "bhyve", [VM_GUEST_VBOX] = "vbox", [VM_GUEST_PARALLELS] = "parallels", }; _Static_assert(nitems(vm_guest_sysctl_names) == VM_GUEST_LAST, "new vm guest type not added to vm_guest_sysctl_names"); /* * Boot time overrides that are not scaled against main memory */ void init_param1(void) { TSENTER(); /* * arm64 and riscv currently hard-code the thread0 kstack size * to KSTACK_PAGES, ignoring the tunable. */ TUNABLE_INT_FETCH("kern.kstack_pages", &kstack_pages); hz = -1; TUNABLE_INT_FETCH("kern.hz", &hz); if (hz == -1) hz = vm_guest > VM_GUEST_NO ? HZ_VM : HZ; /* range check the "hz" value */ if (__predict_false(hz < HZ_MINIMUM)) hz = HZ_MINIMUM; else if (__predict_false(hz > HZ_MAXIMUM)) hz = HZ_MAXIMUM; tick = 1000000 / hz; tick_sbt = SBT_1S / hz; tick_bt = sbttobt(tick_sbt); tick_seconds_max = INT_MAX / hz; /* * Arrange for ticks to wrap 10 minutes after boot to help catch * sign problems sooner. */ ticks = INT_MAX - (hz * 10 * 60); vn_lock_pair_pause_max = hz / 100; if (vn_lock_pair_pause_max == 0) vn_lock_pair_pause_max = 1; #ifdef VM_SWZONE_SIZE_MAX maxswzone = VM_SWZONE_SIZE_MAX; #endif TUNABLE_LONG_FETCH("kern.maxswzone", &maxswzone); #ifdef VM_BCACHE_SIZE_MAX maxbcache = VM_BCACHE_SIZE_MAX; #endif TUNABLE_LONG_FETCH("kern.maxbcache", &maxbcache); msgbufsize = MSGBUF_SIZE; TUNABLE_INT_FETCH("kern.msgbufsize", &msgbufsize); maxtsiz = MAXTSIZ; TUNABLE_ULONG_FETCH("kern.maxtsiz", &maxtsiz); dfldsiz = DFLDSIZ; TUNABLE_ULONG_FETCH("kern.dfldsiz", &dfldsiz); maxdsiz = MAXDSIZ; TUNABLE_ULONG_FETCH("kern.maxdsiz", &maxdsiz); dflssiz = DFLSSIZ; TUNABLE_ULONG_FETCH("kern.dflssiz", &dflssiz); maxssiz = MAXSSIZ; TUNABLE_ULONG_FETCH("kern.maxssiz", &maxssiz); sgrowsiz = SGROWSIZ; TUNABLE_ULONG_FETCH("kern.sgrowsiz", &sgrowsiz); /* * Let the administrator set {NGROUPS_MAX}, but disallow values * less than NGROUPS_MAX which would violate POSIX.1-2008 or * greater than INT_MAX-1 which would result in overflow. */ ngroups_max = NGROUPS_MAX; TUNABLE_INT_FETCH("kern.ngroups", &ngroups_max); if (ngroups_max < NGROUPS_MAX) ngroups_max = NGROUPS_MAX; /* * Only allow to lower the maximal pid. * Prevent setting up a non-bootable system if pid_max is too low. */ TUNABLE_INT_FETCH("kern.pid_max", &pid_max); if (pid_max > PID_MAX) pid_max = PID_MAX; else if (pid_max < 300) pid_max = 300; TUNABLE_INT_FETCH("vfs.unmapped_buf_allowed", &unmapped_buf_allowed); TSEXIT(); } /* * Boot time overrides that are scaled against main memory */ void init_param2(long physpages) { TSENTER(); /* Base parameters */ maxusers = MAXUSERS; TUNABLE_INT_FETCH("kern.maxusers", &maxusers); if (maxusers == 0) { maxusers = physpages / (2 * 1024 * 1024 / PAGE_SIZE); if (maxusers < 32) maxusers = 32; #ifdef VM_MAX_AUTOTUNE_MAXUSERS if (maxusers > VM_MAX_AUTOTUNE_MAXUSERS) maxusers = VM_MAX_AUTOTUNE_MAXUSERS; #endif /* * Scales down the function in which maxusers grows once * we hit 384. */ if (maxusers > 384) maxusers = 384 + ((maxusers - 384) / 8); } /* * The following can be overridden after boot via sysctl. Note: * unless overridden, these macros are ultimately based on maxusers. * Limit maxproc so that kmap entries cannot be exhausted by * processes. */ maxproc = NPROC; TUNABLE_INT_FETCH("kern.maxproc", &maxproc); if (maxproc > (physpages / 12)) maxproc = physpages / 12; if (maxproc > pid_max) maxproc = pid_max; maxprocperuid = (maxproc * 9) / 10; /* * The default limit for maxfiles is 1/12 of the number of * physical page but not less than 16 times maxusers. * At most it can be 1/6 the number of physical pages. */ maxfiles = imax(MAXFILES, physpages / 8); TUNABLE_INT_FETCH("kern.maxfiles", &maxfiles); if (maxfiles > (physpages / 4)) maxfiles = physpages / 4; maxfilesperproc = (maxfiles / 10) * 9; TUNABLE_INT_FETCH("kern.maxfilesperproc", &maxfilesperproc); /* * Cannot be changed after boot. */ nbuf = NBUF; TUNABLE_INT_FETCH("kern.nbuf", &nbuf); TUNABLE_INT_FETCH("kern.bio_transient_maxcnt", &bio_transient_maxcnt); maxphys = MAXPHYS; TUNABLE_ULONG_FETCH("kern.maxphys", &maxphys); if (maxphys == 0) { maxphys = MAXPHYS; } else if (__bitcountl(maxphys) != 1) { /* power of two */ if (flsl(maxphys) == NBBY * sizeof(maxphys)) maxphys = MAXPHYS; else maxphys = 1UL << flsl(maxphys); } if (maxphys < PAGE_SIZE) maxphys = MAXPHYS; /* * Physical buffers are pre-allocated buffers (struct buf) that * are used as temporary holders for I/O, such as paging I/O. */ TUNABLE_INT_FETCH("kern.nswbuf", &nswbuf); /* * The default for maxpipekva is min(1/64 of the kernel address space, * max(1/64 of main memory, 512KB)). See sys_pipe.c for more details. */ maxpipekva = ptoa(physpages / 64); TUNABLE_LONG_FETCH("kern.ipc.maxpipekva", &maxpipekva); if (maxpipekva < 512 * 1024) maxpipekva = 512 * 1024; if (maxpipekva > (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 64) maxpipekva = (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 64; TSEXIT(); } /* * Sysctl stringifying handler for kern.vm_guest. */ static int sysctl_kern_vm_guest(SYSCTL_HANDLER_ARGS) { return (SYSCTL_OUT_STR(req, vm_guest_sysctl_names[vm_guest])); } diff --git a/sys/sys/_maxphys.h b/sys/sys/_maxphys.h new file mode 100644 index 000000000000..48cfc4a054ff --- /dev/null +++ b/sys/sys/_maxphys.h @@ -0,0 +1,10 @@ +/*- + * This file is in the public domain. + */ +#ifndef MAXPHYS +#ifdef __ILP32__ +#define MAXPHYS (128 * 1024) +#else +#define MAXPHYS (1024 * 1024) +#endif +#endif diff --git a/sys/sys/param.h b/sys/sys/param.h index 7a091add62f3..e10b4f506520 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -1,389 +1,383 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_PARAM_H_ #define _SYS_PARAM_H_ #include #define BSD 199506 /* System version (year & month). */ #define BSD4_3 1 #define BSD4_4 1 /* * __FreeBSD_version numbers are documented in the Porter's Handbook. * If you bump the version for any reason, you should update the documentation * there. * Currently this lives here in the doc/ repository: * * documentation/content/en/books/porters-handbook/versions/_index.adoc * * Encoding: Rxx * 'R' is in the range 0 to 4 if this is a release branch or * X.0-CURRENT before releng/X.0 is created, otherwise 'R' is * in the range 5 to 9. * Short hand: MMmmXXX * * __FreeBSD_version is bumped every time there's a change in the base system * that's noteworthy. A noteworthy change is any change which changes the * kernel's KBI in -CURRENT, one that changes some detail about the system that * external software (or the ports system) would want to know about, one that * adds a system call, one that adds or deletes a shipped library, a security * fix, or similar change not specifically noted here. Bumps should be limited * to one per day / a couple per week except for security fixes. * * The approved way to obtain this from a shell script is: * awk '/^\#define[[:space:]]*__FreeBSD_version/ {print $3}' * Other methods to parse this file may work, but are not guaranteed against * future changes. The above script works back to FreeBSD 3.x when this macro * was introduced. This number is propagated to other places needing it that * cannot include sys/param.h and should only be updated here. */ #undef __FreeBSD_version #define __FreeBSD_version 1500018 /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, * which by definition is always true on FreeBSD. This macro is also defined * on other systems that use the kernel of FreeBSD, such as GNU/kFreeBSD. * * It is tempting to use this macro in userland code when we want to enable * kernel-specific routines, and in fact it's fine to do this in code that * is part of FreeBSD itself. However, be aware that as presence of this * macro is still not widespread (e.g. older FreeBSD versions, 3rd party * compilers, etc), it is STRONGLY DISCOURAGED to check for this macro in * external applications without also checking for __FreeBSD__ as an * alternative. */ #undef __FreeBSD_kernel__ #define __FreeBSD_kernel__ #if defined(_KERNEL) || defined(_WANT_P_OSREL) #define P_OSREL_SIGWAIT 700000 #define P_OSREL_SIGSEGV 700004 #define P_OSREL_MAP_ANON 800104 #define P_OSREL_MAP_FSTRICT 1100036 #define P_OSREL_SHUTDOWN_ENOTCONN 1100077 #define P_OSREL_MAP_GUARD 1200035 #define P_OSREL_WRFSBASE 1200041 #define P_OSREL_CK_CYLGRP 1200046 #define P_OSREL_VMTOTAL64 1200054 #define P_OSREL_CK_SUPERBLOCK 1300000 #define P_OSREL_CK_INODE 1300005 #define P_OSREL_POWERPC_NEW_AUX_ARGS 1300070 #define P_OSREL_TIDPID 1400079 #define P_OSREL_ARM64_SPSR 1400084 #define P_OSREL_MAJOR(x) ((x) / 100000) #endif #ifndef LOCORE #include #endif /* * Machine-independent constants (some used in following include files). * Redefined constants are from POSIX 1003.1 limits file. * * MAXCOMLEN should be >= sizeof(ac_comm) (see ) */ #include #define MAXCOMLEN 19 /* max command name remembered */ #define MAXINTERP PATH_MAX /* max interpreter file name length */ #define MAXLOGNAME 33 /* max login name length (incl. NUL) */ #define MAXUPRC CHILD_MAX /* max simultaneous processes */ #define NCARGS ARG_MAX /* max bytes for an exec function */ #define NGROUPS (NGROUPS_MAX+1) /* max number groups */ #define NOFILE OPEN_MAX /* max open files per process */ #define NOGROUP 65535 /* marker for empty group set member */ #define MAXHOSTNAMELEN 256 /* max hostname size */ #define SPECNAMELEN 255 /* max length of devicename */ /* More types and definitions used throughout the kernel. */ #ifdef _KERNEL #include #include #ifndef LOCORE #include #include #endif #ifndef FALSE #define FALSE 0 #endif #ifndef TRUE #define TRUE 1 #endif #endif #ifndef _KERNEL #ifndef LOCORE /* Signals. */ #include #endif #endif /* Machine type dependent parameters. */ #include #ifndef _KERNEL #include +#include #endif #ifndef DEV_BSHIFT #define DEV_BSHIFT 9 /* log2(DEV_BSIZE) */ #endif #define DEV_BSIZE (1<>PAGE_SHIFT) #endif /* * btodb() is messy and perhaps slow because `bytes' may be an off_t. We * want to shift an unsigned type to avoid sign extension and we don't * want to widen `bytes' unnecessarily. Assume that the result fits in * a daddr_t. */ #ifndef btodb #define btodb(bytes) /* calculates (bytes / DEV_BSIZE) */ \ (sizeof (bytes) > sizeof(long) \ ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \ : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT)) #endif #ifndef dbtob #define dbtob(db) /* calculates (db * DEV_BSIZE) */ \ ((off_t)(db) << DEV_BSHIFT) #endif #define PRIMASK 0x0ff #define PCATCH 0x100 /* OR'd with pri for tsleep to check signals */ #define PDROP 0x200 /* OR'd with pri to stop re-entry of interlock mutex */ #define PNOLOCK 0x400 /* OR'd with pri to allow sleeping w/o a lock */ #define PRILASTFLAG 0x400 /* Last flag defined above */ #define NZERO 0 /* default "nice" */ #define NBBY 8 /* number of bits in a byte */ #define NBPW sizeof(int) /* number of bytes per word (integer) */ #define CMASK 022 /* default file mask: S_IWGRP|S_IWOTH */ #define NODEV (dev_t)(-1) /* non-existent device */ /* * File system parameters and macros. * * MAXBSIZE - Filesystems are made out of blocks of at most MAXBSIZE bytes * per block. MAXBSIZE may be made larger without effecting * any existing filesystems as long as it does not exceed MAXPHYS, * and may be made smaller at the risk of not being able to use * filesystems which require a block size exceeding MAXBSIZE. * * MAXBCACHEBUF - Maximum size of a buffer in the buffer cache. This must * be >= MAXBSIZE and can be set differently for different * architectures by defining it in . * Making this larger allows NFS to do larger reads/writes. * * BKVASIZE - Nominal buffer space per buffer, in bytes. BKVASIZE is the * minimum KVM memory reservation the kernel is willing to make. * Filesystems can of course request smaller chunks. Actual * backing memory uses a chunk size of a page (PAGE_SIZE). * The default value here can be overridden on a per-architecture * basis by defining it in . * * If you make BKVASIZE too small you risk seriously fragmenting * the buffer KVM map which may slow things down a bit. If you * make it too big the kernel will not be able to optimally use * the KVM memory reserved for the buffer cache and will wind * up with too-few buffers. * * The default is 16384, roughly 2x the block size used by a * normal UFS filesystem. */ #define MAXBSIZE 65536 /* must be power of 2 */ #ifndef MAXBCACHEBUF #define MAXBCACHEBUF MAXBSIZE /* must be a power of 2 >= MAXBSIZE */ #endif #ifndef BKVASIZE #define BKVASIZE 16384 /* must be power of 2 */ #endif #define BKVAMASK (BKVASIZE-1) /* * MAXPATHLEN defines the longest permissible path length after expanding * symbolic links. It is used to allocate a temporary buffer from the buffer * pool in which to do the name expansion, hence should be a power of two, * and must be less than or equal to MAXBSIZE. MAXSYMLINKS defines the * maximum number of symbolic links that may be expanded in a path name. * It should be set high enough to allow all legitimate uses, but halt * infinite loops reasonably quickly. */ #define MAXPATHLEN PATH_MAX #define MAXSYMLINKS 32 /* Bit map related macros. */ #define setbit(a,i) (((unsigned char *)(a))[(i)/NBBY] |= 1<<((i)%NBBY)) #define clrbit(a,i) (((unsigned char *)(a))[(i)/NBBY] &= ~(1<<((i)%NBBY))) #define isset(a,i) \ (((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) #define isclr(a,i) \ ((((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) == 0) /* Macros for counting and rounding. */ #ifndef howmany #define howmany(x, y) (((x)+((y)-1))/(y)) #endif #define nitems(x) (sizeof((x)) / sizeof((x)[0])) #define rounddown(x, y) (((x)/(y))*(y)) #define rounddown2(x, y) __align_down(x, y) /* if y is power of two */ #define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) /* to any y */ #define roundup2(x, y) __align_up(x, y) /* if y is powers of two */ #define powerof2(x) ((((x)-1)&(x))==0) /* Macros for min/max. */ #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) #ifdef _KERNEL /* * Basic byte order function prototypes for non-inline functions. */ #ifndef LOCORE #ifndef _BYTEORDER_PROTOTYPED #define _BYTEORDER_PROTOTYPED __BEGIN_DECLS __uint32_t htonl(__uint32_t); __uint16_t htons(__uint16_t); __uint32_t ntohl(__uint32_t); __uint16_t ntohs(__uint16_t); __END_DECLS #endif #endif #ifndef _BYTEORDER_FUNC_DEFINED #define _BYTEORDER_FUNC_DEFINED #define htonl(x) __htonl(x) #define htons(x) __htons(x) #define ntohl(x) __ntohl(x) #define ntohs(x) __ntohs(x) #endif /* !_BYTEORDER_FUNC_DEFINED */ #endif /* _KERNEL */ /* * Scale factor for scaled integers used to count %cpu time and load avgs. * * The number of CPU `tick's that map to a unique `%age' can be expressed * by the formula (1 / (2 ^ (FSHIFT - 11))). Since the intermediate * calculation is done with 64-bit precision, the maximum load average that can * be calculated is approximately 2^32 / FSCALE. * * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age', * FSHIFT must be at least 11. This gives a maximum load avg of 2 million. */ #define FSHIFT 11 /* bits to right of fixed binary point */ #define FSCALE (1<> (PAGE_SHIFT - DEV_BSHIFT)) #define ctodb(db) /* calculates pages to devblks */ \ ((db) << (PAGE_SHIFT - DEV_BSHIFT)) /* * Old spelling of __containerof(). */ #define member2struct(s, m, x) \ ((struct s *)(void *)((char *)(x) - offsetof(struct s, m))) /* * Access a variable length array that has been declared as a fixed * length array. */ #define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset]) #endif /* _SYS_PARAM_H_ */