diff --git a/bin/sh/jobs.c b/bin/sh/jobs.c index 765e6a2ed3aa..b1bfa0944d71 100644 --- a/bin/sh/jobs.c +++ b/bin/sh/jobs.c @@ -1,1515 +1,1515 @@ /*- * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Kenneth Almquist. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef lint #if 0 static char sccsid[] = "@(#)jobs.c 8.5 (Berkeley) 5/4/95"; #endif #endif /* not lint */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "shell.h" #if JOBS #include #undef CEOF /* syntax.h redefines this */ #endif #include "redir.h" #include "exec.h" #include "show.h" #include "main.h" #include "parser.h" #include "nodes.h" #include "jobs.h" #include "options.h" #include "trap.h" #include "syntax.h" #include "input.h" #include "output.h" #include "memalloc.h" #include "error.h" #include "mystring.h" #include "var.h" #include "builtins.h" static struct job *jobtab; /* array of jobs */ static int njobs; /* size of array */ static pid_t backgndpid = -1; /* pid of last background process */ static struct job *bgjob = NULL; /* last background process */ #if JOBS static struct job *jobmru; /* most recently used job list */ static pid_t initialpgrp; /* pgrp of shell on invocation */ #endif static int ttyfd = -1; /* mode flags for dowait */ #define DOWAIT_BLOCK 0x1 /* wait until a child exits */ -#define DOWAIT_SIG 0x2 /* if DOWAIT_BLOCK, abort on SIGINT/SIGQUIT */ -#define DOWAIT_SIG_ANY 0x4 /* if DOWAIT_SIG, abort on any signal */ +#define DOWAIT_SIG 0x2 /* if DOWAIT_BLOCK, abort on signal */ +#define DOWAIT_SIG_TRAP 0x4 /* if DOWAIT_SIG, abort on trapped signal only */ #if JOBS static void restartjob(struct job *); #endif static void freejob(struct job *); static int waitcmdloop(struct job *); static struct job *getjob_nonotfound(const char *); static struct job *getjob(const char *); pid_t killjob(const char *, int); static pid_t dowait(int, struct job *); static void checkzombies(void); static void cmdtxt(union node *); static void cmdputs(const char *); #if JOBS static void setcurjob(struct job *); static void deljob(struct job *); static struct job *getcurjob(struct job *); #endif static void printjobcmd(struct job *); static void showjob(struct job *, int); /* * Turn job control on and off. */ static int jobctl; #if JOBS static void jobctl_notty(void) { if (ttyfd >= 0) { close(ttyfd); ttyfd = -1; } if (!iflag) { setsignal(SIGTSTP); setsignal(SIGTTOU); setsignal(SIGTTIN); jobctl = 1; return; } out2fmt_flush("sh: can't access tty; job control turned off\n"); mflag = 0; } void setjobctl(int on) { int i; if (on == jobctl || rootshell == 0) return; if (on) { if (ttyfd != -1) close(ttyfd); if ((ttyfd = open(_PATH_TTY, O_RDWR | O_CLOEXEC)) < 0) { i = 0; while (i <= 2 && !isatty(i)) i++; if (i > 2 || (ttyfd = fcntl(i, F_DUPFD_CLOEXEC, 10)) < 0) { jobctl_notty(); return; } } if (ttyfd < 10) { /* * Keep our TTY file descriptor out of the way of * the user's redirections. */ if ((i = fcntl(ttyfd, F_DUPFD_CLOEXEC, 10)) < 0) { jobctl_notty(); return; } close(ttyfd); ttyfd = i; } do { /* while we are in the background */ initialpgrp = tcgetpgrp(ttyfd); if (initialpgrp < 0) { jobctl_notty(); return; } if (initialpgrp != getpgrp()) { if (!iflag) { initialpgrp = -1; jobctl_notty(); return; } kill(0, SIGTTIN); continue; } } while (0); setsignal(SIGTSTP); setsignal(SIGTTOU); setsignal(SIGTTIN); setpgid(0, rootpid); tcsetpgrp(ttyfd, rootpid); } else { /* turning job control off */ setpgid(0, initialpgrp); if (ttyfd >= 0) { tcsetpgrp(ttyfd, initialpgrp); close(ttyfd); ttyfd = -1; } setsignal(SIGTSTP); setsignal(SIGTTOU); setsignal(SIGTTIN); } jobctl = on; } #endif #if JOBS int fgcmd(int argc __unused, char **argv __unused) { struct job *jp; pid_t pgrp; int status; nextopt(""); jp = getjob(*argptr); if (jp->jobctl == 0) error("job not created under job control"); printjobcmd(jp); flushout(&output); pgrp = jp->ps[0].pid; if (ttyfd >= 0) tcsetpgrp(ttyfd, pgrp); restartjob(jp); jp->foreground = 1; INTOFF; status = waitforjob(jp, (int *)NULL); INTON; return status; } int bgcmd(int argc, char **argv) { struct job *jp; nextopt(""); do { jp = getjob(*argptr); if (jp->jobctl == 0) error("job not created under job control"); if (jp->state == JOBDONE) continue; restartjob(jp); jp->foreground = 0; out1fmt("[%td] ", jp - jobtab + 1); printjobcmd(jp); } while (*argptr != NULL && *++argptr != NULL); return 0; } static void restartjob(struct job *jp) { struct procstat *ps; int i; if (jp->state == JOBDONE) return; setcurjob(jp); INTOFF; kill(-jp->ps[0].pid, SIGCONT); for (ps = jp->ps, i = jp->nprocs ; --i >= 0 ; ps++) { if (WIFSTOPPED(ps->status)) { ps->status = -1; jp->state = 0; } } INTON; } #endif int jobscmd(int argc __unused, char *argv[] __unused) { char *id; int ch, mode; mode = SHOWJOBS_DEFAULT; while ((ch = nextopt("lps")) != '\0') { switch (ch) { case 'l': mode = SHOWJOBS_VERBOSE; break; case 'p': mode = SHOWJOBS_PGIDS; break; case 's': mode = SHOWJOBS_PIDS; break; } } if (*argptr == NULL) showjobs(0, mode); else while ((id = *argptr++) != NULL) showjob(getjob(id), mode); return (0); } static void printjobcmd(struct job *jp) { struct procstat *ps; int i; for (ps = jp->ps, i = jp->nprocs ; --i >= 0 ; ps++) { out1str(ps->cmd); if (i > 0) out1str(" | "); } out1c('\n'); } static void showjob(struct job *jp, int mode) { char s[64]; char statestr[64]; const char *sigstr; struct procstat *ps; struct job *j; int col, curr, i, jobno, prev, procno; char c; procno = (mode == SHOWJOBS_PGIDS) ? 1 : jp->nprocs; jobno = jp - jobtab + 1; curr = prev = 0; #if JOBS if ((j = getcurjob(NULL)) != NULL) { curr = j - jobtab + 1; if ((j = getcurjob(j)) != NULL) prev = j - jobtab + 1; } #endif ps = jp->ps + jp->nprocs - 1; if (jp->state == 0) { strcpy(statestr, "Running"); #if JOBS } else if (jp->state == JOBSTOPPED) { while (!WIFSTOPPED(ps->status) && ps > jp->ps) ps--; if (WIFSTOPPED(ps->status)) i = WSTOPSIG(ps->status); else i = -1; sigstr = strsignal(i); if (sigstr != NULL) strcpy(statestr, sigstr); else strcpy(statestr, "Suspended"); #endif } else if (WIFEXITED(ps->status)) { if (WEXITSTATUS(ps->status) == 0) strcpy(statestr, "Done"); else fmtstr(statestr, 64, "Done(%d)", WEXITSTATUS(ps->status)); } else { i = WTERMSIG(ps->status); sigstr = strsignal(i); if (sigstr != NULL) strcpy(statestr, sigstr); else strcpy(statestr, "Unknown signal"); if (WCOREDUMP(ps->status)) strcat(statestr, " (core dumped)"); } for (ps = jp->ps ; procno > 0 ; ps++, procno--) { /* for each process */ if (mode == SHOWJOBS_PIDS || mode == SHOWJOBS_PGIDS) { out1fmt("%d\n", (int)ps->pid); continue; } if (mode != SHOWJOBS_VERBOSE && ps != jp->ps) continue; if (jobno == curr && ps == jp->ps) c = '+'; else if (jobno == prev && ps == jp->ps) c = '-'; else c = ' '; if (ps == jp->ps) fmtstr(s, 64, "[%d] %c ", jobno, c); else fmtstr(s, 64, " %c ", c); out1str(s); col = strlen(s); if (mode == SHOWJOBS_VERBOSE) { fmtstr(s, 64, "%d ", (int)ps->pid); out1str(s); col += strlen(s); } if (ps == jp->ps) { out1str(statestr); col += strlen(statestr); } do { out1c(' '); col++; } while (col < 30); if (mode == SHOWJOBS_VERBOSE) { out1str(ps->cmd); out1c('\n'); } else printjobcmd(jp); } } /* * Print a list of jobs. If "change" is nonzero, only print jobs whose * statuses have changed since the last call to showjobs. * * If the shell is interrupted in the process of creating a job, the * result may be a job structure containing zero processes. Such structures * will be freed here. */ void showjobs(int change, int mode) { int jobno; struct job *jp; TRACE(("showjobs(%d) called\n", change)); checkzombies(); for (jobno = 1, jp = jobtab ; jobno <= njobs ; jobno++, jp++) { if (! jp->used) continue; if (jp->nprocs == 0) { freejob(jp); continue; } if (change && ! jp->changed) continue; showjob(jp, mode); if (mode == SHOWJOBS_DEFAULT || mode == SHOWJOBS_VERBOSE) { jp->changed = 0; /* Hack: discard jobs for which $! has not been * referenced in interactive mode when they terminate. */ if (jp->state == JOBDONE && !jp->remembered && (iflag || jp != bgjob)) { freejob(jp); } } } } /* * Mark a job structure as unused. */ static void freejob(struct job *jp) { struct procstat *ps; int i; INTOFF; if (bgjob == jp) bgjob = NULL; for (i = jp->nprocs, ps = jp->ps ; --i >= 0 ; ps++) { if (ps->cmd != nullstr) ckfree(ps->cmd); } if (jp->ps != &jp->ps0) ckfree(jp->ps); jp->used = 0; #if JOBS deljob(jp); #endif INTON; } int waitcmd(int argc __unused, char **argv __unused) { struct job *job; int retval; nextopt(""); if (*argptr == NULL) return (waitcmdloop(NULL)); do { job = getjob_nonotfound(*argptr); if (job == NULL) retval = 127; else retval = waitcmdloop(job); argptr++; } while (*argptr != NULL); return (retval); } static int waitcmdloop(struct job *job) { int status, retval, sig; struct job *jp; /* * Loop until a process is terminated or stopped, or a SIGINT is * received. */ do { if (job != NULL) { if (job->state == JOBDONE) { status = job->ps[job->nprocs - 1].status; if (WIFEXITED(status)) retval = WEXITSTATUS(status); else retval = WTERMSIG(status) + 128; if (! iflag || ! job->changed) freejob(job); else { job->remembered = 0; if (job == bgjob) bgjob = NULL; } return retval; } } else { for (jp = jobtab ; jp < jobtab + njobs; jp++) if (jp->used && jp->state == JOBDONE) { if (! iflag || ! jp->changed) freejob(jp); else { jp->remembered = 0; if (jp == bgjob) bgjob = NULL; } } for (jp = jobtab ; ; jp++) { if (jp >= jobtab + njobs) { /* no running procs */ return 0; } if (jp->used && jp->state == 0) break; } } } while (dowait(DOWAIT_BLOCK | DOWAIT_SIG, (struct job *)NULL) != -1); sig = pendingsig_waitcmd; pendingsig_waitcmd = 0; return sig + 128; } int jobidcmd(int argc __unused, char **argv __unused) { struct job *jp; int i; nextopt(""); jp = getjob(*argptr); for (i = 0 ; i < jp->nprocs ; ) { out1fmt("%d", (int)jp->ps[i].pid); out1c(++i < jp->nprocs? ' ' : '\n'); } return 0; } /* * Convert a job name to a job structure. */ static struct job * getjob_nonotfound(const char *name) { int jobno; struct job *found, *jp; size_t namelen; pid_t pid; int i; if (name == NULL) { #if JOBS name = "%+"; #else error("No current job"); #endif } if (name[0] == '%') { if (is_digit(name[1])) { jobno = number(name + 1); if (jobno > 0 && jobno <= njobs && jobtab[jobno - 1].used != 0) return &jobtab[jobno - 1]; #if JOBS } else if ((name[1] == '%' || name[1] == '+') && name[2] == '\0') { if ((jp = getcurjob(NULL)) == NULL) error("No current job"); return (jp); } else if (name[1] == '-' && name[2] == '\0') { if ((jp = getcurjob(NULL)) == NULL || (jp = getcurjob(jp)) == NULL) error("No previous job"); return (jp); #endif } else if (name[1] == '?') { found = NULL; for (jp = jobtab, i = njobs ; --i >= 0 ; jp++) { if (jp->used && jp->nprocs > 0 && strstr(jp->ps[0].cmd, name + 2) != NULL) { if (found) error("%s: ambiguous", name); found = jp; } } if (found != NULL) return (found); } else { namelen = strlen(name); found = NULL; for (jp = jobtab, i = njobs ; --i >= 0 ; jp++) { if (jp->used && jp->nprocs > 0 && strncmp(jp->ps[0].cmd, name + 1, namelen - 1) == 0) { if (found) error("%s: ambiguous", name); found = jp; } } if (found) return found; } } else if (is_number(name)) { pid = (pid_t)number(name); for (jp = jobtab, i = njobs ; --i >= 0 ; jp++) { if (jp->used && jp->nprocs > 0 && jp->ps[jp->nprocs - 1].pid == pid) return jp; } } return NULL; } static struct job * getjob(const char *name) { struct job *jp; jp = getjob_nonotfound(name); if (jp == NULL) error("No such job: %s", name); return (jp); } int killjob(const char *name, int sig) { struct job *jp; int i, ret; jp = getjob(name); if (jp->state == JOBDONE) return 0; if (jp->jobctl) return kill(-jp->ps[0].pid, sig); ret = -1; errno = ESRCH; for (i = 0; i < jp->nprocs; i++) if (jp->ps[i].status == -1 || WIFSTOPPED(jp->ps[i].status)) { if (kill(jp->ps[i].pid, sig) == 0) ret = 0; } else ret = 0; return ret; } /* * Return a new job structure, */ struct job * makejob(union node *node __unused, int nprocs) { int i; struct job *jp; for (i = njobs, jp = jobtab ; ; jp++) { if (--i < 0) { INTOFF; if (njobs == 0) { jobtab = ckmalloc(4 * sizeof jobtab[0]); #if JOBS jobmru = NULL; #endif } else { jp = ckmalloc((njobs + 4) * sizeof jobtab[0]); memcpy(jp, jobtab, njobs * sizeof jp[0]); #if JOBS /* Relocate `next' pointers and list head */ if (jobmru != NULL) jobmru = &jp[jobmru - jobtab]; for (i = 0; i < njobs; i++) if (jp[i].next != NULL) jp[i].next = &jp[jp[i].next - jobtab]; #endif if (bgjob != NULL) bgjob = &jp[bgjob - jobtab]; /* Relocate `ps' pointers */ for (i = 0; i < njobs; i++) if (jp[i].ps == &jobtab[i].ps0) jp[i].ps = &jp[i].ps0; ckfree(jobtab); jobtab = jp; } jp = jobtab + njobs; for (i = 4 ; --i >= 0 ; jobtab[njobs++].used = 0) ; INTON; break; } if (jp->used == 0) break; } INTOFF; jp->state = 0; jp->used = 1; jp->changed = 0; jp->nprocs = 0; jp->foreground = 0; jp->remembered = 0; #if JOBS jp->jobctl = jobctl; jp->next = NULL; #endif if (nprocs > 1) { jp->ps = ckmalloc(nprocs * sizeof (struct procstat)); } else { jp->ps = &jp->ps0; } INTON; TRACE(("makejob(%p, %d) returns %%%td\n", (void *)node, nprocs, jp - jobtab + 1)); return jp; } #if JOBS static void setcurjob(struct job *cj) { struct job *jp, *prev; for (prev = NULL, jp = jobmru; jp != NULL; prev = jp, jp = jp->next) { if (jp == cj) { if (prev != NULL) prev->next = jp->next; else jobmru = jp->next; jp->next = jobmru; jobmru = cj; return; } } cj->next = jobmru; jobmru = cj; } static void deljob(struct job *j) { struct job *jp, *prev; for (prev = NULL, jp = jobmru; jp != NULL; prev = jp, jp = jp->next) { if (jp == j) { if (prev != NULL) prev->next = jp->next; else jobmru = jp->next; return; } } } /* * Return the most recently used job that isn't `nj', and preferably one * that is stopped. */ static struct job * getcurjob(struct job *nj) { struct job *jp; /* Try to find a stopped one.. */ for (jp = jobmru; jp != NULL; jp = jp->next) if (jp->used && jp != nj && jp->state == JOBSTOPPED) return (jp); /* Otherwise the most recently used job that isn't `nj' */ for (jp = jobmru; jp != NULL; jp = jp->next) if (jp->used && jp != nj) return (jp); return (NULL); } #endif /* * Fork of a subshell. If we are doing job control, give the subshell its * own process group. Jp is a job structure that the job is to be added to. * N is the command that will be evaluated by the child. Both jp and n may * be NULL. The mode parameter can be one of the following: * FORK_FG - Fork off a foreground process. * FORK_BG - Fork off a background process. * FORK_NOJOB - Like FORK_FG, but don't give the process its own * process group even if job control is on. * * When job control is turned off, background processes have their standard * input redirected to /dev/null (except for the second and later processes * in a pipeline). */ pid_t forkshell(struct job *jp, union node *n, int mode) { pid_t pid; pid_t pgrp; TRACE(("forkshell(%%%td, %p, %d) called\n", jp - jobtab, (void *)n, mode)); INTOFF; if (mode == FORK_BG && (jp == NULL || jp->nprocs == 0)) checkzombies(); flushall(); pid = fork(); if (pid == -1) { TRACE(("Fork failed, errno=%d\n", errno)); INTON; error("Cannot fork: %s", strerror(errno)); } if (pid == 0) { struct job *p; int wasroot; int i; TRACE(("Child shell %d\n", (int)getpid())); wasroot = rootshell; rootshell = 0; handler = &main_handler; closescript(); INTON; forcelocal = 0; clear_traps(); #if JOBS jobctl = 0; /* do job control only in root shell */ if (wasroot && mode != FORK_NOJOB && mflag) { if (jp == NULL || jp->nprocs == 0) pgrp = getpid(); else pgrp = jp->ps[0].pid; if (setpgid(0, pgrp) == 0 && mode == FORK_FG && ttyfd >= 0) { /*** this causes superfluous TIOCSPGRPS ***/ if (tcsetpgrp(ttyfd, pgrp) < 0) error("tcsetpgrp failed, errno=%d", errno); } setsignal(SIGTSTP); setsignal(SIGTTOU); } else if (mode == FORK_BG) { ignoresig(SIGINT); ignoresig(SIGQUIT); if ((jp == NULL || jp->nprocs == 0) && ! fd0_redirected_p ()) { close(0); if (open(_PATH_DEVNULL, O_RDONLY) != 0) error("cannot open %s: %s", _PATH_DEVNULL, strerror(errno)); } } #else if (mode == FORK_BG) { ignoresig(SIGINT); ignoresig(SIGQUIT); if ((jp == NULL || jp->nprocs == 0) && ! fd0_redirected_p ()) { close(0); if (open(_PATH_DEVNULL, O_RDONLY) != 0) error("cannot open %s: %s", _PATH_DEVNULL, strerror(errno)); } } #endif INTOFF; for (i = njobs, p = jobtab ; --i >= 0 ; p++) if (p->used) freejob(p); INTON; if (wasroot && iflag) { setsignal(SIGINT); setsignal(SIGQUIT); setsignal(SIGTERM); } return pid; } if (rootshell && mode != FORK_NOJOB && mflag) { if (jp == NULL || jp->nprocs == 0) pgrp = pid; else pgrp = jp->ps[0].pid; setpgid(pid, pgrp); } if (mode == FORK_BG) { if (bgjob != NULL && bgjob->state == JOBDONE && !bgjob->remembered && !iflag) freejob(bgjob); backgndpid = pid; /* set $! */ bgjob = jp; } if (jp) { struct procstat *ps = &jp->ps[jp->nprocs++]; ps->pid = pid; ps->status = -1; ps->cmd = nullstr; if (iflag && rootshell && n) ps->cmd = commandtext(n); jp->foreground = mode == FORK_FG; #if JOBS setcurjob(jp); #endif } INTON; TRACE(("In parent shell: child = %d\n", (int)pid)); return pid; } pid_t vforkexecshell(struct job *jp, char **argv, char **envp, const char *path, int idx, int pip[2]) { pid_t pid; struct jmploc jmploc; struct jmploc *savehandler; TRACE(("vforkexecshell(%%%td, %s, %p) called\n", jp - jobtab, argv[0], (void *)pip)); INTOFF; flushall(); savehandler = handler; pid = vfork(); if (pid == -1) { TRACE(("Vfork failed, errno=%d\n", errno)); INTON; error("Cannot fork: %s", strerror(errno)); } if (pid == 0) { TRACE(("Child shell %d\n", (int)getpid())); if (setjmp(jmploc.loc)) _exit(exception == EXEXEC ? exerrno : 2); if (pip != NULL) { close(pip[0]); if (pip[1] != 1) { dup2(pip[1], 1); close(pip[1]); } } handler = &jmploc; shellexec(argv, envp, path, idx); } handler = savehandler; if (jp) { struct procstat *ps = &jp->ps[jp->nprocs++]; ps->pid = pid; ps->status = -1; ps->cmd = nullstr; jp->foreground = 1; #if JOBS setcurjob(jp); #endif } INTON; TRACE(("In parent shell: child = %d\n", (int)pid)); return pid; } /* * Wait for job to finish. * * Under job control we have the problem that while a child process is * running interrupts generated by the user are sent to the child but not * to the shell. This means that an infinite loop started by an inter- * active user may be hard to kill. With job control turned off, an * interactive user may place an interactive program inside a loop. If * the interactive program catches interrupts, the user doesn't want * these interrupts to also abort the loop. The approach we take here * is to have the shell ignore interrupt signals while waiting for a * foreground process to terminate, and then send itself an interrupt * signal if the child process was terminated by an interrupt signal. * Unfortunately, some programs want to do a bit of cleanup and then * exit on interrupt; unless these processes terminate themselves by * sending a signal to themselves (instead of calling exit) they will * confuse this approach. */ int waitforjob(struct job *jp, int *origstatus) { #if JOBS int propagate_int = jp->jobctl && jp->foreground; #endif int status; int st; INTOFF; TRACE(("waitforjob(%%%td) called\n", jp - jobtab + 1)); while (jp->state == 0) if (dowait(DOWAIT_BLOCK | (Tflag ? DOWAIT_SIG | - DOWAIT_SIG_ANY : 0), jp) == -1) + DOWAIT_SIG_TRAP : 0), jp) == -1) dotrap(); #if JOBS if (jp->jobctl) { if (ttyfd >= 0 && tcsetpgrp(ttyfd, rootpid) < 0) error("tcsetpgrp failed, errno=%d\n", errno); } if (jp->state == JOBSTOPPED) setcurjob(jp); #endif status = jp->ps[jp->nprocs - 1].status; if (origstatus != NULL) *origstatus = status; /* convert to 8 bits */ if (WIFEXITED(status)) st = WEXITSTATUS(status); #if JOBS else if (WIFSTOPPED(status)) st = WSTOPSIG(status) + 128; #endif else st = WTERMSIG(status) + 128; if (! JOBS || jp->state == JOBDONE) freejob(jp); if (int_pending()) { if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGINT) CLEAR_PENDING_INT; } #if JOBS else if (rootshell && iflag && propagate_int && WIFSIGNALED(status) && WTERMSIG(status) == SIGINT) kill(getpid(), SIGINT); #endif INTON; return st; } static void dummy_handler(int sig __unused) { } /* * Wait for a process to terminate. */ static pid_t dowait(int mode, struct job *job) { struct sigaction sa, osa; sigset_t mask, omask; pid_t pid; int status; struct procstat *sp; struct job *jp; struct job *thisjob; const char *sigstr; int done; int stopped; int sig; int coredump; int wflags; int restore_sigchld; TRACE(("dowait(%d, %p) called\n", mode, job)); restore_sigchld = 0; if ((mode & DOWAIT_SIG) != 0) { sigfillset(&mask); sigprocmask(SIG_BLOCK, &mask, &omask); INTOFF; if (!issigchldtrapped()) { restore_sigchld = 1; sa.sa_handler = dummy_handler; sa.sa_flags = 0; sigemptyset(&sa.sa_mask); sigaction(SIGCHLD, &sa, &osa); } } do { #if JOBS if (iflag) wflags = WUNTRACED | WCONTINUED; else #endif wflags = 0; if ((mode & (DOWAIT_BLOCK | DOWAIT_SIG)) != DOWAIT_BLOCK) wflags |= WNOHANG; pid = wait3(&status, wflags, (struct rusage *)NULL); TRACE(("wait returns %d, status=%d\n", (int)pid, status)); if (pid == 0 && (mode & DOWAIT_SIG) != 0) { pid = -1; - if (((mode & DOWAIT_SIG_ANY) != 0 ? + if (((mode & DOWAIT_SIG_TRAP) != 0 ? pendingsig : pendingsig_waitcmd) != 0) { errno = EINTR; break; } sigsuspend(&omask); if (int_pending()) break; } } while (pid == -1 && errno == EINTR); if (pid == -1 && errno == ECHILD && job != NULL) job->state = JOBDONE; if ((mode & DOWAIT_SIG) != 0) { if (restore_sigchld) sigaction(SIGCHLD, &osa, NULL); sigprocmask(SIG_SETMASK, &omask, NULL); INTON; } if (pid <= 0) return pid; INTOFF; thisjob = NULL; for (jp = jobtab ; jp < jobtab + njobs ; jp++) { if (jp->used && jp->nprocs > 0) { done = 1; stopped = 1; for (sp = jp->ps ; sp < jp->ps + jp->nprocs ; sp++) { if (sp->pid == -1) continue; if (sp->pid == pid && (sp->status == -1 || WIFSTOPPED(sp->status))) { TRACE(("Changing status of proc %d from 0x%x to 0x%x\n", (int)pid, sp->status, status)); if (WIFCONTINUED(status)) { sp->status = -1; jp->state = 0; } else sp->status = status; thisjob = jp; } if (sp->status == -1) stopped = 0; else if (WIFSTOPPED(sp->status)) done = 0; } if (stopped) { /* stopped or done */ int state = done? JOBDONE : JOBSTOPPED; if (jp->state != state) { TRACE(("Job %td: changing state from %d to %d\n", jp - jobtab + 1, jp->state, state)); jp->state = state; if (jp != job) { if (done && !jp->remembered && !iflag && jp != bgjob) freejob(jp); #if JOBS else if (done) deljob(jp); #endif } } } } } INTON; if (!thisjob || thisjob->state == 0) ; else if ((!rootshell || !iflag || thisjob == job) && thisjob->foreground && thisjob->state != JOBSTOPPED) { sig = 0; coredump = 0; for (sp = thisjob->ps; sp < thisjob->ps + thisjob->nprocs; sp++) if (WIFSIGNALED(sp->status)) { sig = WTERMSIG(sp->status); coredump = WCOREDUMP(sp->status); } if (sig > 0 && sig != SIGINT && sig != SIGPIPE) { sigstr = strsignal(sig); if (sigstr != NULL) out2str(sigstr); else out2str("Unknown signal"); if (coredump) out2str(" (core dumped)"); out2c('\n'); flushout(out2); } } else { TRACE(("Not printing status, rootshell=%d, job=%p\n", rootshell, job)); thisjob->changed = 1; } return pid; } /* * return 1 if there are stopped jobs, otherwise 0 */ int job_warning = 0; int stoppedjobs(void) { int jobno; struct job *jp; if (job_warning) return (0); for (jobno = 1, jp = jobtab; jobno <= njobs; jobno++, jp++) { if (jp->used == 0) continue; if (jp->state == JOBSTOPPED) { out2fmt_flush("You have stopped jobs.\n"); job_warning = 2; return (1); } } return (0); } static void checkzombies(void) { while (njobs > 0 && dowait(0, NULL) > 0) ; } int backgndpidset(void) { return backgndpid != -1; } pid_t backgndpidval(void) { if (bgjob != NULL && !forcelocal) bgjob->remembered = 1; return backgndpid; } /* * Return a string identifying a command (to be printed by the * jobs command. */ static char *cmdnextc; static int cmdnleft; #define MAXCMDTEXT 200 char * commandtext(union node *n) { char *name; cmdnextc = name = ckmalloc(MAXCMDTEXT); cmdnleft = MAXCMDTEXT - 4; cmdtxt(n); *cmdnextc = '\0'; return name; } static void cmdtxtdogroup(union node *n) { cmdputs("; do "); cmdtxt(n); cmdputs("; done"); } static void cmdtxtredir(union node *n, const char *op, int deffd) { char s[2]; if (n->nfile.fd != deffd) { s[0] = n->nfile.fd + '0'; s[1] = '\0'; cmdputs(s); } cmdputs(op); if (n->type == NTOFD || n->type == NFROMFD) { if (n->ndup.dupfd >= 0) s[0] = n->ndup.dupfd + '0'; else s[0] = '-'; s[1] = '\0'; cmdputs(s); } else { cmdtxt(n->nfile.fname); } } static void cmdtxt(union node *n) { union node *np; struct nodelist *lp; if (n == NULL) return; switch (n->type) { case NSEMI: cmdtxt(n->nbinary.ch1); cmdputs("; "); cmdtxt(n->nbinary.ch2); break; case NAND: cmdtxt(n->nbinary.ch1); cmdputs(" && "); cmdtxt(n->nbinary.ch2); break; case NOR: cmdtxt(n->nbinary.ch1); cmdputs(" || "); cmdtxt(n->nbinary.ch2); break; case NPIPE: for (lp = n->npipe.cmdlist ; lp ; lp = lp->next) { cmdtxt(lp->n); if (lp->next) cmdputs(" | "); } break; case NSUBSHELL: cmdputs("("); cmdtxt(n->nredir.n); cmdputs(")"); break; case NREDIR: case NBACKGND: cmdtxt(n->nredir.n); break; case NIF: cmdputs("if "); cmdtxt(n->nif.test); cmdputs("; then "); cmdtxt(n->nif.ifpart); cmdputs("..."); break; case NWHILE: cmdputs("while "); cmdtxt(n->nbinary.ch1); cmdtxtdogroup(n->nbinary.ch2); break; case NUNTIL: cmdputs("until "); cmdtxt(n->nbinary.ch1); cmdtxtdogroup(n->nbinary.ch2); break; case NFOR: cmdputs("for "); cmdputs(n->nfor.var); cmdputs(" in ..."); break; case NCASE: cmdputs("case "); cmdputs(n->ncase.expr->narg.text); cmdputs(" in ..."); break; case NDEFUN: cmdputs(n->narg.text); cmdputs("() ..."); break; case NNOT: cmdputs("! "); cmdtxt(n->nnot.com); break; case NCMD: for (np = n->ncmd.args ; np ; np = np->narg.next) { cmdtxt(np); if (np->narg.next) cmdputs(" "); } for (np = n->ncmd.redirect ; np ; np = np->nfile.next) { cmdputs(" "); cmdtxt(np); } break; case NARG: cmdputs(n->narg.text); break; case NTO: cmdtxtredir(n, ">", 1); break; case NAPPEND: cmdtxtredir(n, ">>", 1); break; case NTOFD: cmdtxtredir(n, ">&", 1); break; case NCLOBBER: cmdtxtredir(n, ">|", 1); break; case NFROM: cmdtxtredir(n, "<", 0); break; case NFROMTO: cmdtxtredir(n, "<>", 0); break; case NFROMFD: cmdtxtredir(n, "<&", 0); break; case NHERE: case NXHERE: cmdputs("<<..."); break; default: cmdputs("???"); break; } } static void cmdputs(const char *s) { const char *p; char *q; char c; int subtype = 0; if (cmdnleft <= 0) return; p = s; q = cmdnextc; while ((c = *p++) != '\0') { if (c == CTLESC) *q++ = *p++; else if (c == CTLVAR) { *q++ = '$'; if (--cmdnleft > 0) *q++ = '{'; subtype = *p++; if ((subtype & VSTYPE) == VSLENGTH && --cmdnleft > 0) *q++ = '#'; } else if (c == '=' && subtype != 0) { *q = "}-+?=##%%\0X"[(subtype & VSTYPE) - VSNORMAL]; if (*q) q++; else cmdnleft++; if (((subtype & VSTYPE) == VSTRIMLEFTMAX || (subtype & VSTYPE) == VSTRIMRIGHTMAX) && --cmdnleft > 0) *q = q[-1], q++; subtype = 0; } else if (c == CTLENDVAR) { *q++ = '}'; } else if (c == CTLBACKQ || c == CTLBACKQ+CTLQUOTE) { cmdnleft -= 5; if (cmdnleft > 0) { *q++ = '$'; *q++ = '('; *q++ = '.'; *q++ = '.'; *q++ = '.'; *q++ = ')'; } } else if (c == CTLARI) { cmdnleft -= 2; if (cmdnleft > 0) { *q++ = '$'; *q++ = '('; *q++ = '('; } p++; } else if (c == CTLENDARI) { if (--cmdnleft > 0) { *q++ = ')'; *q++ = ')'; } } else if (c == CTLQUOTEMARK || c == CTLQUOTEEND) cmdnleft++; /* ignore */ else *q++ = c; if (--cmdnleft <= 0) { *q++ = '.'; *q++ = '.'; *q++ = '.'; break; } } cmdnextc = q; } diff --git a/bin/sh/trap.c b/bin/sh/trap.c index c23e6bc3b1ae..4a185b4874f5 100644 --- a/bin/sh/trap.c +++ b/bin/sh/trap.c @@ -1,550 +1,551 @@ /*- * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Kenneth Almquist. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef lint #if 0 static char sccsid[] = "@(#)trap.c 8.5 (Berkeley) 6/5/95"; #endif #endif /* not lint */ #include __FBSDID("$FreeBSD$"); #include #include #include #include "shell.h" #include "main.h" #include "nodes.h" /* for other headers */ #include "eval.h" #include "jobs.h" #include "show.h" #include "options.h" #include "syntax.h" #include "output.h" #include "memalloc.h" #include "error.h" #include "trap.h" #include "mystring.h" #include "builtins.h" #include "myhistedit.h" /* * Sigmode records the current value of the signal handlers for the various * modes. A value of zero means that the current handler is not known. * S_HARD_IGN indicates that the signal was ignored on entry to the shell, */ #define S_DFL 1 /* default signal handling (SIG_DFL) */ #define S_CATCH 2 /* signal is caught */ #define S_IGN 3 /* signal is ignored (SIG_IGN) */ #define S_HARD_IGN 4 /* signal is ignored permanently */ #define S_RESET 5 /* temporary - to reset a hard ignored sig */ static char sigmode[NSIG]; /* current value of signal */ volatile sig_atomic_t pendingsig; /* indicates some signal received */ -volatile sig_atomic_t pendingsig_waitcmd; /* indicates SIGINT/SIGQUIT received */ +volatile sig_atomic_t pendingsig_waitcmd; /* indicates wait builtin should be interrupted */ static int in_dotrap; /* do we execute in a trap handler? */ static char *volatile trap[NSIG]; /* trap handler commands */ static volatile sig_atomic_t gotsig[NSIG]; /* indicates specified signal received */ static int ignore_sigchld; /* Used while handling SIGCHLD traps. */ static int last_trapsig; static int exiting; /* exitshell() has been called */ static int exiting_exitstatus; /* value passed to exitshell() */ static int getsigaction(int, sig_t *); /* * Map a string to a signal number. * * Note: the signal number may exceed NSIG. */ static int sigstring_to_signum(char *sig) { if (is_number(sig)) { int signo; signo = atoi(sig); return ((signo >= 0 && signo < NSIG) ? signo : (-1)); } else if (strcasecmp(sig, "EXIT") == 0) { return (0); } else { int n; if (strncasecmp(sig, "SIG", 3) == 0) sig += 3; for (n = 1; n < sys_nsig; n++) if (sys_signame[n] && strcasecmp(sys_signame[n], sig) == 0) return (n); } return (-1); } /* * Print a list of valid signal names. */ static void printsignals(void) { int n, outlen; outlen = 0; for (n = 1; n < sys_nsig; n++) { if (sys_signame[n]) { out1fmt("%s", sys_signame[n]); outlen += strlen(sys_signame[n]); } else { out1fmt("%d", n); outlen += 3; /* good enough */ } ++outlen; if (outlen > 71 || n == sys_nsig - 1) { out1str("\n"); outlen = 0; } else { out1c(' '); } } } /* * The trap builtin. */ int trapcmd(int argc __unused, char **argv) { char *action; int signo; int errors = 0; int i; while ((i = nextopt("l")) != '\0') { switch (i) { case 'l': printsignals(); return (0); } } argv = argptr; if (*argv == NULL) { for (signo = 0 ; signo < sys_nsig ; signo++) { if (signo < NSIG && trap[signo] != NULL) { out1str("trap -- "); out1qstr(trap[signo]); if (signo == 0) { out1str(" EXIT\n"); } else if (sys_signame[signo]) { out1fmt(" %s\n", sys_signame[signo]); } else { out1fmt(" %d\n", signo); } } } return 0; } action = NULL; if (*argv && sigstring_to_signum(*argv) == -1) { if (strcmp(*argv, "-") == 0) argv++; else { action = *argv; argv++; } } for (; *argv; argv++) { if ((signo = sigstring_to_signum(*argv)) == -1) { warning("bad signal %s", *argv); errors = 1; continue; } INTOFF; if (action) action = savestr(action); if (trap[signo]) ckfree(trap[signo]); trap[signo] = action; if (signo != 0) setsignal(signo); INTON; } return errors; } /* * Clear traps on a fork. */ void clear_traps(void) { char *volatile *tp; for (tp = trap ; tp <= &trap[NSIG - 1] ; tp++) { if (*tp && **tp) { /* trap not NULL or SIG_IGN */ INTOFF; ckfree(*tp); *tp = NULL; if (tp != &trap[0]) setsignal(tp - trap); INTON; } } } /* * Check if we have any traps enabled. */ int have_traps(void) { char *volatile *tp; for (tp = trap ; tp <= &trap[NSIG - 1] ; tp++) { if (*tp && **tp) /* trap not NULL or SIG_IGN */ return 1; } return 0; } /* * Set the signal handler for the specified signal. The routine figures * out what it should be set to. */ void setsignal(int signo) { int action; sig_t sigact = SIG_DFL; struct sigaction sa; char *t; if ((t = trap[signo]) == NULL) action = S_DFL; else if (*t != '\0') action = S_CATCH; else action = S_IGN; if (action == S_DFL) { switch (signo) { case SIGINT: action = S_CATCH; break; case SIGQUIT: #ifdef DEBUG { extern int debug; if (debug) break; } #endif action = S_CATCH; break; case SIGTERM: if (rootshell && iflag) action = S_IGN; break; #if JOBS case SIGTSTP: case SIGTTOU: if (rootshell && mflag) action = S_IGN; break; #endif } } t = &sigmode[signo]; if (*t == 0) { /* * current setting unknown */ if (!getsigaction(signo, &sigact)) { /* * Pretend it worked; maybe we should give a warning * here, but other shells don't. We don't alter * sigmode, so that we retry every time. */ return; } if (sigact == SIG_IGN) { if (mflag && (signo == SIGTSTP || signo == SIGTTIN || signo == SIGTTOU)) { *t = S_IGN; /* don't hard ignore these */ } else *t = S_HARD_IGN; } else { *t = S_RESET; /* force to be set */ } } if (*t == S_HARD_IGN || *t == action) return; switch (action) { case S_DFL: sigact = SIG_DFL; break; case S_CATCH: sigact = onsig; break; case S_IGN: sigact = SIG_IGN; break; } *t = action; sa.sa_handler = sigact; sa.sa_flags = 0; sigemptyset(&sa.sa_mask); sigaction(signo, &sa, NULL); } /* * Return the current setting for sig w/o changing it. */ static int getsigaction(int signo, sig_t *sigact) { struct sigaction sa; if (sigaction(signo, (struct sigaction *)0, &sa) == -1) return 0; *sigact = (sig_t) sa.sa_handler; return 1; } /* * Ignore a signal. */ void ignoresig(int signo) { if (sigmode[signo] == 0) setsignal(signo); if (sigmode[signo] != S_IGN && sigmode[signo] != S_HARD_IGN) { signal(signo, SIG_IGN); sigmode[signo] = S_IGN; } } int issigchldtrapped(void) { return (trap[SIGCHLD] != NULL && *trap[SIGCHLD] != '\0'); } /* * Signal handler. */ void onsig(int signo) { if (signo == SIGINT && trap[SIGINT] == NULL) { /* * The !in_dotrap here is safe. The only way we can arrive * here with in_dotrap set is that a trap handler set SIGINT to * SIG_DFL and killed itself. */ if (suppressint && !in_dotrap) SET_PENDING_INT; else onint(); return; } /* If we are currently in a wait builtin, prepare to break it */ if (signo == SIGINT || signo == SIGQUIT) pendingsig_waitcmd = signo; if (trap[signo] != NULL && trap[signo][0] != '\0' && (signo != SIGCHLD || !ignore_sigchld)) { gotsig[signo] = 1; pendingsig = signo; + pendingsig_waitcmd = signo; } } /* * Called to execute a trap. Perhaps we should avoid entering new trap * handlers while we are executing a trap handler. */ void dotrap(void) { int i; int savestatus, prev_evalskip, prev_skipcount; in_dotrap++; for (;;) { pendingsig = 0; pendingsig_waitcmd = 0; for (i = 1; i < NSIG; i++) { if (gotsig[i]) { gotsig[i] = 0; if (trap[i]) { /* * Ignore SIGCHLD to avoid infinite * recursion if the trap action does * a fork. */ if (i == SIGCHLD) ignore_sigchld++; /* * Backup current evalskip * state and reset it before * executing a trap, so that the * trap is not disturbed by an * ongoing break/continue/return * statement. */ prev_evalskip = evalskip; prev_skipcount = skipcount; evalskip = 0; last_trapsig = i; savestatus = exitstatus; evalstring(trap[i], 0); /* * If such a command was not * already in progress, allow a * break/continue/return in the * trap action to have an effect * outside of it. */ if (evalskip == 0 || prev_evalskip != 0) { evalskip = prev_evalskip; skipcount = prev_skipcount; exitstatus = savestatus; } if (i == SIGCHLD) ignore_sigchld--; } break; } } if (i >= NSIG) break; } in_dotrap--; } /* * Controls whether the shell is interactive or not. */ void setinteractive(int on) { static int is_interactive = -1; if (on == is_interactive) return; setsignal(SIGINT); setsignal(SIGQUIT); setsignal(SIGTERM); is_interactive = on; } /* * Called to exit the shell. */ void exitshell(int status) { TRACE(("exitshell(%d) pid=%d\n", status, getpid())); exiting = 1; exiting_exitstatus = status; exitshell_savedstatus(); } void exitshell_savedstatus(void) { struct jmploc loc1, loc2; char *p; int sig = 0; sigset_t sigs; if (!exiting) { if (in_dotrap && last_trapsig) { sig = last_trapsig; exiting_exitstatus = sig + 128; } else exiting_exitstatus = oexitstatus; } exitstatus = oexitstatus = exiting_exitstatus; if (!setjmp(loc1.loc)) { handler = &loc1; if ((p = trap[0]) != NULL && *p != '\0') { /* * Reset evalskip, or the trap on EXIT could be * interrupted if the last command was a "return". */ evalskip = 0; trap[0] = NULL; evalstring(p, 0); } } if (!setjmp(loc2.loc)) { handler = &loc2; /* probably unnecessary */ flushall(); #if JOBS setjobctl(0); #endif } if (sig != 0 && sig != SIGSTOP && sig != SIGTSTP && sig != SIGTTIN && sig != SIGTTOU) { signal(sig, SIG_DFL); sigemptyset(&sigs); sigaddset(&sigs, sig); sigprocmask(SIG_UNBLOCK, &sigs, NULL); kill(getpid(), sig); /* If the default action is to ignore, fall back to _exit(). */ } _exit(exiting_exitstatus); } diff --git a/lib/libproc/proc_sym.c b/lib/libproc/proc_sym.c index 90d40a899c33..d4f82629fb7b 100644 --- a/lib/libproc/proc_sym.c +++ b/lib/libproc/proc_sym.c @@ -1,612 +1,616 @@ /*- * Copyright (c) 2010 The FreeBSD Foundation * Copyright (c) 2008 John Birrell (jb@freebsd.org) * All rights reserved. * * Portions of this software were developed by Rui Paulo under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #ifndef NO_CTF #include #include #endif #include #include #include #include #include #include #include #include #include #ifndef NO_CTF #include #endif #include #include "_libproc.h" #ifdef NO_CTF typedef struct ctf_file ctf_file_t; #endif #ifndef NO_CXA_DEMANGLE extern char *__cxa_demangle(const char *, char *, size_t *, int *); #endif /* NO_CXA_DEMANGLE */ static void proc_rdl2prmap(rd_loadobj_t *, prmap_t *); static void demangle(const char *symbol, char *buf, size_t len) { #ifndef NO_CXA_DEMANGLE char *dembuf; if (symbol[0] == '_' && symbol[1] == 'Z' && symbol[2]) { dembuf = __cxa_demangle(symbol, NULL, NULL, NULL); if (!dembuf) goto fail; strlcpy(buf, dembuf, len); free(dembuf); return; } fail: #endif /* NO_CXA_DEMANGLE */ strlcpy(buf, symbol, len); } static void proc_rdl2prmap(rd_loadobj_t *rdl, prmap_t *map) { map->pr_vaddr = rdl->rdl_saddr; map->pr_size = rdl->rdl_eaddr - rdl->rdl_saddr; map->pr_offset = rdl->rdl_offset; map->pr_mflags = 0; if (rdl->rdl_prot & RD_RDL_R) map->pr_mflags |= MA_READ; if (rdl->rdl_prot & RD_RDL_W) map->pr_mflags |= MA_WRITE; if (rdl->rdl_prot & RD_RDL_X) map->pr_mflags |= MA_EXEC; strlcpy(map->pr_mapname, rdl->rdl_path, sizeof(map->pr_mapname)); } char * proc_objname(struct proc_handle *p, uintptr_t addr, char *objname, size_t objnamesz) { size_t i; rd_loadobj_t *rdl; for (i = 0; i < p->nobjs; i++) { rdl = &p->rdobjs[i]; if (addr >= rdl->rdl_saddr && addr < rdl->rdl_eaddr) { strlcpy(objname, rdl->rdl_path, objnamesz); return (objname); } } return (NULL); } prmap_t * proc_obj2map(struct proc_handle *p, const char *objname) { size_t i; prmap_t *map; rd_loadobj_t *rdl; char path[MAXPATHLEN]; rdl = NULL; for (i = 0; i < p->nobjs; i++) { basename_r(p->rdobjs[i].rdl_path, path); if (strcmp(path, objname) == 0) { rdl = &p->rdobjs[i]; break; } } if (rdl == NULL) { if (strcmp(objname, "a.out") == 0 && p->rdexec != NULL) rdl = p->rdexec; else return (NULL); } if ((map = malloc(sizeof(*map))) == NULL) return (NULL); proc_rdl2prmap(rdl, map); return (map); } int proc_iter_objs(struct proc_handle *p, proc_map_f *func, void *cd) { size_t i; rd_loadobj_t *rdl; prmap_t map; char path[MAXPATHLEN]; char last[MAXPATHLEN]; + int error; if (p->nobjs == 0) return (-1); + + error = 0; memset(last, 0, sizeof(last)); for (i = 0; i < p->nobjs; i++) { rdl = &p->rdobjs[i]; proc_rdl2prmap(rdl, &map); basename_r(rdl->rdl_path, path); /* * We shouldn't call the callback twice with the same object. * To do that we are assuming the fact that if there are * repeated object names (i.e. different mappings for the * same object) they occur next to each other. */ if (strcmp(path, last) == 0) continue; - (*func)(cd, &map, path); + if ((error = (*func)(cd, &map, path)) != 0) + break; strlcpy(last, path, sizeof(last)); } - - return (0); + return (error); } prmap_t * proc_addr2map(struct proc_handle *p, uintptr_t addr) { size_t i; int cnt, lastvn = 0; prmap_t *map; rd_loadobj_t *rdl; struct kinfo_vmentry *kves, *kve; /* * If we don't have a cache of listed objects, we need to query * it ourselves. */ if (p->nobjs == 0) { if ((kves = kinfo_getvmmap(p->pid, &cnt)) == NULL) return (NULL); for (i = 0; i < (size_t)cnt; i++) { kve = kves + i; if (kve->kve_type == KVME_TYPE_VNODE) lastvn = i; if (addr >= kve->kve_start && addr < kve->kve_end) { if ((map = malloc(sizeof(*map))) == NULL) { free(kves); return (NULL); } map->pr_vaddr = kve->kve_start; map->pr_size = kve->kve_end - kve->kve_start; map->pr_offset = kve->kve_offset; map->pr_mflags = 0; if (kve->kve_protection & KVME_PROT_READ) map->pr_mflags |= MA_READ; if (kve->kve_protection & KVME_PROT_WRITE) map->pr_mflags |= MA_WRITE; if (kve->kve_protection & KVME_PROT_EXEC) map->pr_mflags |= MA_EXEC; if (kve->kve_flags & KVME_FLAG_COW) map->pr_mflags |= MA_COW; if (kve->kve_flags & KVME_FLAG_NEEDS_COPY) map->pr_mflags |= MA_NEEDS_COPY; if (kve->kve_flags & KVME_FLAG_NOCOREDUMP) map->pr_mflags |= MA_NOCOREDUMP; strlcpy(map->pr_mapname, kves[lastvn].kve_path, sizeof(map->pr_mapname)); free(kves); return (map); } } free(kves); return (NULL); } for (i = 0; i < p->nobjs; i++) { rdl = &p->rdobjs[i]; if (addr >= rdl->rdl_saddr && addr < rdl->rdl_eaddr) { if ((map = malloc(sizeof(*map))) == NULL) return (NULL); proc_rdl2prmap(rdl, map); return (map); } } return (NULL); } /* * Look up the symbol at addr, returning a copy of the symbol and its name. */ static int lookup_addr(Elf *e, Elf_Scn *scn, u_long stridx, uintptr_t off, uintptr_t addr, const char **name, GElf_Sym *symcopy) { GElf_Sym sym; Elf_Data *data; const char *s; uint64_t rsym; int i; if ((data = elf_getdata(scn, NULL)) == NULL) { DPRINTFX("ERROR: elf_getdata() failed: %s", elf_errmsg(-1)); return (1); } for (i = 0; gelf_getsym(data, i, &sym) != NULL; i++) { rsym = off + sym.st_value; if (addr >= rsym && addr < rsym + sym.st_size) { s = elf_strptr(e, stridx, sym.st_name); if (s != NULL) { *name = s; memcpy(symcopy, &sym, sizeof(*symcopy)); /* * DTrace expects the st_value to contain * only the address relative to the start of * the function. */ symcopy->st_value = rsym; return (0); } } } return (1); } int proc_addr2sym(struct proc_handle *p, uintptr_t addr, char *name, size_t namesz, GElf_Sym *symcopy) { GElf_Ehdr ehdr; GElf_Shdr shdr; Elf *e; Elf_Scn *scn, *dynsymscn = NULL, *symtabscn = NULL; prmap_t *map; const char *s; uintptr_t off; u_long symtabstridx = 0, dynsymstridx = 0; int fd, error = -1; if ((map = proc_addr2map(p, addr)) == NULL) return (-1); if ((fd = open(map->pr_mapname, O_RDONLY, 0)) < 0) { DPRINTF("ERROR: open %s failed", map->pr_mapname); goto err0; } if ((e = elf_begin(fd, ELF_C_READ, NULL)) == NULL) { DPRINTFX("ERROR: elf_begin() failed: %s", elf_errmsg(-1)); goto err1; } if (gelf_getehdr(e, &ehdr) == NULL) { DPRINTFX("ERROR: gelf_getehdr() failed: %s", elf_errmsg(-1)); goto err2; } /* * Find the index of the STRTAB and SYMTAB sections to locate * symbol names. */ scn = NULL; while ((scn = elf_nextscn(e, scn)) != NULL) { gelf_getshdr(scn, &shdr); switch (shdr.sh_type) { case SHT_SYMTAB: symtabscn = scn; symtabstridx = shdr.sh_link; break; case SHT_DYNSYM: dynsymscn = scn; dynsymstridx = shdr.sh_link; break; } } off = ehdr.e_type == ET_EXEC ? 0 : map->pr_vaddr; /* * First look up the symbol in the dynsymtab, and fall back to the * symtab if the lookup fails. */ error = lookup_addr(e, dynsymscn, dynsymstridx, off, addr, &s, symcopy); if (error == 0) goto out; error = lookup_addr(e, symtabscn, symtabstridx, off, addr, &s, symcopy); if (error != 0) goto err2; out: demangle(s, name, namesz); err2: elf_end(e); err1: close(fd); err0: free(map); return (error); } prmap_t * proc_name2map(struct proc_handle *p, const char *name) { size_t i; int cnt; prmap_t *map = NULL; char tmppath[MAXPATHLEN]; struct kinfo_vmentry *kves, *kve; rd_loadobj_t *rdl; /* * If we haven't iterated over the list of loaded objects, * librtld_db isn't yet initialized and it's very likely * that librtld_db called us. We need to do the heavy * lifting here to find the symbol librtld_db is looking for. */ if (p->nobjs == 0) { if ((kves = kinfo_getvmmap(proc_getpid(p), &cnt)) == NULL) return (NULL); for (i = 0; i < (size_t)cnt; i++) { kve = kves + i; basename_r(kve->kve_path, tmppath); if (strcmp(tmppath, name) == 0) { map = proc_addr2map(p, kve->kve_start); break; } } free(kves); } else for (i = 0; i < p->nobjs; i++) { rdl = &p->rdobjs[i]; basename_r(rdl->rdl_path, tmppath); if (strcmp(tmppath, name) == 0) { if ((map = malloc(sizeof(*map))) == NULL) return (NULL); proc_rdl2prmap(rdl, map); break; } } if (map == NULL && strcmp(name, "a.out") == 0 && p->rdexec != NULL) map = proc_addr2map(p, p->rdexec->rdl_saddr); return (map); } /* * Look up the symbol with the given name and return a copy of it. */ static int lookup_name(Elf *e, Elf_Scn *scn, u_long stridx, const char *symbol, GElf_Sym *symcopy, prsyminfo_t *si) { GElf_Sym sym; Elf_Data *data; char *s; int i; if ((data = elf_getdata(scn, NULL)) == NULL) { DPRINTFX("ERROR: elf_getdata() failed: %s", elf_errmsg(-1)); return (1); } for (i = 0; gelf_getsym(data, i, &sym) != NULL; i++) { s = elf_strptr(e, stridx, sym.st_name); if (s != NULL && strcmp(s, symbol) == 0) { memcpy(symcopy, &sym, sizeof(*symcopy)); if (si != NULL) si->prs_id = i; return (0); } } return (1); } int proc_name2sym(struct proc_handle *p, const char *object, const char *symbol, GElf_Sym *symcopy, prsyminfo_t *si) { Elf *e; Elf_Scn *scn, *dynsymscn = NULL, *symtabscn = NULL; GElf_Shdr shdr; GElf_Ehdr ehdr; prmap_t *map; uintptr_t off; u_long symtabstridx = 0, dynsymstridx = 0; int fd, error = -1; if ((map = proc_name2map(p, object)) == NULL) { DPRINTFX("ERROR: couldn't find object %s", object); goto err0; } if ((fd = open(map->pr_mapname, O_RDONLY, 0)) < 0) { DPRINTF("ERROR: open %s failed", map->pr_mapname); goto err0; } if ((e = elf_begin(fd, ELF_C_READ, NULL)) == NULL) { DPRINTFX("ERROR: elf_begin() failed: %s", elf_errmsg(-1)); goto err1; } if (gelf_getehdr(e, &ehdr) == NULL) { DPRINTFX("ERROR: gelf_getehdr() failed: %s", elf_errmsg(-1)); goto err2; } /* * Find the index of the STRTAB and SYMTAB sections to locate * symbol names. */ scn = NULL; while ((scn = elf_nextscn(e, scn)) != NULL) { gelf_getshdr(scn, &shdr); switch (shdr.sh_type) { case SHT_SYMTAB: symtabscn = scn; symtabstridx = shdr.sh_link; break; case SHT_DYNSYM: dynsymscn = scn; dynsymstridx = shdr.sh_link; break; } } /* * First look up the symbol in the dynsymtab, and fall back to the * symtab if the lookup fails. */ error = lookup_name(e, dynsymscn, dynsymstridx, symbol, symcopy, si); if (error == 0) goto out; error = lookup_name(e, symtabscn, symtabstridx, symbol, symcopy, si); if (error == 0) goto out; out: off = ehdr.e_type == ET_EXEC ? 0 : map->pr_vaddr; symcopy->st_value += off; err2: elf_end(e); err1: close(fd); err0: free(map); return (error); } ctf_file_t * proc_name2ctf(struct proc_handle *p, const char *name) { #ifndef NO_CTF ctf_file_t *ctf; prmap_t *map; int error; if ((map = proc_name2map(p, name)) == NULL) return (NULL); ctf = ctf_open(map->pr_mapname, &error); free(map); return (ctf); #else (void)p; (void)name; return (NULL); #endif } int proc_iter_symbyaddr(struct proc_handle *p, const char *object, int which, int mask, proc_sym_f *func, void *cd) { Elf *e; int i, fd; prmap_t *map; Elf_Scn *scn, *foundscn = NULL; Elf_Data *data; GElf_Ehdr ehdr; GElf_Shdr shdr; GElf_Sym sym; unsigned long stridx = -1; char *s; int error = -1; if ((map = proc_name2map(p, object)) == NULL) return (-1); if ((fd = open(map->pr_mapname, O_RDONLY)) < 0) { DPRINTF("ERROR: open %s failed", map->pr_mapname); goto err0; } if ((e = elf_begin(fd, ELF_C_READ, NULL)) == NULL) { DPRINTFX("ERROR: elf_begin() failed: %s", elf_errmsg(-1)); goto err1; } if (gelf_getehdr(e, &ehdr) == NULL) { DPRINTFX("ERROR: gelf_getehdr() failed: %s", elf_errmsg(-1)); goto err2; } /* * Find the section we are looking for. */ scn = NULL; while ((scn = elf_nextscn(e, scn)) != NULL) { gelf_getshdr(scn, &shdr); if (which == PR_SYMTAB && shdr.sh_type == SHT_SYMTAB) { foundscn = scn; break; } else if (which == PR_DYNSYM && shdr.sh_type == SHT_DYNSYM) { foundscn = scn; break; } } if (!foundscn) return (-1); stridx = shdr.sh_link; if ((data = elf_getdata(foundscn, NULL)) == NULL) { DPRINTFX("ERROR: elf_getdata() failed: %s", elf_errmsg(-1)); goto err2; } for (i = 0; gelf_getsym(data, i, &sym) != NULL; i++) { if (GELF_ST_BIND(sym.st_info) == STB_LOCAL && (mask & BIND_LOCAL) == 0) continue; if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL && (mask & BIND_GLOBAL) == 0) continue; if (GELF_ST_BIND(sym.st_info) == STB_WEAK && (mask & BIND_WEAK) == 0) continue; if (GELF_ST_TYPE(sym.st_info) == STT_NOTYPE && (mask & TYPE_NOTYPE) == 0) continue; if (GELF_ST_TYPE(sym.st_info) == STT_OBJECT && (mask & TYPE_OBJECT) == 0) continue; if (GELF_ST_TYPE(sym.st_info) == STT_FUNC && (mask & TYPE_FUNC) == 0) continue; if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && (mask & TYPE_SECTION) == 0) continue; if (GELF_ST_TYPE(sym.st_info) == STT_FILE && (mask & TYPE_FILE) == 0) continue; s = elf_strptr(e, stridx, sym.st_name); if (ehdr.e_type != ET_EXEC) sym.st_value += map->pr_vaddr; - (*func)(cd, &sym, s); + if ((error = (*func)(cd, &sym, s)) != 0) + goto err2; } error = 0; err2: elf_end(e); err1: close(fd); err0: free(map); return (error); } diff --git a/sys/arm/allwinner/a10_gpio.c b/sys/arm/allwinner/a10_gpio.c index 23bf399d3dc3..12d69890a500 100644 --- a/sys/arm/allwinner/a10_gpio.c +++ b/sys/arm/allwinner/a10_gpio.c @@ -1,528 +1,529 @@ /*- * Copyright (c) 2013 Ganbold Tsagaankhuu * Copyright (c) 2012 Oleksandr Tymoshenko * Copyright (c) 2012 Luiz Otavio O Souza. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "a10_gpio.h" /* * A10 have 9 banks of gpio. * 32 pins per bank: * PA0 - PA17 | PB0 - PB23 | PC0 - PC24 * PD0 - PD27 | PE0 - PE31 | PF0 - PF5 * PG0 - PG9 | PH0 - PH27 | PI0 - PI12 */ #define A10_GPIO_PINS 288 #define A10_GPIO_DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | \ GPIO_PIN_PULLUP | GPIO_PIN_PULLDOWN) #define A10_GPIO_NONE 0 #define A10_GPIO_PULLUP 1 #define A10_GPIO_PULLDOWN 2 #define A10_GPIO_INPUT 0 #define A10_GPIO_OUTPUT 1 struct a10_gpio_softc { device_t sc_dev; struct mtx sc_mtx; struct resource * sc_mem_res; struct resource * sc_irq_res; bus_space_tag_t sc_bst; bus_space_handle_t sc_bsh; void * sc_intrhand; int sc_gpio_npins; struct gpio_pin sc_gpio_pins[A10_GPIO_PINS]; }; #define A10_GPIO_LOCK(_sc) mtx_lock(&_sc->sc_mtx) #define A10_GPIO_UNLOCK(_sc) mtx_unlock(&_sc->sc_mtx) #define A10_GPIO_LOCK_ASSERT(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED) #define A10_GPIO_GP_CFG(_bank, _pin) 0x00 + ((_bank) * 0x24) + ((_pin)<<2) #define A10_GPIO_GP_DAT(_bank) 0x10 + ((_bank) * 0x24) #define A10_GPIO_GP_DRV(_bank, _pin) 0x14 + ((_bank) * 0x24) + ((_pin)<<2) #define A10_GPIO_GP_PUL(_bank, _pin) 0x1c + ((_bank) * 0x24) + ((_pin)<<2) #define A10_GPIO_GP_INT_CFG0 0x200 #define A10_GPIO_GP_INT_CFG1 0x204 #define A10_GPIO_GP_INT_CFG2 0x208 #define A10_GPIO_GP_INT_CFG3 0x20c #define A10_GPIO_GP_INT_CTL 0x210 #define A10_GPIO_GP_INT_STA 0x214 #define A10_GPIO_GP_INT_DEB 0x218 static struct a10_gpio_softc *a10_gpio_sc; #define A10_GPIO_WRITE(_sc, _off, _val) \ bus_space_write_4(_sc->sc_bst, _sc->sc_bsh, _off, _val) #define A10_GPIO_READ(_sc, _off) \ bus_space_read_4(_sc->sc_bst, _sc->sc_bsh, _off) static uint32_t a10_gpio_get_function(struct a10_gpio_softc *sc, uint32_t pin) { uint32_t bank, func, offset; bank = pin / 32; pin = pin - 32 * bank; func = pin >> 3; offset = ((pin & 0x07) << 2); A10_GPIO_LOCK(sc); func = (A10_GPIO_READ(sc, A10_GPIO_GP_CFG(bank, func)) >> offset) & 7; A10_GPIO_UNLOCK(sc); return (func); } static uint32_t a10_gpio_func_flag(uint32_t nfunc) { switch (nfunc) { case A10_GPIO_INPUT: return (GPIO_PIN_INPUT); case A10_GPIO_OUTPUT: return (GPIO_PIN_OUTPUT); } return (0); } static void a10_gpio_set_function(struct a10_gpio_softc *sc, uint32_t pin, uint32_t f) { uint32_t bank, func, data, offset; /* Must be called with lock held. */ A10_GPIO_LOCK_ASSERT(sc); bank = pin / 32; pin = pin - 32 * bank; func = pin >> 3; offset = ((pin & 0x07) << 2); data = A10_GPIO_READ(sc, A10_GPIO_GP_CFG(bank, func)); data &= ~(7 << offset); data |= (f << offset); A10_GPIO_WRITE(sc, A10_GPIO_GP_CFG(bank, func), data); } static void a10_gpio_set_pud(struct a10_gpio_softc *sc, uint32_t pin, uint32_t state) { uint32_t bank, offset, pull, val; /* Must be called with lock held. */ A10_GPIO_LOCK_ASSERT(sc); bank = pin / 32; pin = pin - 32 * bank; pull = pin >> 4; offset = ((pin & 0x0f) << 1); val = A10_GPIO_READ(sc, A10_GPIO_GP_PUL(bank, pull)); val &= ~(0x03 << offset); val |= (state << offset); A10_GPIO_WRITE(sc, A10_GPIO_GP_PUL(bank, pull), val); } static void a10_gpio_pin_configure(struct a10_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { A10_GPIO_LOCK(sc); /* * Manage input/output. */ if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; a10_gpio_set_function(sc, pin->gp_pin, A10_GPIO_OUTPUT); } else { pin->gp_flags |= GPIO_PIN_INPUT; a10_gpio_set_function(sc, pin->gp_pin, A10_GPIO_INPUT); } } /* Manage Pull-up/pull-down. */ pin->gp_flags &= ~(GPIO_PIN_PULLUP|GPIO_PIN_PULLDOWN); if (flags & (GPIO_PIN_PULLUP|GPIO_PIN_PULLDOWN)) { if (flags & GPIO_PIN_PULLUP) { pin->gp_flags |= GPIO_PIN_PULLUP; a10_gpio_set_pud(sc, pin->gp_pin, A10_GPIO_PULLUP); } else { pin->gp_flags |= GPIO_PIN_PULLDOWN; a10_gpio_set_pud(sc, pin->gp_pin, A10_GPIO_PULLDOWN); } } else a10_gpio_set_pud(sc, pin->gp_pin, A10_GPIO_NONE); A10_GPIO_UNLOCK(sc); } static int a10_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = A10_GPIO_PINS - 1; return (0); } static int a10_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct a10_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); A10_GPIO_LOCK(sc); *caps = sc->sc_gpio_pins[i].gp_caps; A10_GPIO_UNLOCK(sc); return (0); } static int a10_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct a10_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); A10_GPIO_LOCK(sc); *flags = sc->sc_gpio_pins[i].gp_flags; A10_GPIO_UNLOCK(sc); return (0); } static int a10_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct a10_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); A10_GPIO_LOCK(sc); memcpy(name, sc->sc_gpio_pins[i].gp_name, GPIOMAXNAME); A10_GPIO_UNLOCK(sc); return (0); } static int a10_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct a10_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); a10_gpio_pin_configure(sc, &sc->sc_gpio_pins[i], flags); return (0); } static int a10_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct a10_gpio_softc *sc = device_get_softc(dev); uint32_t bank, offset, data; int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); bank = pin / 32; pin = pin - 32 * bank; offset = pin & 0x1f; A10_GPIO_LOCK(sc); data = A10_GPIO_READ(sc, A10_GPIO_GP_DAT(bank)); if (value) data |= (1 << offset); else data &= ~(1 << offset); A10_GPIO_WRITE(sc, A10_GPIO_GP_DAT(bank), data); A10_GPIO_UNLOCK(sc); return (0); } static int a10_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct a10_gpio_softc *sc = device_get_softc(dev); uint32_t bank, offset, reg_data; int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); bank = pin / 32; pin = pin - 32 * bank; offset = pin & 0x1f; A10_GPIO_LOCK(sc); reg_data = A10_GPIO_READ(sc, A10_GPIO_GP_DAT(bank)); A10_GPIO_UNLOCK(sc); *val = (reg_data & (1 << offset)) ? 1 : 0; return (0); } static int a10_gpio_pin_toggle(device_t dev, uint32_t pin) { struct a10_gpio_softc *sc = device_get_softc(dev); uint32_t bank, data, offset; int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); bank = pin / 32; pin = pin - 32 * bank; offset = pin & 0x1f; A10_GPIO_LOCK(sc); data = A10_GPIO_READ(sc, A10_GPIO_GP_DAT(bank)); if (data & (1 << offset)) data &= ~(1 << offset); else data |= (1 << offset); A10_GPIO_WRITE(sc, A10_GPIO_GP_DAT(bank), data); A10_GPIO_UNLOCK(sc); return (0); } static int a10_gpio_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "allwinner,sun4i-gpio")) return (ENXIO); device_set_desc(dev, "Allwinner GPIO controller"); return (BUS_PROBE_DEFAULT); } static int a10_gpio_attach(device_t dev) { struct a10_gpio_softc *sc = device_get_softc(dev); uint32_t func; int i, rid; phandle_t gpio; sc->sc_dev = dev; mtx_init(&sc->sc_mtx, "a10 gpio", "gpio", MTX_DEF); rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!sc->sc_mem_res) { device_printf(dev, "cannot allocate memory window\n"); - return (ENXIO); + goto fail; } sc->sc_bst = rman_get_bustag(sc->sc_mem_res); sc->sc_bsh = rman_get_bushandle(sc->sc_mem_res); rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (!sc->sc_irq_res) { - bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); device_printf(dev, "cannot allocate interrupt\n"); - return (ENXIO); + goto fail; } /* Find our node. */ gpio = ofw_bus_get_node(sc->sc_dev); if (!OF_hasprop(gpio, "gpio-controller")) /* Node is not a GPIO controller. */ goto fail; /* Initialize the software controlled pins. */ for (i = 0; i < A10_GPIO_PINS; i++) { snprintf(sc->sc_gpio_pins[i].gp_name, GPIOMAXNAME, "pin %d", i); func = a10_gpio_get_function(sc, i); sc->sc_gpio_pins[i].gp_pin = i; sc->sc_gpio_pins[i].gp_caps = A10_GPIO_DEFAULT_CAPS; sc->sc_gpio_pins[i].gp_flags = a10_gpio_func_flag(func); } sc->sc_gpio_npins = i; device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); a10_gpio_sc = sc; return (bus_generic_attach(dev)); fail: if (sc->sc_irq_res) bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res); if (sc->sc_mem_res) bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); + mtx_destroy(&sc->sc_mtx); + return (ENXIO); } static int a10_gpio_detach(device_t dev) { return (EBUSY); } static device_method_t a10_gpio_methods[] = { /* Device interface */ DEVMETHOD(device_probe, a10_gpio_probe), DEVMETHOD(device_attach, a10_gpio_attach), DEVMETHOD(device_detach, a10_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, a10_gpio_pin_max), DEVMETHOD(gpio_pin_getname, a10_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, a10_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, a10_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, a10_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, a10_gpio_pin_get), DEVMETHOD(gpio_pin_set, a10_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, a10_gpio_pin_toggle), DEVMETHOD_END }; static devclass_t a10_gpio_devclass; static driver_t a10_gpio_driver = { "gpio", a10_gpio_methods, sizeof(struct a10_gpio_softc), }; DRIVER_MODULE(a10_gpio, simplebus, a10_gpio_driver, a10_gpio_devclass, 0, 0); int a10_emac_gpio_config(uint32_t pin) { struct a10_gpio_softc *sc = a10_gpio_sc; if (sc == NULL) return (ENXIO); /* Configure pin mux settings for MII. */ A10_GPIO_LOCK(sc); a10_gpio_set_function(sc, pin, A10_GPIO_PULLDOWN); A10_GPIO_UNLOCK(sc); return (0); } diff --git a/sys/arm/altera/socfpga/socfpga_gpio.c b/sys/arm/altera/socfpga/socfpga_gpio.c index 196c1e94b718..1a50c7c33bfb 100644 --- a/sys/arm/altera/socfpga/socfpga_gpio.c +++ b/sys/arm/altera/socfpga/socfpga_gpio.c @@ -1,437 +1,438 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * SOCFPGA General-Purpose I/O Interface. * Chapter 22, Cyclone V Device Handbook (CV-5V2 2014.07.22) */ /* * The GPIO modules are instances of the Synopsys® DesignWare® APB General * Purpose Programming I/O (DW_apb_gpio) peripheral. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #define READ4(_sc, _reg) \ bus_read_4((_sc)->res[0], _reg) #define WRITE4(_sc, _reg, _val) \ bus_write_4((_sc)->res[0], _reg, _val) #define GPIO_SWPORTA_DR 0x00 /* Port A Data Register */ #define GPIO_SWPORTA_DDR 0x04 /* Port A Data Direction Register */ #define GPIO_INTEN 0x30 /* Interrupt Enable Register */ #define GPIO_INTMASK 0x34 /* Interrupt Mask Register */ #define GPIO_INTTYPE_LEVEL 0x38 /* Interrupt Level Register */ #define GPIO_INT_POLARITY 0x3C /* Interrupt Polarity Register */ #define GPIO_INTSTATUS 0x40 /* Interrupt Status Register */ #define GPIO_RAW_INTSTATUS 0x44 /* Raw Interrupt Status Register */ #define GPIO_DEBOUNCE 0x48 /* Debounce Enable Register */ #define GPIO_PORTA_EOI 0x4C /* Clear Interrupt Register */ #define GPIO_EXT_PORTA 0x50 /* External Port A Register */ #define GPIO_LS_SYNC 0x60 /* Synchronization Level Register */ #define GPIO_ID_CODE 0x64 /* ID Code Register */ #define GPIO_VER_ID_CODE 0x6C /* GPIO Version Register */ #define GPIO_CONFIG_REG2 0x70 /* Configuration Register 2 */ #define ENCODED_ID_PWIDTH_M 0x1f /* Width of GPIO Port N Mask */ #define ENCODED_ID_PWIDTH_S(n) (5 * n) /* Width of GPIO Port N Shift */ #define GPIO_CONFIG_REG1 0x74 /* Configuration Register 1 */ enum port_no { PORTA, PORTB, PORTC, PORTD, }; #define NR_GPIO_MAX 32 /* Maximum pins per port */ #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT) /* * GPIO interface */ static int socfpga_gpio_pin_max(device_t, int *); static int socfpga_gpio_pin_getcaps(device_t, uint32_t, uint32_t *); static int socfpga_gpio_pin_getname(device_t, uint32_t, char *); static int socfpga_gpio_pin_getflags(device_t, uint32_t, uint32_t *); static int socfpga_gpio_pin_setflags(device_t, uint32_t, uint32_t); static int socfpga_gpio_pin_set(device_t, uint32_t, unsigned int); static int socfpga_gpio_pin_get(device_t, uint32_t, unsigned int *); static int socfpga_gpio_pin_toggle(device_t, uint32_t pin); struct socfpga_gpio_softc { struct resource *res[1]; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; struct mtx sc_mtx; int gpio_npins; struct gpio_pin gpio_pins[NR_GPIO_MAX]; }; struct socfpga_gpio_softc *gpio_sc; static struct resource_spec socfpga_gpio_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; static int socfpga_gpio_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "snps,dw-apb-gpio")) return (ENXIO); device_set_desc(dev, "DesignWare General-Purpose I/O Interface"); return (BUS_PROBE_DEFAULT); } static int socfpga_gpio_attach(device_t dev) { struct socfpga_gpio_softc *sc; int version; int nr_pins; int cfg2; int i; sc = device_get_softc(dev); sc->dev = dev; mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF); if (bus_alloc_resources(dev, socfpga_gpio_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); + mtx_destroy(&sc->sc_mtx); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); gpio_sc = sc; version = READ4(sc, GPIO_VER_ID_CODE); #if 0 device_printf(sc->dev, "Version = 0x%08x\n", version); #endif /* * Take number of pins from hardware. * XXX: Assume we have GPIO port A only. */ cfg2 = READ4(sc, GPIO_CONFIG_REG2); nr_pins = (cfg2 >> ENCODED_ID_PWIDTH_S(PORTA)) & \ ENCODED_ID_PWIDTH_M; sc->gpio_npins = nr_pins + 1; for (i = 0; i < sc->gpio_npins; i++) { sc->gpio_pins[i].gp_pin = i; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; sc->gpio_pins[i].gp_flags = (READ4(sc, GPIO_SWPORTA_DDR) & (1 << i)) ? GPIO_PIN_OUTPUT: GPIO_PIN_INPUT; snprintf(sc->gpio_pins[i].gp_name, GPIOMAXNAME, "socfpga_gpio%d.%d", device_get_unit(dev), i); } device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); } static int socfpga_gpio_pin_max(device_t dev, int *maxpin) { struct socfpga_gpio_softc *sc; sc = device_get_softc(dev); *maxpin = sc->gpio_npins - 1; return (0); } static int socfpga_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct socfpga_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME); GPIO_UNLOCK(sc); return (0); } static int socfpga_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct socfpga_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *caps = sc->gpio_pins[i].gp_caps; GPIO_UNLOCK(sc); return (0); } static int socfpga_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct socfpga_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *flags = sc->gpio_pins[i].gp_flags; GPIO_UNLOCK(sc); return (0); } static int socfpga_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct socfpga_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *val = (READ4(sc, GPIO_EXT_PORTA) & (1 << i)) ? 1 : 0; GPIO_UNLOCK(sc); return (0); } static int socfpga_gpio_pin_toggle(device_t dev, uint32_t pin) { struct socfpga_gpio_softc *sc; int reg; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); reg = READ4(sc, GPIO_SWPORTA_DR); if (reg & (1 << i)) reg &= ~(1 << i); else reg |= (1 << i); WRITE4(sc, GPIO_SWPORTA_DR, reg); GPIO_UNLOCK(sc); return (0); } static void socfpga_gpio_pin_configure(struct socfpga_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { int reg; GPIO_LOCK(sc); /* * Manage input/output */ reg = READ4(sc, GPIO_SWPORTA_DDR); if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; reg |= (1 << pin->gp_pin); } else { pin->gp_flags |= GPIO_PIN_INPUT; reg &= ~(1 << pin->gp_pin); } } WRITE4(sc, GPIO_SWPORTA_DDR, reg); GPIO_UNLOCK(sc); } static int socfpga_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct socfpga_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); socfpga_gpio_pin_configure(sc, &sc->gpio_pins[i], flags); return (0); } static int socfpga_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct socfpga_gpio_softc *sc; int reg; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); reg = READ4(sc, GPIO_SWPORTA_DR); if (value) reg |= (1 << i); else reg &= ~(1 << i); WRITE4(sc, GPIO_SWPORTA_DR, reg); GPIO_UNLOCK(sc); return (0); } static device_method_t socfpga_gpio_methods[] = { DEVMETHOD(device_probe, socfpga_gpio_probe), DEVMETHOD(device_attach, socfpga_gpio_attach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, socfpga_gpio_pin_max), DEVMETHOD(gpio_pin_getname, socfpga_gpio_pin_getname), DEVMETHOD(gpio_pin_getcaps, socfpga_gpio_pin_getcaps), DEVMETHOD(gpio_pin_getflags, socfpga_gpio_pin_getflags), DEVMETHOD(gpio_pin_get, socfpga_gpio_pin_get), DEVMETHOD(gpio_pin_toggle, socfpga_gpio_pin_toggle), DEVMETHOD(gpio_pin_setflags, socfpga_gpio_pin_setflags), DEVMETHOD(gpio_pin_set, socfpga_gpio_pin_set), { 0, 0 } }; static driver_t socfpga_gpio_driver = { "gpio", socfpga_gpio_methods, sizeof(struct socfpga_gpio_softc), }; static devclass_t socfpga_gpio_devclass; DRIVER_MODULE(socfpga_gpio, simplebus, socfpga_gpio_driver, socfpga_gpio_devclass, 0, 0); diff --git a/sys/arm/freescale/imx/imx_gpio.c b/sys/arm/freescale/imx/imx_gpio.c index c23f75be3f7c..911c9bbbf748 100644 --- a/sys/arm/freescale/imx/imx_gpio.c +++ b/sys/arm/freescale/imx/imx_gpio.c @@ -1,481 +1,484 @@ /*- * Copyright (c) 2012, 2013 The FreeBSD Foundation * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Freescale i.MX515 GPIO driver. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define GPIO_LOCK_INIT(_sc) mtx_init(&_sc->sc_mtx, \ device_get_nameunit(_sc->sc_dev), "imx_gpio", MTX_DEF) #define GPIO_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define GPIO_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define GPIO_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); #define WRITE4(_sc, _r, _v) \ bus_space_write_4((_sc)->sc_iot, (_sc)->sc_ioh, (_r), (_v)) #define READ4(_sc, _r) \ bus_space_read_4((_sc)->sc_iot, (_sc)->sc_ioh, (_r)) #define SET4(_sc, _r, _m) \ WRITE4((_sc), (_r), READ4((_sc), (_r)) | (_m)) #define CLEAR4(_sc, _r, _m) \ WRITE4((_sc), (_r), READ4((_sc), (_r)) & ~(_m)) /* Registers definition for Freescale i.MX515 GPIO controller */ #define IMX_GPIO_DR_REG 0x000 /* Pin Data */ #define IMX_GPIO_OE_REG 0x004 /* Set Pin Output */ #define IMX_GPIO_PSR_REG 0x008 /* Pad Status */ #define IMX_GPIO_ICR1_REG 0x00C /* Interrupt Configuration */ #define IMX_GPIO_ICR2_REG 0x010 /* Interrupt Configuration */ #define GPIO_ICR_COND_LOW 0 #define GPIO_ICR_COND_HIGH 1 #define GPIO_ICR_COND_RISE 2 #define GPIO_ICR_COND_FALL 3 #define IMX_GPIO_IMR_REG 0x014 /* Interrupt Mask Register */ #define IMX_GPIO_ISR_REG 0x018 /* Interrupt Status Register */ #define IMX_GPIO_EDGE_REG 0x01C /* Edge Detect Register */ #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT) #define NGPIO 32 struct imx51_gpio_softc { device_t dev; struct mtx sc_mtx; struct resource *sc_res[11]; /* 1 x mem, 2 x IRQ, 8 x IRQ */ void *gpio_ih[11]; /* 1 ptr is not a big waste */ int sc_l_irq; /* Last irq resource */ bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; int gpio_npins; struct gpio_pin gpio_pins[NGPIO]; }; static struct ofw_compat_data compat_data[] = { {"fsl,imx6q-gpio", 1}, {"fsl,imx53-gpio", 1}, {"fsl,imx51-gpio", 1}, {NULL, 0} }; static struct resource_spec imx_gpio_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE }, { -1, 0 } }; static struct resource_spec imx_gpio0irq_spec[] = { { SYS_RES_IRQ, 2, RF_ACTIVE }, { SYS_RES_IRQ, 3, RF_ACTIVE }, { SYS_RES_IRQ, 4, RF_ACTIVE }, { SYS_RES_IRQ, 5, RF_ACTIVE }, { SYS_RES_IRQ, 6, RF_ACTIVE }, { SYS_RES_IRQ, 7, RF_ACTIVE }, { SYS_RES_IRQ, 8, RF_ACTIVE }, { SYS_RES_IRQ, 9, RF_ACTIVE }, { -1, 0 } }; /* * Helpers */ static void imx51_gpio_pin_configure(struct imx51_gpio_softc *, struct gpio_pin *, uint32_t); /* * Driver stuff */ static int imx51_gpio_probe(device_t); static int imx51_gpio_attach(device_t); static int imx51_gpio_detach(device_t); static int imx51_gpio_intr(void *); /* * GPIO interface */ static int imx51_gpio_pin_max(device_t, int *); static int imx51_gpio_pin_getcaps(device_t, uint32_t, uint32_t *); static int imx51_gpio_pin_getflags(device_t, uint32_t, uint32_t *); static int imx51_gpio_pin_getname(device_t, uint32_t, char *); static int imx51_gpio_pin_setflags(device_t, uint32_t, uint32_t); static int imx51_gpio_pin_set(device_t, uint32_t, unsigned int); static int imx51_gpio_pin_get(device_t, uint32_t, unsigned int *); static int imx51_gpio_pin_toggle(device_t, uint32_t pin); static void imx51_gpio_pin_configure(struct imx51_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { GPIO_LOCK(sc); /* * Manage input/output */ if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; SET4(sc, IMX_GPIO_OE_REG, (1 << pin->gp_pin)); } else { pin->gp_flags |= GPIO_PIN_INPUT; CLEAR4(sc, IMX_GPIO_OE_REG, (1 << pin->gp_pin)); } } GPIO_UNLOCK(sc); } static int imx51_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = NGPIO - 1; return (0); } static int imx51_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct imx51_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *caps = sc->gpio_pins[i].gp_caps; GPIO_UNLOCK(sc); return (0); } static int imx51_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct imx51_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *flags = sc->gpio_pins[i].gp_flags; GPIO_UNLOCK(sc); return (0); } static int imx51_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct imx51_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME); GPIO_UNLOCK(sc); return (0); } static int imx51_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct imx51_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); imx51_gpio_pin_configure(sc, &sc->gpio_pins[i], flags); return (0); } static int imx51_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct imx51_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); if (value) SET4(sc, IMX_GPIO_DR_REG, (1 << i)); else CLEAR4(sc, IMX_GPIO_DR_REG, (1 << i)); GPIO_UNLOCK(sc); return (0); } static int imx51_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct imx51_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *val = (READ4(sc, IMX_GPIO_DR_REG) >> i) & 1; GPIO_UNLOCK(sc); return (0); } static int imx51_gpio_pin_toggle(device_t dev, uint32_t pin) { struct imx51_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); WRITE4(sc, IMX_GPIO_DR_REG, (READ4(sc, IMX_GPIO_DR_REG) ^ (1 << i))); GPIO_UNLOCK(sc); return (0); } static int imx51_gpio_intr(void *arg) { struct imx51_gpio_softc *sc; uint32_t input, value; sc = arg; input = READ4(sc, IMX_GPIO_ISR_REG); value = input & READ4(sc, IMX_GPIO_IMR_REG); WRITE4(sc, IMX_GPIO_ISR_REG, input); if (!value) goto intr_done; /* TODO: interrupt handling */ intr_done: return (FILTER_HANDLED); } static int imx51_gpio_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data != 0) { device_set_desc(dev, "Freescale i.MX GPIO Controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int imx51_gpio_attach(device_t dev) { struct imx51_gpio_softc *sc; int i, irq; sc = device_get_softc(dev); mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF); if (bus_alloc_resources(dev, imx_gpio_spec, sc->sc_res)) { device_printf(dev, "could not allocate resources\n"); + bus_release_resources(dev, imx_gpio_spec, sc->sc_res); + mtx_destroy(&sc->sc_mtx); return (ENXIO); } sc->dev = dev; sc->gpio_npins = NGPIO; sc->sc_l_irq = 2; sc->sc_iot = rman_get_bustag(sc->sc_res[0]); sc->sc_ioh = rman_get_bushandle(sc->sc_res[0]); if (bus_alloc_resources(dev, imx_gpio0irq_spec, &sc->sc_res[3]) == 0) { /* * First GPIO unit able to serve +8 interrupts for 8 first * pins. */ sc->sc_l_irq = 10; } for (irq = 1; irq <= sc->sc_l_irq; irq ++) { if ((bus_setup_intr(dev, sc->sc_res[irq], INTR_TYPE_MISC, imx51_gpio_intr, NULL, sc, &sc->gpio_ih[irq]))) { device_printf(dev, "WARNING: unable to register interrupt handler\n"); + imx51_gpio_detach(dev); return (ENXIO); } } for (i = 0; i < sc->gpio_npins; i++) { sc->gpio_pins[i].gp_pin = i; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; sc->gpio_pins[i].gp_flags = (READ4(sc, IMX_GPIO_OE_REG) & (1 << i)) ? GPIO_PIN_OUTPUT: GPIO_PIN_INPUT; snprintf(sc->gpio_pins[i].gp_name, GPIOMAXNAME, "imx_gpio%d.%d", device_get_unit(dev), i); } device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); } static int imx51_gpio_detach(device_t dev) { + int irq; struct imx51_gpio_softc *sc; sc = device_get_softc(dev); KASSERT(mtx_initialized(&sc->sc_mtx), ("gpio mutex not initialized")); bus_generic_detach(dev); - - if (sc->sc_res[3]) - bus_release_resources(dev, imx_gpio0irq_spec, &sc->sc_res[3]); - - if (sc->sc_res[0]) - bus_release_resources(dev, imx_gpio_spec, sc->sc_res); - + for (irq = 1; irq <= sc->sc_l_irq; irq ++) { + if (sc->gpio_ih[irq]) + bus_teardown_intr(dev, sc->sc_res[irq], sc->gpio_ih[irq]); + } + bus_release_resources(dev, imx_gpio0irq_spec, &sc->sc_res[3]); + bus_release_resources(dev, imx_gpio_spec, sc->sc_res); mtx_destroy(&sc->sc_mtx); return(0); } static device_method_t imx51_gpio_methods[] = { DEVMETHOD(device_probe, imx51_gpio_probe), DEVMETHOD(device_attach, imx51_gpio_attach), DEVMETHOD(device_detach, imx51_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, imx51_gpio_pin_max), DEVMETHOD(gpio_pin_getname, imx51_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, imx51_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, imx51_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, imx51_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, imx51_gpio_pin_get), DEVMETHOD(gpio_pin_set, imx51_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, imx51_gpio_pin_toggle), {0, 0}, }; static driver_t imx51_gpio_driver = { "gpio", imx51_gpio_methods, sizeof(struct imx51_gpio_softc), }; static devclass_t imx51_gpio_devclass; DRIVER_MODULE(imx51_gpio, simplebus, imx51_gpio_driver, imx51_gpio_devclass, 0, 0); diff --git a/sys/arm/freescale/vybrid/vf_gpio.c b/sys/arm/freescale/vybrid/vf_gpio.c index a31ff782aacf..1fcf32903d09 100644 --- a/sys/arm/freescale/vybrid/vf_gpio.c +++ b/sys/arm/freescale/vybrid/vf_gpio.c @@ -1,369 +1,370 @@ /*- * Copyright (c) 2013 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Vybrid Family General-Purpose Input/Output (GPIO) * Chapter 7, Vybrid Reference Manual, Rev. 5, 07/2013 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include #include #define GPIO_PDOR(n) (0x00 + 0x40 * (n >> 5)) #define GPIO_PSOR(n) (0x04 + 0x40 * (n >> 5)) #define GPIO_PCOR(n) (0x08 + 0x40 * (n >> 5)) #define GPIO_PTOR(n) (0x0C + 0x40 * (n >> 5)) #define GPIO_PDIR(n) (0x10 + 0x40 * (n >> 5)) #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT) /* * GPIO interface */ static int vf_gpio_pin_max(device_t, int *); static int vf_gpio_pin_getcaps(device_t, uint32_t, uint32_t *); static int vf_gpio_pin_getname(device_t, uint32_t, char *); static int vf_gpio_pin_getflags(device_t, uint32_t, uint32_t *); static int vf_gpio_pin_setflags(device_t, uint32_t, uint32_t); static int vf_gpio_pin_set(device_t, uint32_t, unsigned int); static int vf_gpio_pin_get(device_t, uint32_t, unsigned int *); static int vf_gpio_pin_toggle(device_t, uint32_t pin); struct vf_gpio_softc { struct resource *res[1]; bus_space_tag_t bst; bus_space_handle_t bsh; struct mtx sc_mtx; int gpio_npins; struct gpio_pin gpio_pins[NGPIO]; }; struct vf_gpio_softc *gpio_sc; static struct resource_spec vf_gpio_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; static int vf_gpio_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "fsl,mvf600-gpio")) return (ENXIO); device_set_desc(dev, "Vybrid Family GPIO Unit"); return (BUS_PROBE_DEFAULT); } static int vf_gpio_attach(device_t dev) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF); if (bus_alloc_resources(dev, vf_gpio_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); + mtx_destroy(&sc->sc_mtx); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); gpio_sc = sc; sc->gpio_npins = NGPIO; for (i = 0; i < sc->gpio_npins; i++) { sc->gpio_pins[i].gp_pin = i; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; sc->gpio_pins[i].gp_flags = (READ4(sc, GPIO_PDOR(i)) & (1 << (i % 32))) ? GPIO_PIN_OUTPUT: GPIO_PIN_INPUT; snprintf(sc->gpio_pins[i].gp_name, GPIOMAXNAME, "vf_gpio%d.%d", device_get_unit(dev), i); } device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); } static int vf_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = NGPIO - 1; return (0); } static int vf_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME); GPIO_UNLOCK(sc); return (0); } static int vf_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *caps = sc->gpio_pins[i].gp_caps; GPIO_UNLOCK(sc); return (0); } static int vf_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *flags = sc->gpio_pins[i].gp_flags; GPIO_UNLOCK(sc); return (0); } static int vf_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *val = (READ4(sc, GPIO_PDIR(i)) & (1 << (i % 32))) ? 1 : 0; GPIO_UNLOCK(sc); return (0); } static int vf_gpio_pin_toggle(device_t dev, uint32_t pin) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); WRITE4(sc, GPIO_PTOR(i), (1 << (i % 32))); GPIO_UNLOCK(sc); return (0); } static void vf_gpio_pin_configure(struct vf_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { GPIO_LOCK(sc); /* * Manage input/output */ if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; } else { pin->gp_flags |= GPIO_PIN_INPUT; WRITE4(sc, GPIO_PCOR(pin->gp_pin), (1 << (pin->gp_pin % 32))); } } GPIO_UNLOCK(sc); } static int vf_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); vf_gpio_pin_configure(sc, &sc->gpio_pins[i], flags); return (0); } static int vf_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct vf_gpio_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); if (value) WRITE4(sc, GPIO_PSOR(i), (1 << (i % 32))); else WRITE4(sc, GPIO_PCOR(i), (1 << (i % 32))); GPIO_UNLOCK(sc); return (0); } static device_method_t vf_gpio_methods[] = { DEVMETHOD(device_probe, vf_gpio_probe), DEVMETHOD(device_attach, vf_gpio_attach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, vf_gpio_pin_max), DEVMETHOD(gpio_pin_getname, vf_gpio_pin_getname), DEVMETHOD(gpio_pin_getcaps, vf_gpio_pin_getcaps), DEVMETHOD(gpio_pin_getflags, vf_gpio_pin_getflags), DEVMETHOD(gpio_pin_get, vf_gpio_pin_get), DEVMETHOD(gpio_pin_toggle, vf_gpio_pin_toggle), DEVMETHOD(gpio_pin_setflags, vf_gpio_pin_setflags), DEVMETHOD(gpio_pin_set, vf_gpio_pin_set), { 0, 0 } }; static driver_t vf_gpio_driver = { "gpio", vf_gpio_methods, sizeof(struct vf_gpio_softc), }; static devclass_t vf_gpio_devclass; DRIVER_MODULE(vf_gpio, simplebus, vf_gpio_driver, vf_gpio_devclass, 0, 0); diff --git a/sys/arm/rockchip/rk30xx_gpio.c b/sys/arm/rockchip/rk30xx_gpio.c index 6eb7db742b99..0728dbf09891 100644 --- a/sys/arm/rockchip/rk30xx_gpio.c +++ b/sys/arm/rockchip/rk30xx_gpio.c @@ -1,625 +1,624 @@ /*- * Copyright (c) 2013 Ganbold Tsagaankhuu * Copyright (c) 2012 Oleksandr Tymoshenko * Copyright (c) 2012 Luiz Otavio O Souza. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "rk30xx_grf.h" #include "rk30xx_pmu.h" /* * RK3188 has 4 banks of gpio. * 32 pins per bank * PA0 - PA7 | PB0 - PB7 * PC0 - PC7 | PD0 - PD7 */ #define RK30_GPIO_PINS 32 #define RK30_GPIO_DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | \ GPIO_PIN_PULLUP | GPIO_PIN_PULLDOWN) #define RK30_GPIO_NONE 0 #define RK30_GPIO_PULLUP 1 #define RK30_GPIO_PULLDOWN 2 struct rk30_gpio_softc { device_t sc_dev; struct mtx sc_mtx; struct resource * sc_mem_res; struct resource * sc_irq_res; bus_space_tag_t sc_bst; bus_space_handle_t sc_bsh; void * sc_intrhand; int sc_bank; int sc_gpio_npins; struct gpio_pin sc_gpio_pins[RK30_GPIO_PINS]; }; /* We use our base address to find out our bank number. */ static unsigned long rk30_gpio_base_addr[4] = { 0x2000a000, 0x2003c000, 0x2003e000, 0x20080000 }; static struct rk30_gpio_softc *rk30_gpio_sc = NULL; typedef int (*gpios_phandler_t)(phandle_t, pcell_t *, int); struct gpio_ctrl_entry { const char *compat; gpios_phandler_t handler; }; int rk30_gpios_prop_handle(phandle_t ctrl, pcell_t *gpios, int len); static int rk30_gpio_init(void); struct gpio_ctrl_entry gpio_controllers[] = { { "rockchip,rk30xx-gpio", &rk30_gpios_prop_handle }, { "rockchip,rk30xx-gpio", &rk30_gpios_prop_handle }, { "rockchip,rk30xx-gpio", &rk30_gpios_prop_handle }, { "rockchip,rk30xx-gpio", &rk30_gpios_prop_handle }, { NULL, NULL } }; #define RK30_GPIO_LOCK(_sc) mtx_lock(&_sc->sc_mtx) #define RK30_GPIO_UNLOCK(_sc) mtx_unlock(&_sc->sc_mtx) #define RK30_GPIO_LOCK_ASSERT(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED) #define RK30_GPIO_SWPORT_DR 0x00 #define RK30_GPIO_SWPORT_DDR 0x04 #define RK30_GPIO_INTEN 0x30 #define RK30_GPIO_INTMASK 0x34 #define RK30_GPIO_INTTYPE_LEVEL 0x38 #define RK30_GPIO_INT_POLARITY 0x3c #define RK30_GPIO_INT_STATUS 0x40 #define RK30_GPIO_INT_RAWSTATUS 0x44 #define RK30_GPIO_DEBOUNCE 0x48 #define RK30_GPIO_PORT_EOI 0x4c #define RK30_GPIO_EXT_PORT 0x50 #define RK30_GPIO_LS_SYNC 0x60 #define RK30_GPIO_WRITE(_sc, _off, _val) \ bus_space_write_4(_sc->sc_bst, _sc->sc_bsh, _off, _val) #define RK30_GPIO_READ(_sc, _off) \ bus_space_read_4(_sc->sc_bst, _sc->sc_bsh, _off) static uint32_t rk30_gpio_get_function(struct rk30_gpio_softc *sc, uint32_t pin) { if (RK30_GPIO_READ(sc, RK30_GPIO_SWPORT_DDR) & (1U << pin)) return (GPIO_PIN_OUTPUT); else return (GPIO_PIN_INPUT); } static void rk30_gpio_set_function(struct rk30_gpio_softc *sc, uint32_t pin, uint32_t func) { uint32_t data; /* Must be called with lock held. */ RK30_GPIO_LOCK_ASSERT(sc); data = RK30_GPIO_READ(sc, RK30_GPIO_SWPORT_DDR); if (func == GPIO_PIN_OUTPUT) data |= (1U << pin); else data &= ~(1U << pin); RK30_GPIO_WRITE(sc, RK30_GPIO_SWPORT_DDR, data); } static void rk30_gpio_set_pud(struct rk30_gpio_softc *sc, uint32_t pin, uint32_t state) { uint32_t pud; /* Must be called with lock held. */ RK30_GPIO_LOCK_ASSERT(sc); switch (state) { case GPIO_PIN_PULLUP: pud = RK30_GPIO_PULLUP; break; case GPIO_PIN_PULLDOWN: pud = RK30_GPIO_PULLDOWN; break; default: pud = RK30_GPIO_NONE; } /* * The pull up/down registers for GPIO0A and half of GPIO0B * (the first 12 pins on bank 0) are at a different location. */ if (sc->sc_bank == 0 && pin < 12) rk30_pmu_gpio_pud(pin, pud); else rk30_grf_gpio_pud(sc->sc_bank, pin, pud); } static void rk30_gpio_pin_configure(struct rk30_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { RK30_GPIO_LOCK(sc); /* * Manage input/output. */ if (flags & (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT | GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) pin->gp_flags |= GPIO_PIN_OUTPUT; else pin->gp_flags |= GPIO_PIN_INPUT; rk30_gpio_set_function(sc, pin->gp_pin, pin->gp_flags); } /* Manage Pull-up/pull-down. */ pin->gp_flags &= ~(GPIO_PIN_PULLUP | GPIO_PIN_PULLDOWN); if (flags & (GPIO_PIN_PULLUP | GPIO_PIN_PULLDOWN)) { if (flags & GPIO_PIN_PULLUP) pin->gp_flags |= GPIO_PIN_PULLUP; else pin->gp_flags |= GPIO_PIN_PULLDOWN; } rk30_gpio_set_pud(sc, pin->gp_pin, pin->gp_flags); RK30_GPIO_UNLOCK(sc); } static int rk30_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = RK30_GPIO_PINS - 1; return (0); } static int rk30_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct rk30_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); RK30_GPIO_LOCK(sc); *caps = sc->sc_gpio_pins[i].gp_caps; RK30_GPIO_UNLOCK(sc); return (0); } static int rk30_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct rk30_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); RK30_GPIO_LOCK(sc); *flags = sc->sc_gpio_pins[i].gp_flags; RK30_GPIO_UNLOCK(sc); return (0); } static int rk30_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct rk30_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); RK30_GPIO_LOCK(sc); memcpy(name, sc->sc_gpio_pins[i].gp_name, GPIOMAXNAME); RK30_GPIO_UNLOCK(sc); return (0); } static int rk30_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct rk30_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); rk30_gpio_pin_configure(sc, &sc->sc_gpio_pins[i], flags); return (0); } static int rk30_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { int i; struct rk30_gpio_softc *sc; uint32_t data; sc = device_get_softc(dev); for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); RK30_GPIO_LOCK(sc); data = RK30_GPIO_READ(sc, RK30_GPIO_SWPORT_DR); if (value) data |= (1U << pin); else data &= ~(1U << pin); RK30_GPIO_WRITE(sc, RK30_GPIO_SWPORT_DR, data); RK30_GPIO_UNLOCK(sc); return (0); } static int rk30_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { int i; struct rk30_gpio_softc *sc; uint32_t data; sc = device_get_softc(dev); for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); RK30_GPIO_LOCK(sc); data = RK30_GPIO_READ(sc, RK30_GPIO_EXT_PORT); RK30_GPIO_UNLOCK(sc); *val = (data & (1U << pin)) ? 1 : 0; return (0); } static int rk30_gpio_pin_toggle(device_t dev, uint32_t pin) { int i; struct rk30_gpio_softc *sc; uint32_t data; sc = device_get_softc(dev); for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); RK30_GPIO_LOCK(sc); data = RK30_GPIO_READ(sc, RK30_GPIO_SWPORT_DR); if (data & (1U << pin)) data &= ~(1U << pin); else data |= (1U << pin); RK30_GPIO_WRITE(sc, RK30_GPIO_SWPORT_DR, data); RK30_GPIO_UNLOCK(sc); return (0); } static int rk30_gpio_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "rockchip,rk30xx-gpio")) return (ENXIO); device_set_desc(dev, "Rockchip RK30XX GPIO controller"); return (BUS_PROBE_DEFAULT); } static int rk30_gpio_attach(device_t dev) { struct rk30_gpio_softc *sc = device_get_softc(dev); int i, rid; phandle_t gpio; unsigned long start; if (rk30_gpio_sc) return (ENXIO); sc->sc_dev = dev; + mtx_init(&sc->sc_mtx, "rk30 gpio", "gpio", MTX_DEF); rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!sc->sc_mem_res) { device_printf(dev, "cannot allocate memory window\n"); - return (ENXIO); + goto fail; } sc->sc_bst = rman_get_bustag(sc->sc_mem_res); sc->sc_bsh = rman_get_bushandle(sc->sc_mem_res); /* Check the unit we are attaching by our base address. */ sc->sc_bank = -1; start = rman_get_start(sc->sc_mem_res); for (i = 0; i < nitems(rk30_gpio_base_addr); i++) { if (rk30_gpio_base_addr[i] == start) { sc->sc_bank = i; break; } } if (sc->sc_bank == -1) { device_printf(dev, "unsupported device unit (only GPIO0..3 are supported)\n"); - bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); - return (ENXIO); + goto fail; } rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (!sc->sc_irq_res) { - bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); device_printf(dev, "cannot allocate interrupt\n"); - return (ENXIO); + goto fail; } /* Find our node. */ gpio = ofw_bus_get_node(sc->sc_dev); if (!OF_hasprop(gpio, "gpio-controller")) /* Node is not a GPIO controller. */ goto fail; - mtx_init(&sc->sc_mtx, "rk30 gpio", "gpio", MTX_DEF); - /* Initialize the software controlled pins. */ for (i = 0; i < RK30_GPIO_PINS; i++) { snprintf(sc->sc_gpio_pins[i].gp_name, GPIOMAXNAME, "pin %d", i); sc->sc_gpio_pins[i].gp_pin = i; sc->sc_gpio_pins[i].gp_caps = RK30_GPIO_DEFAULT_CAPS; sc->sc_gpio_pins[i].gp_flags = rk30_gpio_get_function(sc, i); } sc->sc_gpio_npins = i; device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); rk30_gpio_sc = sc; rk30_gpio_init(); return (bus_generic_attach(dev)); fail: if (sc->sc_irq_res) bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res); if (sc->sc_mem_res) bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); + mtx_destroy(&sc->sc_mtx); + return (ENXIO); } static int rk30_gpio_detach(device_t dev) { return (EBUSY); } static device_method_t rk30_gpio_methods[] = { /* Device interface */ DEVMETHOD(device_probe, rk30_gpio_probe), DEVMETHOD(device_attach, rk30_gpio_attach), DEVMETHOD(device_detach, rk30_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, rk30_gpio_pin_max), DEVMETHOD(gpio_pin_getname, rk30_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, rk30_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, rk30_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, rk30_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, rk30_gpio_pin_get), DEVMETHOD(gpio_pin_set, rk30_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, rk30_gpio_pin_toggle), DEVMETHOD_END }; static devclass_t rk30_gpio_devclass; static driver_t rk30_gpio_driver = { "gpio", rk30_gpio_methods, sizeof(struct rk30_gpio_softc), }; DRIVER_MODULE(rk30_gpio, simplebus, rk30_gpio_driver, rk30_gpio_devclass, 0, 0); int rk30_gpios_prop_handle(phandle_t ctrl, pcell_t *gpios, int len) { struct rk30_gpio_softc *sc; pcell_t gpio_cells; int inc, t, tuples, tuple_size; int dir, flags, pin, i; u_long gpio_ctrl, size; sc = rk30_gpio_sc; if (sc == NULL) return ENXIO; if (OF_getprop(ctrl, "#gpio-cells", &gpio_cells, sizeof(pcell_t)) < 0) return (ENXIO); gpio_cells = fdt32_to_cpu(gpio_cells); if (gpio_cells != 2) return (ENXIO); tuple_size = gpio_cells * sizeof(pcell_t) + sizeof(phandle_t); tuples = len / tuple_size; if (fdt_regsize(ctrl, &gpio_ctrl, &size)) return (ENXIO); /* * Skip controller reference, since controller's phandle is given * explicitly (in a function argument). */ inc = sizeof(ihandle_t) / sizeof(pcell_t); gpios += inc; for (t = 0; t < tuples; t++) { pin = fdt32_to_cpu(gpios[0]); dir = fdt32_to_cpu(gpios[1]); flags = fdt32_to_cpu(gpios[2]); for (i = 0; i < sc->sc_gpio_npins; i++) { if (sc->sc_gpio_pins[i].gp_pin == pin) break; } if (i >= sc->sc_gpio_npins) return (EINVAL); rk30_gpio_pin_configure(sc, &sc->sc_gpio_pins[i], flags); if (dir == 1) { /* Input. */ rk30_gpio_pin_set(sc->sc_dev, pin, GPIO_PIN_INPUT); } else { /* Output. */ rk30_gpio_pin_set(sc->sc_dev, pin, GPIO_PIN_OUTPUT); } gpios += gpio_cells + inc; } return (0); } #define MAX_PINS_PER_NODE 5 #define GPIOS_PROP_CELLS 4 static int rk30_gpio_init(void) { phandle_t child, parent, root, ctrl; pcell_t gpios[MAX_PINS_PER_NODE * GPIOS_PROP_CELLS]; struct gpio_ctrl_entry *e; int len, rv; root = OF_finddevice("/"); len = 0; parent = root; /* Traverse through entire tree to find nodes with 'gpios' prop */ for (child = OF_child(parent); child != 0; child = OF_peer(child)) { /* Find a 'leaf'. Start the search from this node. */ while (OF_child(child)) { parent = child; child = OF_child(child); } if ((len = OF_getproplen(child, "gpios")) > 0) { if (len > sizeof(gpios)) return (ENXIO); /* Get 'gpios' property. */ OF_getprop(child, "gpios", &gpios, len); e = (struct gpio_ctrl_entry *)&gpio_controllers; /* Find and call a handler. */ for (; e->compat; e++) { /* * First cell of 'gpios' property should * contain a ref. to a node defining GPIO * controller. */ ctrl = OF_node_from_xref(fdt32_to_cpu(gpios[0])); if (fdt_is_compatible(ctrl, e->compat)) /* Call a handler. */ if ((rv = e->handler(ctrl, (pcell_t *)&gpios, len))) return (rv); } } if (OF_peer(child) == 0) { /* No more siblings. */ child = parent; parent = OF_parent(child); } } return (0); } diff --git a/sys/arm/samsung/exynos/exynos5_pad.c b/sys/arm/samsung/exynos/exynos5_pad.c index c28ebb1c68e3..0e58bf56299e 100644 --- a/sys/arm/samsung/exynos/exynos5_pad.c +++ b/sys/arm/samsung/exynos/exynos5_pad.c @@ -1,828 +1,839 @@ /*- * Copyright (c) 2014 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Samsung Exynos 5 Pad Control * Chapter 4, Exynos 5 Dual User's Manual Public Rev 1.00 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include #include #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT) #define MAX_PORTS 5 #define MAX_NGPIO 253 #define N_EXT_INTS 16 #define EXYNOS5250 1 #define EXYNOS5420 2 #define PIN_IN 0 #define PIN_OUT 1 #define READ4(_sc, _port, _reg) \ bus_space_read_4(_sc->bst[_port], _sc->bsh[_port], _reg) #define WRITE4(_sc, _port, _reg, _val) \ bus_space_write_4(_sc->bst[_port], _sc->bsh[_port], _reg, _val) /* * GPIO interface */ static int pad_pin_max(device_t, int *); static int pad_pin_getcaps(device_t, uint32_t, uint32_t *); static int pad_pin_getname(device_t, uint32_t, char *); static int pad_pin_getflags(device_t, uint32_t, uint32_t *); static int pad_pin_setflags(device_t, uint32_t, uint32_t); static int pad_pin_set(device_t, uint32_t, unsigned int); static int pad_pin_get(device_t, uint32_t, unsigned int *); static int pad_pin_toggle(device_t, uint32_t pin); struct gpio_bank { char *name; uint32_t port; uint32_t con; uint32_t ngpio; uint32_t ext_con; uint32_t ext_flt_con; uint32_t mask; uint32_t pend; }; struct pad_softc { struct resource *res[MAX_PORTS * 2]; bus_space_tag_t bst[MAX_PORTS]; bus_space_handle_t bsh[MAX_PORTS]; struct mtx sc_mtx; int gpio_npins; struct gpio_pin gpio_pins[MAX_NGPIO]; void *gpio_ih[MAX_PORTS]; device_t dev; int model; struct resource_spec *pad_spec; struct gpio_bank *gpio_map; struct interrupt_entry *interrupt_table; int nports; }; struct pad_softc *gpio_sc; static struct resource_spec pad_spec_5250[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_MEMORY, 1, RF_ACTIVE }, { SYS_RES_MEMORY, 2, RF_ACTIVE }, { SYS_RES_MEMORY, 3, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE }, { SYS_RES_IRQ, 2, RF_ACTIVE }, { SYS_RES_IRQ, 3, RF_ACTIVE }, { -1, 0 } }; static struct resource_spec pad_spec_5420[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_MEMORY, 1, RF_ACTIVE }, { SYS_RES_MEMORY, 2, RF_ACTIVE }, { SYS_RES_MEMORY, 3, RF_ACTIVE }, { SYS_RES_MEMORY, 4, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE }, { SYS_RES_IRQ, 2, RF_ACTIVE }, { SYS_RES_IRQ, 3, RF_ACTIVE }, { SYS_RES_IRQ, 4, RF_ACTIVE }, { -1, 0 } }; static struct ofw_compat_data compat_data[] = { {"samsung,exynos5420-padctrl", EXYNOS5420}, {"samsung,exynos5250-padctrl", EXYNOS5250}, {NULL, 0} }; struct pad_intr { uint32_t enabled; void (*ih) (void *); void *ih_user; }; static struct pad_intr intr_map[MAX_NGPIO]; struct interrupt_entry { int gpio_number; char *combiner_source_name; }; struct interrupt_entry interrupt_table_5250[N_EXT_INTS] = { { 147, "EINT[15]" }, { 146, "EINT[14]" }, { 145, "EINT[13]" }, { 144, "EINT[12]" }, { 143, "EINT[11]" }, { 142, "EINT[10]" }, { 141, "EINT[9]" }, { 140, "EINT[8]" }, { 139, "EINT[7]" }, { 138, "EINT[6]" }, { 137, "EINT[5]" }, { 136, "EINT[4]" }, { 135, "EINT[3]" }, { 134, "EINT[2]" }, { 133, "EINT[1]" }, { 132, "EINT[0]" }, }; struct interrupt_entry interrupt_table_5420[N_EXT_INTS] = { { 23, "EINT[15]" }, { 22, "EINT[14]" }, { 21, "EINT[13]" }, { 20, "EINT[12]" }, { 19, "EINT[11]" }, { 18, "EINT[10]" }, { 17, "EINT[9]" }, { 16, "EINT[8]" }, { 15, "EINT[7]" }, { 14, "EINT[6]" }, { 13, "EINT[5]" }, { 12, "EINT[4]" }, { 11, "EINT[3]" }, { 10, "EINT[2]" }, { 9, "EINT[1]" }, { 8, "EINT[0]" }, }; /* * 253 multi-functional input/output ports */ static struct gpio_bank gpio_map_5250[] = { /* first 132 gpio */ { "gpa0", 0, 0x000, 8, 0x700, 0x800, 0x900, 0xA00 }, { "gpa1", 0, 0x020, 6, 0x704, 0x808, 0x904, 0xA04 }, { "gpa2", 0, 0x040, 8, 0x708, 0x810, 0x908, 0xA08 }, { "gpb0", 0, 0x060, 5, 0x70C, 0x818, 0x90C, 0xA0C }, { "gpb1", 0, 0x080, 5, 0x710, 0x820, 0x910, 0xA10 }, { "gpb2", 0, 0x0A0, 4, 0x714, 0x828, 0x914, 0xA14 }, { "gpb3", 0, 0x0C0, 4, 0x718, 0x830, 0x918, 0xA18 }, { "gpc0", 0, 0x0E0, 7, 0x71C, 0x838, 0x91C, 0xA1C }, { "gpc1", 0, 0x100, 4, 0x720, 0x840, 0x920, 0xA20 }, { "gpc2", 0, 0x120, 7, 0x724, 0x848, 0x924, 0xA24 }, { "gpc3", 0, 0x140, 7, 0x728, 0x850, 0x928, 0xA28 }, { "gpd0", 0, 0x160, 4, 0x72C, 0x858, 0x92C, 0xA2C }, { "gpd1", 0, 0x180, 8, 0x730, 0x860, 0x930, 0xA30 }, { "gpy0", 0, 0x1A0, 6, 0, 0, 0, 0 }, { "gpy1", 0, 0x1C0, 4, 0, 0, 0, 0 }, { "gpy2", 0, 0x1E0, 6, 0, 0, 0, 0 }, { "gpy3", 0, 0x200, 8, 0, 0, 0, 0 }, { "gpy4", 0, 0x220, 8, 0, 0, 0, 0 }, { "gpy5", 0, 0x240, 8, 0, 0, 0, 0 }, { "gpy6", 0, 0x260, 8, 0, 0, 0, 0 }, { "gpc4", 0, 0x2E0, 7, 0x734, 0x868, 0x934, 0xA34 }, /* next 32 */ { "gpx0", 0, 0xC00, 8, 0xE00, 0xE80, 0xF00, 0xF40 }, { "gpx1", 0, 0xC20, 8, 0xE04, 0xE88, 0xF04, 0xF44 }, { "gpx2", 0, 0xC40, 8, 0xE08, 0xE90, 0xF08, 0xF48 }, { "gpx3", 0, 0xC60, 8, 0xE0C, 0xE98, 0xF0C, 0xF4C }, { "gpe0", 1, 0x000, 8, 0x700, 0x800, 0x900, 0xA00 }, { "gpe1", 1, 0x020, 2, 0x704, 0x808, 0x904, 0xA04 }, { "gpf0", 1, 0x040, 4, 0x708, 0x810, 0x908, 0xA08 }, { "gpf1", 1, 0x060, 4, 0x70C, 0x818, 0x90C, 0xA0C }, { "gpg0", 1, 0x080, 8, 0x710, 0x820, 0x910, 0xA10 }, { "gpg1", 1, 0x0A0, 8, 0x714, 0x828, 0x914, 0xA14 }, { "gpg2", 1, 0x0C0, 2, 0x718, 0x830, 0x918, 0xA18 }, { "gph0", 1, 0x0E0, 4, 0x71C, 0x838, 0x91C, 0xA1C }, { "gph1", 1, 0x100, 8, 0x720, 0x840, 0x920, 0xA20 }, { "gpv0", 2, 0x000, 8, 0x700, 0x800, 0x900, 0xA00 }, { "gpv1", 2, 0x020, 8, 0x704, 0x808, 0x904, 0xA04 }, { "gpv2", 2, 0x060, 8, 0x708, 0x810, 0x908, 0xA08 }, { "gpv3", 2, 0x080, 8, 0x70C, 0x818, 0x90C, 0xA0C }, { "gpv4", 2, 0x0C0, 2, 0x710, 0x820, 0x910, 0xA10 }, { "gpz", 3, 0x000, 7, 0x700, 0x800, 0x900, 0xA00 }, { NULL, -1, -1, -1, -1, -1, -1, -1 }, }; static struct gpio_bank gpio_map_5420[] = { /* First 40 */ { "gpy7", 0, 0x000, 8, 0x700, 0x800, 0x900, 0xA00 }, { "gpx0", 0, 0xC00, 8, 0x704, 0xE80, 0xF00, 0xF40 }, { "gpx1", 0, 0xC20, 8, 0x708, 0xE88, 0xF04, 0xF44 }, { "gpx2", 0, 0xC40, 8, 0x70C, 0xE90, 0xF08, 0xF48 }, { "gpx3", 0, 0xC60, 8, 0x710, 0xE98, 0xF0C, 0xF4C }, /* Next 85 */ { "gpc0", 1, 0x000, 8, 0x700, 0x800, 0x900, 0xA00 }, { "gpc1", 1, 0x020, 8, 0x704, 0x808, 0x904, 0xA04 }, { "gpc2", 1, 0x040, 7, 0x708, 0x810, 0x908, 0xA08 }, { "gpc3", 1, 0x060, 4, 0x70C, 0x818, 0x90C, 0xA0C }, { "gpc4", 1, 0x080, 2, 0x710, 0x820, 0x910, 0xA10 }, { "gpd1", 1, 0x0A0, 8, 0x714, 0x828, 0x914, 0xA14 }, { "gpy0", 1, 0x0C0, 6, 0x718, 0x830, 0x918, 0xA18 }, { "gpy1", 1, 0x0E0, 4, 0x71C, 0x838, 0x91C, 0xA1C }, { "gpy2", 1, 0x100, 6, 0x720, 0x840, 0x920, 0xA20 }, { "gpy3", 1, 0x120, 8, 0x724, 0x848, 0x924, 0xA24 }, { "gpy4", 1, 0x140, 8, 0x728, 0x850, 0x928, 0xA28 }, { "gpy5", 1, 0x160, 8, 0x72C, 0x858, 0x92C, 0xA2C }, { "gpy6", 1, 0x180, 8, 0x730, 0x860, 0x930, 0xA30 }, /* Next 46 */ { "gpe0", 2, 0x000, 8, 0x700, 0x800, 0x900, 0xA00 }, { "gpe1", 2, 0x020, 2, 0x704, 0x808, 0x904, 0xA04 }, { "gpf0", 2, 0x040, 6, 0x708, 0x810, 0x908, 0xA08 }, { "gpf1", 2, 0x060, 8, 0x70C, 0x818, 0x90C, 0xA0C }, { "gpg0", 2, 0x080, 8, 0x710, 0x820, 0x910, 0xA10 }, { "gpg1", 2, 0x0A0, 8, 0x714, 0x828, 0x914, 0xA14 }, { "gpg2", 2, 0x0C0, 2, 0x718, 0x830, 0x918, 0xA18 }, { "gpj4", 2, 0x0E0, 4, 0x71C, 0x838, 0x91C, 0xA1C }, /* Next 54 */ { "gpa0", 3, 0x000, 8, 0x700, 0x800, 0x900, 0xA00 }, { "gpa1", 3, 0x020, 6, 0x704, 0x808, 0x904, 0xA04 }, { "gpa2", 3, 0x040, 8, 0x708, 0x810, 0x908, 0xA08 }, { "gpb0", 3, 0x060, 5, 0x70C, 0x818, 0x90C, 0xA0C }, { "gpb1", 3, 0x080, 5, 0x710, 0x820, 0x910, 0xA10 }, { "gpb2", 3, 0x0A0, 4, 0x714, 0x828, 0x914, 0xA14 }, { "gpb3", 3, 0x0C0, 8, 0x718, 0x830, 0x918, 0xA18 }, { "gpb4", 3, 0x0E0, 2, 0x71C, 0x838, 0x91C, 0xA1C }, { "gph0", 3, 0x100, 8, 0x720, 0x840, 0x920, 0xA20 }, /* Last 7 */ { "gpz", 4, 0x000, 7, 0x700, 0x800, 0x900, 0xA00 }, { NULL, -1, -1, -1, -1, -1, -1, -1 }, }; static int get_bank(struct pad_softc *sc, int gpio_number, struct gpio_bank *bank, int *pin_shift) { int ngpio; int i; int n; n = 0; for (i = 0; sc->gpio_map[i].ngpio != -1; i++) { ngpio = sc->gpio_map[i].ngpio; if ((n + ngpio) > gpio_number) { *bank = sc->gpio_map[i]; *pin_shift = (gpio_number - n); return (0); }; n += ngpio; }; return (-1); } static int port_intr(void *arg) { struct port_softc *sc; sc = arg; return (FILTER_HANDLED); } static void ext_intr(void *arg) { struct pad_softc *sc; void (*ih) (void *); void *ih_user; int ngpio; int found; int reg; int i,j; int n,k; sc = arg; n = 0; for (i = 0; sc->gpio_map[i].ngpio != -1; i++) { found = 0; ngpio = sc->gpio_map[i].ngpio; if (sc->gpio_map[i].pend == 0) { n += ngpio; continue; } reg = READ4(sc, sc->gpio_map[i].port, sc->gpio_map[i].pend); for (j = 0; j < ngpio; j++) { if (reg & (1 << j)) { found = 1; k = (n + j); if (intr_map[k].enabled == 1) { ih = intr_map[k].ih; ih_user = intr_map[k].ih_user; ih(ih_user); } } } if (found) { /* ACK */ WRITE4(sc, sc->gpio_map[i].port, sc->gpio_map[i].pend, reg); } n += ngpio; } } int pad_setup_intr(int gpio_number, void (*ih)(void *), void *ih_user) { struct interrupt_entry *entry; struct pad_intr *pad_irq; struct gpio_bank bank; struct pad_softc *sc; int pin_shift; int reg; int i; sc = gpio_sc; if (sc == NULL) { device_printf(sc->dev, "Error: pad is not attached\n"); return (-1); } if (get_bank(sc, gpio_number, &bank, &pin_shift) != 0) return (-1); entry = NULL; for (i = 0; i < N_EXT_INTS; i++) if (sc->interrupt_table[i].gpio_number == gpio_number) entry = &(sc->interrupt_table[i]); if (entry == NULL) { device_printf(sc->dev, "Cant find interrupt source for %d\n", gpio_number); return (-1); } #if 0 printf("Request interrupt name %s\n", entry->combiner_source_name); #endif pad_irq = &intr_map[gpio_number]; pad_irq->enabled = 1; pad_irq->ih = ih; pad_irq->ih_user = ih_user; /* Setup port as external interrupt source */ reg = READ4(sc, bank.port, bank.con); reg |= (0xf << (pin_shift * 4)); #if 0 printf("writing 0x%08x to 0x%08x\n", reg, bank.con); #endif WRITE4(sc, bank.port, bank.con, reg); /* * Configure interrupt pin * * 0x0 = Sets Low level * 0x1 = Sets High level * 0x2 = Triggers Falling edge * 0x3 = Triggers Rising edge * 0x4 = Triggers Both edge * * TODO: add parameter. For now configure as 0x0 */ reg = READ4(sc, bank.port, bank.ext_con); reg &= ~(0x7 << (pin_shift * 4)); WRITE4(sc, bank.port, bank.ext_con, reg); /* Unmask */ reg = READ4(sc, bank.port, bank.mask); reg &= ~(1 << pin_shift); WRITE4(sc, bank.port, bank.mask, reg); combiner_setup_intr(entry->combiner_source_name, ext_intr, sc); return (0); } static int pad_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data != 0) { device_set_desc(dev, "Exynos Pad Control"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int pad_attach(device_t dev) { struct gpio_bank bank; struct pad_softc *sc; int pin_shift; int reg; int i; sc = device_get_softc(dev); mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF); sc->model = ofw_bus_search_compatible(dev, compat_data)->ocd_data; switch (sc->model) { case EXYNOS5250: sc->pad_spec = pad_spec_5250; sc->gpio_map = gpio_map_5250; sc->interrupt_table = interrupt_table_5250; sc->gpio_npins = 253; sc->nports = 4; break; case EXYNOS5420: sc->pad_spec = pad_spec_5420; sc->gpio_map = gpio_map_5420; sc->interrupt_table = interrupt_table_5420; sc->gpio_npins = 232; sc->nports = 5; break; default: - return (-1); + goto fail; }; if (bus_alloc_resources(dev, sc->pad_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); - return (ENXIO); + goto fail; } /* Memory interface */ for (i = 0; i < sc->nports; i++) { sc->bst[i] = rman_get_bustag(sc->res[i]); sc->bsh[i] = rman_get_bushandle(sc->res[i]); }; sc->dev = dev; gpio_sc = sc; for (i = 0; i < sc->nports; i++) { if ((bus_setup_intr(dev, sc->res[sc->nports + i], INTR_TYPE_BIO | INTR_MPSAFE, port_intr, NULL, sc, &sc->gpio_ih[i]))) { device_printf(dev, "ERROR: Unable to register interrupt handler\n"); - return (ENXIO); + goto fail; } - }; + } for (i = 0; i < sc->gpio_npins; i++) { sc->gpio_pins[i].gp_pin = i; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; if (get_bank(sc, i, &bank, &pin_shift) != 0) continue; pin_shift *= 4; reg = READ4(sc, bank.port, bank.con); if (reg & (PIN_OUT << pin_shift)) sc->gpio_pins[i].gp_flags = GPIO_PIN_OUTPUT; else sc->gpio_pins[i].gp_flags = GPIO_PIN_INPUT; /* TODO: add other pin statuses */ snprintf(sc->gpio_pins[i].gp_name, GPIOMAXNAME, "pad%d.%d", device_get_unit(dev), i); } device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); + +fail: + for (i = 0; i < sc->nports; i++) { + if (sc->gpio_ih[i]) + bus_teardown_intr(dev, sc->res[sc->nports + i], + sc->gpio_ih[i]); + } + bus_release_resources(dev, sc->pad_spec, sc->res); + mtx_destroy(&sc->sc_mtx); + + return (ENXIO); } static int pad_pin_max(device_t dev, int *maxpin) { struct pad_softc *sc; sc = device_get_softc(dev); *maxpin = sc->gpio_npins - 1; return (0); } static int pad_pin_getname(device_t dev, uint32_t pin, char *name) { struct pad_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME); GPIO_UNLOCK(sc); return (0); } static int pad_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct pad_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *caps = sc->gpio_pins[i].gp_caps; GPIO_UNLOCK(sc); return (0); } static int pad_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct pad_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *flags = sc->gpio_pins[i].gp_flags; GPIO_UNLOCK(sc); return (0); } static int pad_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct gpio_bank bank; struct pad_softc *sc; int pin_shift; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); if (get_bank(sc, pin, &bank, &pin_shift) != 0) return (EINVAL); GPIO_LOCK(sc); if (READ4(sc, bank.port, bank.con + 0x4) & (1 << pin_shift)) *val = 1; else *val = 0; GPIO_UNLOCK(sc); return (0); } static int pad_pin_toggle(device_t dev, uint32_t pin) { struct gpio_bank bank; struct pad_softc *sc; int pin_shift; int reg; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); if (get_bank(sc, pin, &bank, &pin_shift) != 0) return (EINVAL); GPIO_LOCK(sc); reg = READ4(sc, bank.port, bank.con + 0x4); if (reg & (1 << pin_shift)) reg &= ~(1 << pin_shift); else reg |= (1 << pin_shift); WRITE4(sc, bank.port, bank.con + 0x4, reg); GPIO_UNLOCK(sc); return (0); } static void pad_pin_configure(struct pad_softc *sc, struct gpio_pin *pin, unsigned int flags) { struct gpio_bank bank; int pin_shift; int reg; GPIO_LOCK(sc); /* * Manage input/output */ if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (get_bank(sc, pin->gp_pin, &bank, &pin_shift) != 0) return; pin_shift *= 4; #if 0 printf("bank is 0x%08x pin_shift %d\n", bank.con, pin_shift); #endif if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; reg = READ4(sc, bank.port, bank.con); reg &= ~(0xf << pin_shift); reg |= (PIN_OUT << pin_shift); WRITE4(sc, bank.port, bank.con, reg); } else { pin->gp_flags |= GPIO_PIN_INPUT; reg = READ4(sc, bank.port, bank.con); reg &= ~(0xf << pin_shift); WRITE4(sc, bank.port, bank.con, reg); } } GPIO_UNLOCK(sc); } static int pad_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct pad_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); pad_pin_configure(sc, &sc->gpio_pins[i], flags); return (0); } static int pad_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct pad_softc *sc; struct gpio_bank bank; int pin_shift; int reg; int i; sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); if (get_bank(sc, pin, &bank, &pin_shift) != 0) return (EINVAL); GPIO_LOCK(sc); reg = READ4(sc, bank.port, bank.con + 0x4); reg &= ~(PIN_OUT << pin_shift); if (value) reg |= (PIN_OUT << pin_shift); WRITE4(sc, bank.port, bank.con + 0x4, reg); GPIO_UNLOCK(sc); return (0); } static device_method_t pad_methods[] = { DEVMETHOD(device_probe, pad_probe), DEVMETHOD(device_attach, pad_attach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, pad_pin_max), DEVMETHOD(gpio_pin_getname, pad_pin_getname), DEVMETHOD(gpio_pin_getcaps, pad_pin_getcaps), DEVMETHOD(gpio_pin_getflags, pad_pin_getflags), DEVMETHOD(gpio_pin_get, pad_pin_get), DEVMETHOD(gpio_pin_toggle, pad_pin_toggle), DEVMETHOD(gpio_pin_setflags, pad_pin_setflags), DEVMETHOD(gpio_pin_set, pad_pin_set), { 0, 0 } }; static driver_t pad_driver = { "gpio", pad_methods, sizeof(struct pad_softc), }; static devclass_t pad_devclass; DRIVER_MODULE(pad, simplebus, pad_driver, pad_devclass, 0, 0); diff --git a/sys/arm/ti/ti_gpio.c b/sys/arm/ti/ti_gpio.c index ed24958f63ec..8874ff35046e 100644 --- a/sys/arm/ti/ti_gpio.c +++ b/sys/arm/ti/ti_gpio.c @@ -1,1098 +1,1094 @@ /*- * Copyright (c) 2011 Ben Gray . * Copyright (c) 2014 Luiz Otavio O Souza . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /** * Beware that the OMAP4 datasheet(s) lists GPIO banks 1-6, whereas the code * here uses 0-5. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "ti_gpio_if.h" #if !defined(SOC_OMAP4) && !defined(SOC_TI_AM335X) #error "Unknown SoC" #endif /* Register definitions */ #define TI_GPIO_REVISION 0x0000 #define TI_GPIO_SYSCONFIG 0x0010 #define TI_GPIO_IRQSTATUS_RAW_0 0x0024 #define TI_GPIO_IRQSTATUS_RAW_1 0x0028 #define TI_GPIO_IRQSTATUS_0 0x002C #define TI_GPIO_IRQSTATUS_1 0x0030 #define TI_GPIO_IRQSTATUS_SET_0 0x0034 #define TI_GPIO_IRQSTATUS_SET_1 0x0038 #define TI_GPIO_IRQSTATUS_CLR_0 0x003C #define TI_GPIO_IRQSTATUS_CLR_1 0x0040 #define TI_GPIO_IRQWAKEN_0 0x0044 #define TI_GPIO_IRQWAKEN_1 0x0048 #define TI_GPIO_SYSSTATUS 0x0114 #define TI_GPIO_IRQSTATUS1 0x0118 #define TI_GPIO_IRQENABLE1 0x011C #define TI_GPIO_WAKEUPENABLE 0x0120 #define TI_GPIO_IRQSTATUS2 0x0128 #define TI_GPIO_IRQENABLE2 0x012C #define TI_GPIO_CTRL 0x0130 #define TI_GPIO_OE 0x0134 #define TI_GPIO_DATAIN 0x0138 #define TI_GPIO_DATAOUT 0x013C #define TI_GPIO_LEVELDETECT0 0x0140 #define TI_GPIO_LEVELDETECT1 0x0144 #define TI_GPIO_RISINGDETECT 0x0148 #define TI_GPIO_FALLINGDETECT 0x014C #define TI_GPIO_DEBOUNCENABLE 0x0150 #define TI_GPIO_DEBOUNCINGTIME 0x0154 #define TI_GPIO_CLEARWKUPENA 0x0180 #define TI_GPIO_SETWKUENA 0x0184 #define TI_GPIO_CLEARDATAOUT 0x0190 #define TI_GPIO_SETDATAOUT 0x0194 /* Other SoC Specific definitions */ #define OMAP4_MAX_GPIO_BANKS 6 #define OMAP4_FIRST_GPIO_BANK 1 #define OMAP4_INTR_PER_BANK 1 #define OMAP4_GPIO_REV 0x50600801 #define AM335X_MAX_GPIO_BANKS 4 #define AM335X_FIRST_GPIO_BANK 0 #define AM335X_INTR_PER_BANK 2 #define AM335X_GPIO_REV 0x50600801 #define PINS_PER_BANK 32 #define TI_GPIO_BANK(p) ((p) / PINS_PER_BANK) #define TI_GPIO_MASK(p) (1U << ((p) % PINS_PER_BANK)) static struct ti_gpio_softc *ti_gpio_sc = NULL; +static int ti_gpio_detach(device_t); static u_int ti_max_gpio_banks(void) { switch(ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: return (OMAP4_MAX_GPIO_BANKS); #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: return (AM335X_MAX_GPIO_BANKS); #endif } return (0); } static u_int ti_max_gpio_intrs(void) { switch(ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: return (OMAP4_MAX_GPIO_BANKS * OMAP4_INTR_PER_BANK); #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: return (AM335X_MAX_GPIO_BANKS * AM335X_INTR_PER_BANK); #endif } return (0); } static u_int ti_first_gpio_bank(void) { switch(ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: return (OMAP4_FIRST_GPIO_BANK); #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: return (AM335X_FIRST_GPIO_BANK); #endif } return (0); } static uint32_t ti_gpio_rev(void) { switch(ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: return (OMAP4_GPIO_REV); #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: return (AM335X_GPIO_REV); #endif } return (0); } /** * ti_gpio_mem_spec - Resource specification used when allocating resources * ti_gpio_irq_spec - Resource specification used when allocating resources * * This driver module can have up to six independent memory regions, each * region typically controls 32 GPIO pins. * * On OMAP3 and OMAP4 there is only one physical interrupt line per bank, * but there are two set of registers which control the interrupt delivery * to internal subsystems. The first set of registers control the * interrupts delivery to the MPU and the second set control the * interrupts delivery to the DSP. * * On AM335x there are two physical interrupt lines for each GPIO module. * Each interrupt line is controlled by a set of registers. */ static struct resource_spec ti_gpio_mem_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_MEMORY, 1, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_MEMORY, 2, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_MEMORY, 3, RF_ACTIVE | RF_OPTIONAL }, #if !defined(SOC_TI_AM335X) { SYS_RES_MEMORY, 4, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_MEMORY, 5, RF_ACTIVE | RF_OPTIONAL }, #endif { -1, 0, 0 } }; static struct resource_spec ti_gpio_irq_spec[] = { { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 4, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 5, RF_ACTIVE | RF_OPTIONAL }, #if defined(SOC_TI_AM335X) { SYS_RES_IRQ, 6, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 7, RF_ACTIVE | RF_OPTIONAL }, #endif { -1, 0, 0 } }; /** * Macros for driver mutex locking */ #define TI_GPIO_LOCK(_sc) mtx_lock_spin(&(_sc)->sc_mtx) #define TI_GPIO_UNLOCK(_sc) mtx_unlock_spin(&(_sc)->sc_mtx) #define TI_GPIO_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit((_sc)->sc_dev), \ "ti_gpio", MTX_SPIN) #define TI_GPIO_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx) #define TI_GPIO_ASSERT_LOCKED(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED) #define TI_GPIO_ASSERT_UNLOCKED(_sc) mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED) /** * ti_gpio_read_4 - reads a 32-bit value from one of the GPIO registers * @sc: GPIO device context * @bank: The bank to read from * @off: The offset of a register from the GPIO register address range * * * RETURNS: * 32-bit value read from the register. */ static inline uint32_t ti_gpio_read_4(struct ti_gpio_softc *sc, unsigned int bank, bus_size_t off) { return (bus_read_4(sc->sc_mem_res[bank], off)); } /** * ti_gpio_write_4 - writes a 32-bit value to one of the GPIO registers * @sc: GPIO device context * @bank: The bank to write to * @off: The offset of a register from the GPIO register address range * @val: The value to write into the register * * RETURNS: * nothing */ static inline void ti_gpio_write_4(struct ti_gpio_softc *sc, unsigned int bank, bus_size_t off, uint32_t val) { bus_write_4(sc->sc_mem_res[bank], off, val); } static inline void ti_gpio_intr_clr(struct ti_gpio_softc *sc, unsigned int bank, uint32_t mask) { /* We clear both set of registers. */ ti_gpio_write_4(sc, bank, TI_GPIO_IRQSTATUS_CLR_0, mask); ti_gpio_write_4(sc, bank, TI_GPIO_IRQSTATUS_CLR_1, mask); } static inline void ti_gpio_intr_set(struct ti_gpio_softc *sc, unsigned int bank, uint32_t mask) { /* * On OMAP4 we unmask only the MPU interrupt and on AM335x we * also activate only the first interrupt. */ ti_gpio_write_4(sc, bank, TI_GPIO_IRQSTATUS_SET_0, mask); } static inline void ti_gpio_intr_ack(struct ti_gpio_softc *sc, unsigned int bank, uint32_t mask) { /* * Acknowledge the interrupt on both registers even if we use only * the first one. */ ti_gpio_write_4(sc, bank, TI_GPIO_IRQSTATUS_0, mask); ti_gpio_write_4(sc, bank, TI_GPIO_IRQSTATUS_1, mask); } static inline uint32_t ti_gpio_intr_status(struct ti_gpio_softc *sc, unsigned int bank) { uint32_t reg; /* Get the status from both registers. */ reg = ti_gpio_read_4(sc, bank, TI_GPIO_IRQSTATUS_0); reg |= ti_gpio_read_4(sc, bank, TI_GPIO_IRQSTATUS_1); return (reg); } /** * ti_gpio_pin_max - Returns the maximum number of GPIO pins * @dev: gpio device handle * @maxpin: pointer to a value that upon return will contain the maximum number * of pins in the device. * * * LOCKING: * No locking required, returns static data. * * RETURNS: * Returns 0 on success otherwise an error code */ static int ti_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = ti_max_gpio_banks() * PINS_PER_BANK - 1; return (0); } static int ti_gpio_valid_pin(struct ti_gpio_softc *sc, int pin) { if (pin >= sc->sc_maxpin || TI_GPIO_BANK(pin) >= ti_max_gpio_banks() || sc->sc_mem_res[TI_GPIO_BANK(pin)] == NULL) { return (EINVAL); } return (0); } /** * ti_gpio_pin_getcaps - Gets the capabilties of a given pin * @dev: gpio device handle * @pin: the number of the pin * @caps: pointer to a value that upon return will contain the capabilities * * Currently all pins have the same capability, notably: * - GPIO_PIN_INPUT * - GPIO_PIN_OUTPUT * - GPIO_PIN_PULLUP * - GPIO_PIN_PULLDOWN * * LOCKING: * No locking required, returns static data. * * RETURNS: * Returns 0 on success otherwise an error code */ static int ti_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct ti_gpio_softc *sc; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, pin) != 0) return (EINVAL); *caps = (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | GPIO_PIN_PULLUP | GPIO_PIN_PULLDOWN); return (0); } /** * ti_gpio_pin_getflags - Gets the current flags of a given pin * @dev: gpio device handle * @pin: the number of the pin * @flags: upon return will contain the current flags of the pin * * Reads the current flags of a given pin, here we actually read the H/W * registers to determine the flags, rather than storing the value in the * setflags call. * * LOCKING: * Internally locks the context * * RETURNS: * Returns 0 on success otherwise an error code */ static int ti_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct ti_gpio_softc *sc; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, pin) != 0) return (EINVAL); /* Get the current pin state */ TI_GPIO_LOCK(sc); TI_GPIO_GET_FLAGS(dev, pin, flags); TI_GPIO_UNLOCK(sc); return (0); } /** * ti_gpio_pin_getname - Gets the name of a given pin * @dev: gpio device handle * @pin: the number of the pin * @name: buffer to put the name in * * The driver simply calls the pins gpio_n, where 'n' is obviously the number * of the pin. * * LOCKING: * No locking required, returns static data. * * RETURNS: * Returns 0 on success otherwise an error code */ static int ti_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct ti_gpio_softc *sc; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, pin) != 0) return (EINVAL); /* Set a very simple name */ snprintf(name, GPIOMAXNAME, "gpio_%u", pin); name[GPIOMAXNAME - 1] = '\0'; return (0); } /** * ti_gpio_pin_setflags - Sets the flags for a given pin * @dev: gpio device handle * @pin: the number of the pin * @flags: the flags to set * * The flags of the pin correspond to things like input/output mode, pull-ups, * pull-downs, etc. This driver doesn't support all flags, only the following: * - GPIO_PIN_INPUT * - GPIO_PIN_OUTPUT * - GPIO_PIN_PULLUP * - GPIO_PIN_PULLDOWN * * LOCKING: * Internally locks the context * * RETURNS: * Returns 0 on success otherwise an error code */ static int ti_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct ti_gpio_softc *sc; uint32_t oe; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, pin) != 0) return (EINVAL); /* Set the GPIO mode and state */ TI_GPIO_LOCK(sc); if (TI_GPIO_SET_FLAGS(dev, pin, flags) != 0) { TI_GPIO_UNLOCK(sc); return (EINVAL); } /* If configuring as an output set the "output enable" bit */ oe = ti_gpio_read_4(sc, TI_GPIO_BANK(pin), TI_GPIO_OE); if (flags & GPIO_PIN_INPUT) oe |= TI_GPIO_MASK(pin); else oe &= ~TI_GPIO_MASK(pin); ti_gpio_write_4(sc, TI_GPIO_BANK(pin), TI_GPIO_OE, oe); TI_GPIO_UNLOCK(sc); return (0); } /** * ti_gpio_pin_set - Sets the current level on a GPIO pin * @dev: gpio device handle * @pin: the number of the pin * @value: non-zero value will drive the pin high, otherwise the pin is * driven low. * * * LOCKING: * Internally locks the context * * RETURNS: * Returns 0 on success otherwise a error code */ static int ti_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct ti_gpio_softc *sc; uint32_t reg; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, pin) != 0) return (EINVAL); TI_GPIO_LOCK(sc); if (value == GPIO_PIN_LOW) reg = TI_GPIO_CLEARDATAOUT; else reg = TI_GPIO_SETDATAOUT; ti_gpio_write_4(sc, TI_GPIO_BANK(pin), reg, TI_GPIO_MASK(pin)); TI_GPIO_UNLOCK(sc); return (0); } /** * ti_gpio_pin_get - Gets the current level on a GPIO pin * @dev: gpio device handle * @pin: the number of the pin * @value: pointer to a value that upond return will contain the pin value * * The pin must be configured as an input pin beforehand, otherwise this * function will fail. * * LOCKING: * Internally locks the context * * RETURNS: * Returns 0 on success otherwise a error code */ static int ti_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *value) { struct ti_gpio_softc *sc; uint32_t oe, reg, val; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, pin) != 0) return (EINVAL); /* * Return data from output latch when set as output and from the * input register otherwise. */ TI_GPIO_LOCK(sc); oe = ti_gpio_read_4(sc, TI_GPIO_BANK(pin), TI_GPIO_OE); if (oe & TI_GPIO_MASK(pin)) reg = TI_GPIO_DATAIN; else reg = TI_GPIO_DATAOUT; val = ti_gpio_read_4(sc, TI_GPIO_BANK(pin), reg); *value = (val & TI_GPIO_MASK(pin)) ? 1 : 0; TI_GPIO_UNLOCK(sc); return (0); } /** * ti_gpio_pin_toggle - Toggles a given GPIO pin * @dev: gpio device handle * @pin: the number of the pin * * * LOCKING: * Internally locks the context * * RETURNS: * Returns 0 on success otherwise a error code */ static int ti_gpio_pin_toggle(device_t dev, uint32_t pin) { struct ti_gpio_softc *sc; uint32_t reg, val; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, pin) != 0) return (EINVAL); /* Toggle the pin */ TI_GPIO_LOCK(sc); val = ti_gpio_read_4(sc, TI_GPIO_BANK(pin), TI_GPIO_DATAOUT); if (val & TI_GPIO_MASK(pin)) reg = TI_GPIO_CLEARDATAOUT; else reg = TI_GPIO_SETDATAOUT; ti_gpio_write_4(sc, TI_GPIO_BANK(pin), reg, TI_GPIO_MASK(pin)); TI_GPIO_UNLOCK(sc); return (0); } /** * ti_gpio_intr - ISR for all GPIO modules * @arg: the soft context pointer * * LOCKING: * Internally locks the context * */ static int ti_gpio_intr(void *arg) { int bank_last, irq; struct intr_event *event; struct ti_gpio_softc *sc; uint32_t reg; sc = (struct ti_gpio_softc *)arg; bank_last = -1; reg = 0; /* squelch bogus gcc warning */ for (irq = 0; irq < sc->sc_maxpin; irq++) { /* Read interrupt status only once for each bank. */ if (TI_GPIO_BANK(irq) != bank_last) { reg = ti_gpio_intr_status(sc, TI_GPIO_BANK(irq)); bank_last = TI_GPIO_BANK(irq); } if ((reg & TI_GPIO_MASK(irq)) == 0) continue; event = sc->sc_events[irq]; if (event != NULL && !TAILQ_EMPTY(&event->ie_handlers)) intr_event_handle(event, NULL); else device_printf(sc->sc_dev, "Stray IRQ %d\n", irq); /* Ack the IRQ Status bit. */ ti_gpio_intr_ack(sc, TI_GPIO_BANK(irq), TI_GPIO_MASK(irq)); } return (FILTER_HANDLED); } static int ti_gpio_attach_intr(device_t dev) { int i; struct ti_gpio_softc *sc; sc = device_get_softc(dev); for (i = 0; i < ti_max_gpio_intrs(); i++) { if (sc->sc_irq_res[i] == NULL) break; /* * Register our interrupt filter for each of the IRQ resources. */ if (bus_setup_intr(dev, sc->sc_irq_res[i], INTR_TYPE_MISC | INTR_MPSAFE, ti_gpio_intr, NULL, sc, &sc->sc_irq_hdl[i]) != 0) { device_printf(dev, "WARNING: unable to register interrupt filter\n"); return (-1); } } return (0); } static int ti_gpio_detach_intr(device_t dev) { int i; struct ti_gpio_softc *sc; /* Teardown our interrupt filters. */ sc = device_get_softc(dev); for (i = 0; i < ti_max_gpio_intrs(); i++) { if (sc->sc_irq_res[i] == NULL) break; if (sc->sc_irq_hdl[i]) { bus_teardown_intr(dev, sc->sc_irq_res[i], sc->sc_irq_hdl[i]); } } return (0); } static int ti_gpio_bank_init(device_t dev, int bank) { int pin; struct ti_gpio_softc *sc; uint32_t flags, reg_oe, rev; sc = device_get_softc(dev); /* Enable the interface and functional clocks for the module. */ ti_prcm_clk_enable(GPIO0_CLK + ti_first_gpio_bank() + bank); /* * Read the revision number of the module. TI don't publish the * actual revision numbers, so instead the values have been * determined by experimentation. */ rev = ti_gpio_read_4(sc, bank, TI_GPIO_REVISION); /* Check the revision. */ if (rev != ti_gpio_rev()) { device_printf(dev, "Warning: could not determine the revision " "of GPIO module %d (revision:0x%08x)\n", bank, rev); return (EINVAL); } /* Disable interrupts for all pins. */ ti_gpio_intr_clr(sc, bank, 0xffffffff); /* Init OE register based on pads configuration. */ reg_oe = 0xffffffff; for (pin = 0; pin < PINS_PER_BANK; pin++) { TI_GPIO_GET_FLAGS(dev, PINS_PER_BANK * bank + pin, &flags); if (flags & GPIO_PIN_OUTPUT) reg_oe &= ~(1UL << pin); } ti_gpio_write_4(sc, bank, TI_GPIO_OE, reg_oe); return (0); } /** * ti_gpio_attach - attach function for the driver * @dev: gpio device handle * * Allocates and sets up the driver context for all GPIO banks. This function * expects the memory ranges and IRQs to already be allocated to the driver. * * LOCKING: * None * * RETURNS: * Always returns 0 */ static int ti_gpio_attach(device_t dev) { struct ti_gpio_softc *sc; unsigned int i; int err; if (ti_gpio_sc != NULL) return (ENXIO); ti_gpio_sc = sc = device_get_softc(dev); sc->sc_dev = dev; TI_GPIO_LOCK_INIT(sc); ti_gpio_pin_max(dev, &sc->sc_maxpin); sc->sc_maxpin++; /* There are up to 6 different GPIO register sets located in different * memory areas on the chip. The memory range should have been set for * the driver when it was added as a child. */ if (bus_alloc_resources(dev, ti_gpio_mem_spec, sc->sc_mem_res) != 0) { device_printf(dev, "Error: could not allocate mem resources\n"); + ti_gpio_detach(dev); return (ENXIO); } /* Request the IRQ resources */ if (bus_alloc_resources(dev, ti_gpio_irq_spec, sc->sc_irq_res) != 0) { - bus_release_resources(dev, ti_gpio_mem_spec, sc->sc_mem_res); device_printf(dev, "Error: could not allocate irq resources\n"); + ti_gpio_detach(dev); return (ENXIO); } /* Setup the IRQ resources */ if (ti_gpio_attach_intr(dev) != 0) { - ti_gpio_detach_intr(dev); - bus_release_resources(dev, ti_gpio_irq_spec, sc->sc_irq_res); - bus_release_resources(dev, ti_gpio_mem_spec, sc->sc_mem_res); + device_printf(dev, "Error: could not setup irq handlers\n"); + ti_gpio_detach(dev); return (ENXIO); } /* * Initialize the interrupt settings. The default is active-low * interrupts. */ sc->sc_irq_trigger = malloc( sizeof(*sc->sc_irq_trigger) * sc->sc_maxpin, M_DEVBUF, M_WAITOK | M_ZERO); sc->sc_irq_polarity = malloc( sizeof(*sc->sc_irq_polarity) * sc->sc_maxpin, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < sc->sc_maxpin; i++) { sc->sc_irq_trigger[i] = INTR_TRIGGER_LEVEL; sc->sc_irq_polarity[i] = INTR_POLARITY_LOW; } sc->sc_events = malloc(sizeof(struct intr_event *) * sc->sc_maxpin, M_DEVBUF, M_WAITOK | M_ZERO); /* We need to go through each block and ensure the clocks are running and * the module is enabled. It might be better to do this only when the * pins are configured which would result in less power used if the GPIO * pins weren't used ... */ for (i = 0; i < ti_max_gpio_banks(); i++) { if (sc->sc_mem_res[i] != NULL) { /* Initialize the GPIO module. */ err = ti_gpio_bank_init(dev, i); if (err != 0) { - ti_gpio_detach_intr(dev); - bus_release_resources(dev, ti_gpio_irq_spec, - sc->sc_irq_res); - bus_release_resources(dev, ti_gpio_mem_spec, - sc->sc_mem_res); + ti_gpio_detach(dev); return (err); } } } /* Finish of the probe call */ device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); } /** * ti_gpio_detach - detach function for the driver * @dev: scm device handle * * Allocates and sets up the driver context, this simply entails creating a * bus mappings for the SCM register set. * * LOCKING: * None * * RETURNS: * Always returns 0 */ static int ti_gpio_detach(device_t dev) { struct ti_gpio_softc *sc = device_get_softc(dev); unsigned int i; KASSERT(mtx_initialized(&sc->sc_mtx), ("gpio mutex not initialized")); /* Disable all interrupts */ for (i = 0; i < ti_max_gpio_banks(); i++) { if (sc->sc_mem_res[i] != NULL) ti_gpio_intr_clr(sc, i, 0xffffffff); } - bus_generic_detach(dev); - - free(sc->sc_events, M_DEVBUF); - free(sc->sc_irq_polarity, M_DEVBUF); - free(sc->sc_irq_trigger, M_DEVBUF); - + if (sc->sc_events) + free(sc->sc_events, M_DEVBUF); + if (sc->sc_irq_polarity) + free(sc->sc_irq_polarity, M_DEVBUF); + if (sc->sc_irq_trigger) + free(sc->sc_irq_trigger, M_DEVBUF); /* Release the memory and IRQ resources. */ ti_gpio_detach_intr(dev); bus_release_resources(dev, ti_gpio_irq_spec, sc->sc_irq_res); bus_release_resources(dev, ti_gpio_mem_spec, sc->sc_mem_res); - TI_GPIO_LOCK_DESTROY(sc); return (0); } static uint32_t ti_gpio_intr_reg(struct ti_gpio_softc *sc, int irq) { if (ti_gpio_valid_pin(sc, irq) != 0) return (0); if (sc->sc_irq_trigger[irq] == INTR_TRIGGER_LEVEL) { if (sc->sc_irq_polarity[irq] == INTR_POLARITY_LOW) return (TI_GPIO_LEVELDETECT0); else if (sc->sc_irq_polarity[irq] == INTR_POLARITY_HIGH) return (TI_GPIO_LEVELDETECT1); } else if (sc->sc_irq_trigger[irq] == INTR_TRIGGER_EDGE) { if (sc->sc_irq_polarity[irq] == INTR_POLARITY_LOW) return (TI_GPIO_FALLINGDETECT); else if (sc->sc_irq_polarity[irq] == INTR_POLARITY_HIGH) return (TI_GPIO_RISINGDETECT); } return (0); } static void ti_gpio_mask_irq(void *source) { int irq; uint32_t reg, val; irq = (int)source; if (ti_gpio_valid_pin(ti_gpio_sc, irq) != 0) return; TI_GPIO_LOCK(ti_gpio_sc); ti_gpio_intr_clr(ti_gpio_sc, TI_GPIO_BANK(irq), TI_GPIO_MASK(irq)); reg = ti_gpio_intr_reg(ti_gpio_sc, irq); if (reg != 0) { val = ti_gpio_read_4(ti_gpio_sc, TI_GPIO_BANK(irq), reg); val &= ~TI_GPIO_MASK(irq); ti_gpio_write_4(ti_gpio_sc, TI_GPIO_BANK(irq), reg, val); } TI_GPIO_UNLOCK(ti_gpio_sc); } static void ti_gpio_unmask_irq(void *source) { int irq; uint32_t reg, val; irq = (int)source; if (ti_gpio_valid_pin(ti_gpio_sc, irq) != 0) return; TI_GPIO_LOCK(ti_gpio_sc); reg = ti_gpio_intr_reg(ti_gpio_sc, irq); if (reg != 0) { val = ti_gpio_read_4(ti_gpio_sc, TI_GPIO_BANK(irq), reg); val |= TI_GPIO_MASK(irq); ti_gpio_write_4(ti_gpio_sc, TI_GPIO_BANK(irq), reg, val); ti_gpio_intr_set(ti_gpio_sc, TI_GPIO_BANK(irq), TI_GPIO_MASK(irq)); } TI_GPIO_UNLOCK(ti_gpio_sc); } static int ti_gpio_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *res) { int pin; if (type != SYS_RES_IRQ) return (ENXIO); /* Unmask the interrupt. */ pin = rman_get_start(res); ti_gpio_unmask_irq((void *)(uintptr_t)pin); return (0); } static int ti_gpio_deactivate_resource(device_t dev, device_t child, int type, int rid, struct resource *res) { int pin; if (type != SYS_RES_IRQ) return (ENXIO); /* Mask the interrupt. */ pin = rman_get_start(res); ti_gpio_mask_irq((void *)(uintptr_t)pin); return (0); } static int ti_gpio_config_intr(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { struct ti_gpio_softc *sc; uint32_t oldreg, reg, val; sc = device_get_softc(dev); if (ti_gpio_valid_pin(sc, irq) != 0) return (EINVAL); /* There is no standard trigger or polarity. */ if (trig == INTR_TRIGGER_CONFORM || pol == INTR_POLARITY_CONFORM) return (EINVAL); TI_GPIO_LOCK(sc); /* * TRM recommends add the new event before remove the old one to * avoid losing interrupts. */ oldreg = ti_gpio_intr_reg(sc, irq); sc->sc_irq_trigger[irq] = trig; sc->sc_irq_polarity[irq] = pol; reg = ti_gpio_intr_reg(sc, irq); if (reg != 0) { /* Apply the new settings. */ val = ti_gpio_read_4(sc, TI_GPIO_BANK(irq), reg); val |= TI_GPIO_MASK(irq); ti_gpio_write_4(sc, TI_GPIO_BANK(irq), reg, val); } if (oldreg != 0) { /* Remove the old settings. */ val = ti_gpio_read_4(sc, TI_GPIO_BANK(irq), oldreg); val &= ~TI_GPIO_MASK(irq); ti_gpio_write_4(sc, TI_GPIO_BANK(irq), oldreg, val); } TI_GPIO_UNLOCK(sc); return (0); } static int ti_gpio_setup_intr(device_t dev, device_t child, struct resource *ires, int flags, driver_filter_t *filt, driver_intr_t *handler, void *arg, void **cookiep) { struct ti_gpio_softc *sc; struct intr_event *event; int pin, error; sc = device_get_softc(dev); pin = rman_get_start(ires); if (ti_gpio_valid_pin(sc, pin) != 0) panic("%s: bad pin %d", __func__, pin); event = sc->sc_events[pin]; if (event == NULL) { error = intr_event_create(&event, (void *)(uintptr_t)pin, 0, pin, ti_gpio_mask_irq, ti_gpio_unmask_irq, NULL, NULL, "gpio%d pin%d:", device_get_unit(dev), pin); if (error != 0) return (error); sc->sc_events[pin] = event; } intr_event_add_handler(event, device_get_nameunit(child), filt, handler, arg, intr_priority(flags), flags, cookiep); return (0); } static int ti_gpio_teardown_intr(device_t dev, device_t child, struct resource *ires, void *cookie) { struct ti_gpio_softc *sc; int pin, err; sc = device_get_softc(dev); pin = rman_get_start(ires); if (ti_gpio_valid_pin(sc, pin) != 0) panic("%s: bad pin %d", __func__, pin); if (sc->sc_events[pin] == NULL) panic("Trying to teardown unoccupied IRQ"); err = intr_event_remove_handler(cookie); if (!err) sc->sc_events[pin] = NULL; return (err); } static phandle_t ti_gpio_get_node(device_t bus, device_t dev) { /* We only have one child, the GPIO bus, which needs our own node. */ return (ofw_bus_get_node(bus)); } static device_method_t ti_gpio_methods[] = { DEVMETHOD(device_attach, ti_gpio_attach), DEVMETHOD(device_detach, ti_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, ti_gpio_pin_max), DEVMETHOD(gpio_pin_getname, ti_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, ti_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, ti_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, ti_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, ti_gpio_pin_get), DEVMETHOD(gpio_pin_set, ti_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, ti_gpio_pin_toggle), /* Bus interface */ DEVMETHOD(bus_activate_resource, ti_gpio_activate_resource), DEVMETHOD(bus_deactivate_resource, ti_gpio_deactivate_resource), DEVMETHOD(bus_config_intr, ti_gpio_config_intr), DEVMETHOD(bus_setup_intr, ti_gpio_setup_intr), DEVMETHOD(bus_teardown_intr, ti_gpio_teardown_intr), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_node, ti_gpio_get_node), {0, 0}, }; driver_t ti_gpio_driver = { "gpio", ti_gpio_methods, sizeof(struct ti_gpio_softc), }; diff --git a/sys/arm/ti/ti_pruss.c b/sys/arm/ti/ti_pruss.c index 03a76761b2be..9a981d80906e 100644 --- a/sys/arm/ti/ti_pruss.c +++ b/sys/arm/ti/ti_pruss.c @@ -1,304 +1,305 @@ /*- * Copyright (c) 2013 Rui Paulo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEBUG #define DPRINTF(fmt, ...) do { \ printf("%s: ", __func__); \ printf(fmt, __VA_ARGS__); \ } while (0) #else #define DPRINTF(fmt, ...) #endif static device_probe_t ti_pruss_probe; static device_attach_t ti_pruss_attach; static device_detach_t ti_pruss_detach; static void ti_pruss_intr(void *); static d_open_t ti_pruss_open; static d_mmap_t ti_pruss_mmap; static void ti_pruss_kq_read_detach(struct knote *); static int ti_pruss_kq_read_event(struct knote *, long); static d_kqfilter_t ti_pruss_kqfilter; #define TI_PRUSS_IRQS 8 struct ti_pruss_softc { struct mtx sc_mtx; struct resource *sc_mem_res; struct resource *sc_irq_res[TI_PRUSS_IRQS]; void *sc_intr[TI_PRUSS_IRQS]; bus_space_tag_t sc_bt; bus_space_handle_t sc_bh; struct cdev *sc_pdev; struct selinfo sc_selinfo; }; static struct cdevsw ti_pruss_cdevsw = { .d_version = D_VERSION, .d_name = "ti_pruss", .d_open = ti_pruss_open, .d_mmap = ti_pruss_mmap, .d_kqfilter = ti_pruss_kqfilter, }; static device_method_t ti_pruss_methods[] = { DEVMETHOD(device_probe, ti_pruss_probe), DEVMETHOD(device_attach, ti_pruss_attach), DEVMETHOD(device_detach, ti_pruss_detach), DEVMETHOD_END }; static driver_t ti_pruss_driver = { "ti_pruss", ti_pruss_methods, sizeof(struct ti_pruss_softc) }; static devclass_t ti_pruss_devclass; DRIVER_MODULE(ti_pruss, simplebus, ti_pruss_driver, ti_pruss_devclass, 0, 0); static struct resource_spec ti_pruss_irq_spec[] = { { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE }, { SYS_RES_IRQ, 2, RF_ACTIVE }, { SYS_RES_IRQ, 3, RF_ACTIVE }, { SYS_RES_IRQ, 4, RF_ACTIVE }, { SYS_RES_IRQ, 5, RF_ACTIVE }, { SYS_RES_IRQ, 6, RF_ACTIVE }, { SYS_RES_IRQ, 7, RF_ACTIVE }, { -1, 0, 0 } }; static struct ti_pruss_irq_arg { int irq; struct ti_pruss_softc *sc; } ti_pruss_irq_args[TI_PRUSS_IRQS]; static __inline uint32_t ti_pruss_reg_read(struct ti_pruss_softc *sc, uint32_t reg) { return (bus_space_read_4(sc->sc_bt, sc->sc_bh, reg)); } static __inline void ti_pruss_reg_write(struct ti_pruss_softc *sc, uint32_t reg, uint32_t val) { bus_space_write_4(sc->sc_bt, sc->sc_bh, reg, val); } static int ti_pruss_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "ti,pruss-v1") || ofw_bus_is_compatible(dev, "ti,pruss-v2")) { device_set_desc(dev, "TI Programmable Realtime Unit Subsystem"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int ti_pruss_attach(device_t dev) { struct ti_pruss_softc *sc; int rid, i; if (ti_prcm_clk_enable(PRUSS_CLK) != 0) { device_printf(dev, "could not enable PRUSS clock\n"); return (ENXIO); } sc = device_get_softc(dev); rid = 0; mtx_init(&sc->sc_mtx, "TI PRUSS", NULL, MTX_DEF); sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem_res == NULL) { device_printf(dev, "could not allocate memory resource\n"); return (ENXIO); } sc->sc_bt = rman_get_bustag(sc->sc_mem_res); sc->sc_bh = rman_get_bushandle(sc->sc_mem_res); if (bus_alloc_resources(dev, ti_pruss_irq_spec, sc->sc_irq_res) != 0) { device_printf(dev, "could not allocate interrupt resource\n"); ti_pruss_detach(dev); return (ENXIO); } for (i = 0; i < TI_PRUSS_IRQS; i++) { ti_pruss_irq_args[i].irq = i; ti_pruss_irq_args[i].sc = sc; if (bus_setup_intr(dev, sc->sc_irq_res[i], INTR_MPSAFE | INTR_TYPE_MISC, NULL, ti_pruss_intr, &ti_pruss_irq_args[i], &sc->sc_intr[i]) != 0) { device_printf(dev, "unable to setup the interrupt handler\n"); ti_pruss_detach(dev); return (ENXIO); } } if (ti_pruss_reg_read(sc, PRUSS_AM18XX_INTC) == PRUSS_AM18XX_REV) device_printf(dev, "AM18xx PRU-ICSS\n"); else if (ti_pruss_reg_read(sc, PRUSS_AM33XX_INTC) == PRUSS_AM33XX_REV) device_printf(dev, "AM33xx PRU-ICSS\n"); sc->sc_pdev = make_dev(&ti_pruss_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "pruss%d", device_get_unit(dev)); sc->sc_pdev->si_drv1 = dev; return (0); } static int ti_pruss_detach(device_t dev) { struct ti_pruss_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < TI_PRUSS_IRQS; i++) { if (sc->sc_intr[i]) bus_teardown_intr(dev, sc->sc_irq_res[i], sc->sc_intr[i]); if (sc->sc_irq_res[i]) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->sc_irq_res[i]), sc->sc_irq_res[i]); } if (sc->sc_mem_res) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_mem_res), sc->sc_mem_res); if (sc->sc_pdev) destroy_dev(sc->sc_pdev); return (0); } static void ti_pruss_intr(void *arg) { struct ti_pruss_irq_arg *iap; struct ti_pruss_softc *sc; iap = arg; sc = iap->sc; DPRINTF("interrupt %p", sc); KNOTE_UNLOCKED(&sc->sc_selinfo.si_note, iap->irq); } static int ti_pruss_open(struct cdev *cdev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) { return (0); } static int ti_pruss_mmap(struct cdev *cdev, vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr) { device_t dev = cdev->si_drv1; struct ti_pruss_softc *sc = device_get_softc(dev); if (offset > rman_get_size(sc->sc_mem_res)) return (-1); *paddr = rman_get_start(sc->sc_mem_res) + offset; + *memattr = VM_MEMATTR_UNCACHEABLE; return (0); } static struct filterops ti_pruss_kq_read = { .f_isfd = 1, .f_detach = ti_pruss_kq_read_detach, .f_event = ti_pruss_kq_read_event, }; static void ti_pruss_kq_read_detach(struct knote *kn) { struct ti_pruss_softc *sc = kn->kn_hook; knlist_remove(&sc->sc_selinfo.si_note, kn, 0); } static int ti_pruss_kq_read_event(struct knote *kn, long hint) { kn->kn_data = hint; return (hint); } static int ti_pruss_kqfilter(struct cdev *cdev, struct knote *kn) { device_t dev = cdev->si_drv1; struct ti_pruss_softc *sc = device_get_softc(dev); switch (kn->kn_filter) { case EVFILT_READ: kn->kn_hook = sc; kn->kn_fop = &ti_pruss_kq_read; knlist_add(&sc->sc_selinfo.si_note, kn, 1); break; default: return (EINVAL); } return (0); } diff --git a/sys/boot/common/load_elf.c b/sys/boot/common/load_elf.c index 4c801e9e363b..6ab2ba4b135e 100644 --- a/sys/boot/common/load_elf.c +++ b/sys/boot/common/load_elf.c @@ -1,1019 +1,1027 @@ /*- * Copyright (c) 1998 Michael Smith * Copyright (c) 1998 Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #define FREEBSD_ELF #include #include "bootstrap.h" #define COPYOUT(s,d,l) archsw.arch_copyout((vm_offset_t)(s), d, l) #if defined(__i386__) && __ELF_WORD_SIZE == 64 #undef ELF_TARG_CLASS #undef ELF_TARG_MACH #define ELF_TARG_CLASS ELFCLASS64 #define ELF_TARG_MACH EM_X86_64 #endif typedef struct elf_file { Elf_Phdr *ph; Elf_Ehdr *ehdr; Elf_Sym *symtab; Elf_Hashelt *hashtab; Elf_Hashelt nbuckets; Elf_Hashelt nchains; Elf_Hashelt *buckets; Elf_Hashelt *chains; Elf_Rel *rel; size_t relsz; Elf_Rela *rela; size_t relasz; char *strtab; size_t strsz; int fd; caddr_t firstpage; size_t firstlen; int kernel; u_int64_t off; } *elf_file_t; static int __elfN(loadimage)(struct preloaded_file *mp, elf_file_t ef, u_int64_t loadaddr); static int __elfN(lookup_symbol)(struct preloaded_file *mp, elf_file_t ef, const char* name, Elf_Sym* sym); static int __elfN(reloc_ptr)(struct preloaded_file *mp, elf_file_t ef, Elf_Addr p, void *val, size_t len); static int __elfN(parse_modmetadata)(struct preloaded_file *mp, elf_file_t ef, Elf_Addr p_start, Elf_Addr p_end); static symaddr_fn __elfN(symaddr); static char *fake_modname(const char *name); const char *__elfN(kerneltype) = "elf kernel"; const char *__elfN(moduletype) = "elf module"; u_int64_t __elfN(relocation_offset) = 0; static int __elfN(load_elf_header)(char *filename, elf_file_t ef) { ssize_t bytes_read; Elf_Ehdr *ehdr; int err; /* * Open the image, read and validate the ELF header */ if (filename == NULL) /* can't handle nameless */ return (EFTYPE); if ((ef->fd = open(filename, O_RDONLY)) == -1) return (errno); ef->firstpage = malloc(PAGE_SIZE); if (ef->firstpage == NULL) { close(ef->fd); return (ENOMEM); } bytes_read = read(ef->fd, ef->firstpage, PAGE_SIZE); ef->firstlen = (size_t)bytes_read; if (bytes_read < 0 || ef->firstlen <= sizeof(Elf_Ehdr)) { err = EFTYPE; /* could be EIO, but may be small file */ goto error; } ehdr = ef->ehdr = (Elf_Ehdr *)ef->firstpage; /* Is it ELF? */ if (!IS_ELF(*ehdr)) { err = EFTYPE; goto error; } if (ehdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || /* Layout ? */ ehdr->e_ident[EI_DATA] != ELF_TARG_DATA || ehdr->e_ident[EI_VERSION] != EV_CURRENT || /* Version ? */ ehdr->e_version != EV_CURRENT || ehdr->e_machine != ELF_TARG_MACH) { /* Machine ? */ err = EFTYPE; goto error; } return (0); error: if (ef->firstpage != NULL) { free(ef->firstpage); ef->firstpage = NULL; } if (ef->fd != -1) { close(ef->fd); ef->fd = -1; } return (err); } /* * Attempt to load the file (file) as an ELF module. It will be stored at * (dest), and a pointer to a module structure describing the loaded object * will be saved in (result). */ int __elfN(loadfile)(char *filename, u_int64_t dest, struct preloaded_file **result) { return (__elfN(loadfile_raw)(filename, dest, result, 0)); } int __elfN(loadfile_raw)(char *filename, u_int64_t dest, struct preloaded_file **result, int multiboot) { struct preloaded_file *fp, *kfp; struct elf_file ef; Elf_Ehdr *ehdr; int err; fp = NULL; bzero(&ef, sizeof(struct elf_file)); ef.fd = -1; err = __elfN(load_elf_header)(filename, &ef); if (err != 0) return (err); ehdr = ef.ehdr; /* * Check to see what sort of module we are. */ kfp = file_findfile(NULL, __elfN(kerneltype)); - if (ehdr->e_type == ET_DYN) { +#ifdef __powerpc__ + /* + * Kernels can be ET_DYN, so just assume the first loaded object is the + * kernel. This assumption will be checked later. + */ + if (kfp == NULL) + ef.kernel = 1; +#endif + if (ef.kernel || ehdr->e_type == ET_EXEC) { + /* Looks like a kernel */ + if (kfp != NULL) { + printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: kernel already loaded\n"); + err = EPERM; + goto oerr; + } + /* + * Calculate destination address based on kernel entrypoint + */ + dest = (ehdr->e_entry & ~PAGE_MASK); + if (dest == 0) { + printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: not a kernel (maybe static binary?)\n"); + err = EPERM; + goto oerr; + } + ef.kernel = 1; + + } else if (ehdr->e_type == ET_DYN) { /* Looks like a kld module */ if (multiboot != 0) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: can't load module as multiboot\n"); err = EPERM; goto oerr; } if (kfp == NULL) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: can't load module before kernel\n"); err = EPERM; goto oerr; } if (strcmp(__elfN(kerneltype), kfp->f_type)) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: can't load module with kernel type '%s'\n", kfp->f_type); err = EPERM; goto oerr; } /* Looks OK, got ahead */ ef.kernel = 0; - } else if (ehdr->e_type == ET_EXEC) { - /* Looks like a kernel */ - if (kfp != NULL) { - printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: kernel already loaded\n"); - err = EPERM; - goto oerr; - } - /* - * Calculate destination address based on kernel entrypoint - */ - dest = (ehdr->e_entry & ~PAGE_MASK); - if (dest == 0) { - printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: not a kernel (maybe static binary?)\n"); - err = EPERM; - goto oerr; - } - ef.kernel = 1; - } else { err = EFTYPE; goto oerr; } if (archsw.arch_loadaddr != NULL) dest = archsw.arch_loadaddr(LOAD_ELF, ehdr, dest); else dest = roundup(dest, PAGE_SIZE); /* * Ok, we think we should handle this. */ fp = file_alloc(); if (fp == NULL) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: cannot allocate module info\n"); err = EPERM; goto out; } if (ef.kernel == 1 && multiboot == 0) setenv("kernelname", filename, 1); fp->f_name = strdup(filename); if (multiboot == 0) fp->f_type = strdup(ef.kernel ? __elfN(kerneltype) : __elfN(moduletype)); else fp->f_type = strdup("elf multiboot kernel"); #ifdef ELF_VERBOSE if (ef.kernel) printf("%s entry at 0x%jx\n", filename, (uintmax_t)ehdr->e_entry); #else printf("%s ", filename); #endif fp->f_size = __elfN(loadimage)(fp, &ef, dest); if (fp->f_size == 0 || fp->f_addr == 0) goto ioerr; /* save exec header as metadata */ file_addmetadata(fp, MODINFOMD_ELFHDR, sizeof(*ehdr), ehdr); /* Load OK, return module pointer */ *result = (struct preloaded_file *)fp; err = 0; goto out; ioerr: err = EIO; oerr: file_discard(fp); out: if (ef.firstpage) free(ef.firstpage); if (ef.fd != -1) close(ef.fd); return(err); } /* * With the file (fd) open on the image, and (ehdr) containing * the Elf header, load the image at (off) */ static int __elfN(loadimage)(struct preloaded_file *fp, elf_file_t ef, u_int64_t off) { int i; u_int j; Elf_Ehdr *ehdr; Elf_Phdr *phdr, *php; Elf_Shdr *shdr; char *shstr; int ret; vm_offset_t firstaddr; vm_offset_t lastaddr; size_t chunk; ssize_t result; Elf_Addr ssym, esym; Elf_Dyn *dp; Elf_Addr adp; Elf_Addr ctors; int ndp; int symstrindex; int symtabindex; Elf_Size size; u_int fpcopy; Elf_Sym sym; Elf_Addr p_start, p_end; dp = NULL; shdr = NULL; ret = 0; firstaddr = lastaddr = 0; ehdr = ef->ehdr; if (ef->kernel) { #if defined(__i386__) || defined(__amd64__) #if __ELF_WORD_SIZE == 64 off = - (off & 0xffffffffff000000ull);/* x86_64 relocates after locore */ #else off = - (off & 0xff000000u); /* i386 relocates after locore */ #endif #elif defined(__powerpc__) /* * On the purely virtual memory machines like e500, the kernel is * linked against its final VA range, which is most often not * available at the loader stage, but only after kernel initializes * and completes its VM settings. In such cases we cannot use p_vaddr * field directly to load ELF segments, but put them at some * 'load-time' locations. */ if (off & 0xf0000000u) { off = -(off & 0xf0000000u); /* * XXX the physical load address should not be hardcoded. Note * that the Book-E kernel assumes that it's loaded at a 16MB * boundary for now... */ off += 0x01000000; ehdr->e_entry += off; #ifdef ELF_VERBOSE printf("Converted entry 0x%08x\n", ehdr->e_entry); #endif } else off = 0; #elif defined(__arm__) /* * The elf headers in some kernels specify virtual addresses in all * header fields. More recently, the e_entry and p_paddr fields are the * proper physical addresses. Even when the p_paddr fields are correct, * the MI code below uses the p_vaddr fields with an offset added for * loading (doing so is arguably wrong). To make loading work, we need * an offset that represents the difference between physical and virtual * addressing. ARM kernels are always linked at 0xCnnnnnnn. Depending * on the headers, the offset value passed in may be physical or virtual * (because it typically comes from e_entry), but we always replace * whatever is passed in with the va<->pa offset. On the other hand, we * always remove the high-order part of the entry address whether it's * physical or virtual, because it will be adjusted later for the actual * physical entry point based on where the image gets loaded. */ off = -0xc0000000; ehdr->e_entry &= ~0xf0000000; #ifdef ELF_VERBOSE printf("ehdr->e_entry 0x%08x, va<->pa off %llx\n", ehdr->e_entry, off); #endif #else off = 0; /* other archs use direct mapped kernels */ #endif __elfN(relocation_offset) = off; } ef->off = off; if ((ehdr->e_phoff + ehdr->e_phnum * sizeof(*phdr)) > ef->firstlen) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: program header not within first page\n"); goto out; } phdr = (Elf_Phdr *)(ef->firstpage + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { /* We want to load PT_LOAD segments only.. */ if (phdr[i].p_type != PT_LOAD) continue; #ifdef ELF_VERBOSE printf("Segment: 0x%lx@0x%lx -> 0x%lx-0x%lx", (long)phdr[i].p_filesz, (long)phdr[i].p_offset, (long)(phdr[i].p_vaddr + off), (long)(phdr[i].p_vaddr + off + phdr[i].p_memsz - 1)); #else if ((phdr[i].p_flags & PF_W) == 0) { printf("text=0x%lx ", (long)phdr[i].p_filesz); } else { printf("data=0x%lx", (long)phdr[i].p_filesz); if (phdr[i].p_filesz < phdr[i].p_memsz) printf("+0x%lx", (long)(phdr[i].p_memsz -phdr[i].p_filesz)); printf(" "); } #endif fpcopy = 0; if (ef->firstlen > phdr[i].p_offset) { fpcopy = ef->firstlen - phdr[i].p_offset; archsw.arch_copyin(ef->firstpage + phdr[i].p_offset, phdr[i].p_vaddr + off, fpcopy); } if (phdr[i].p_filesz > fpcopy) { if (kern_pread(ef->fd, phdr[i].p_vaddr + off + fpcopy, phdr[i].p_filesz - fpcopy, phdr[i].p_offset + fpcopy) != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: read failed\n"); goto out; } } /* clear space from oversized segments; eg: bss */ if (phdr[i].p_filesz < phdr[i].p_memsz) { #ifdef ELF_VERBOSE printf(" (bss: 0x%lx-0x%lx)", (long)(phdr[i].p_vaddr + off + phdr[i].p_filesz), (long)(phdr[i].p_vaddr + off + phdr[i].p_memsz - 1)); #endif kern_bzero(phdr[i].p_vaddr + off + phdr[i].p_filesz, phdr[i].p_memsz - phdr[i].p_filesz); } #ifdef ELF_VERBOSE printf("\n"); #endif if (archsw.arch_loadseg != NULL) archsw.arch_loadseg(ehdr, phdr + i, off); if (firstaddr == 0 || firstaddr > (phdr[i].p_vaddr + off)) firstaddr = phdr[i].p_vaddr + off; if (lastaddr == 0 || lastaddr < (phdr[i].p_vaddr + off + phdr[i].p_memsz)) lastaddr = phdr[i].p_vaddr + off + phdr[i].p_memsz; } lastaddr = roundup(lastaddr, sizeof(long)); /* * Get the section headers. We need this for finding the .ctors * section as well as for loading any symbols. Both may be hard * to do if reading from a .gz file as it involves seeking. I * think the rule is going to have to be that you must strip a * file to remove symbols before gzipping it. */ chunk = ehdr->e_shnum * ehdr->e_shentsize; if (chunk == 0 || ehdr->e_shoff == 0) goto nosyms; shdr = alloc_pread(ef->fd, ehdr->e_shoff, chunk); if (shdr == NULL) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: failed to read section headers"); goto nosyms; } file_addmetadata(fp, MODINFOMD_SHDR, chunk, shdr); /* * Read the section string table and look for the .ctors section. * We need to tell the kernel where it is so that it can call the * ctors. */ chunk = shdr[ehdr->e_shstrndx].sh_size; if (chunk) { shstr = alloc_pread(ef->fd, shdr[ehdr->e_shstrndx].sh_offset, chunk); if (shstr) { for (i = 0; i < ehdr->e_shnum; i++) { if (strcmp(shstr + shdr[i].sh_name, ".ctors") != 0) continue; ctors = shdr[i].sh_addr; file_addmetadata(fp, MODINFOMD_CTORS_ADDR, sizeof(ctors), &ctors); size = shdr[i].sh_size; file_addmetadata(fp, MODINFOMD_CTORS_SIZE, sizeof(size), &size); break; } free(shstr); } } /* * Now load any symbols. */ symtabindex = -1; symstrindex = -1; for (i = 0; i < ehdr->e_shnum; i++) { if (shdr[i].sh_type != SHT_SYMTAB) continue; for (j = 0; j < ehdr->e_phnum; j++) { if (phdr[j].p_type != PT_LOAD) continue; if (shdr[i].sh_offset >= phdr[j].p_offset && (shdr[i].sh_offset + shdr[i].sh_size <= phdr[j].p_offset + phdr[j].p_filesz)) { shdr[i].sh_offset = 0; shdr[i].sh_size = 0; break; } } if (shdr[i].sh_offset == 0 || shdr[i].sh_size == 0) continue; /* alread loaded in a PT_LOAD above */ /* Save it for loading below */ symtabindex = i; symstrindex = shdr[i].sh_link; } if (symtabindex < 0 || symstrindex < 0) goto nosyms; /* Ok, committed to a load. */ #ifndef ELF_VERBOSE printf("syms=["); #endif ssym = lastaddr; for (i = symtabindex; i >= 0; i = symstrindex) { #ifdef ELF_VERBOSE char *secname; switch(shdr[i].sh_type) { case SHT_SYMTAB: /* Symbol table */ secname = "symtab"; break; case SHT_STRTAB: /* String table */ secname = "strtab"; break; default: secname = "WHOA!!"; break; } #endif size = shdr[i].sh_size; archsw.arch_copyin(&size, lastaddr, sizeof(size)); lastaddr += sizeof(size); #ifdef ELF_VERBOSE printf("\n%s: 0x%jx@0x%jx -> 0x%jx-0x%jx", secname, (uintmax_t)shdr[i].sh_size, (uintmax_t)shdr[i].sh_offset, (uintmax_t)lastaddr, (uintmax_t)(lastaddr + shdr[i].sh_size)); #else if (i == symstrindex) printf("+"); printf("0x%lx+0x%lx", (long)sizeof(size), (long)size); #endif if (lseek(ef->fd, (off_t)shdr[i].sh_offset, SEEK_SET) == -1) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: could not seek for symbols - skipped!"); lastaddr = ssym; ssym = 0; goto nosyms; } result = archsw.arch_readin(ef->fd, lastaddr, shdr[i].sh_size); if (result < 0 || (size_t)result != shdr[i].sh_size) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: could not read symbols - skipped! (%ju != %ju)", (uintmax_t)result, (uintmax_t)shdr[i].sh_size); lastaddr = ssym; ssym = 0; goto nosyms; } /* Reset offsets relative to ssym */ lastaddr += shdr[i].sh_size; lastaddr = roundup(lastaddr, sizeof(size)); if (i == symtabindex) symtabindex = -1; else if (i == symstrindex) symstrindex = -1; } esym = lastaddr; #ifndef ELF_VERBOSE printf("]"); #endif file_addmetadata(fp, MODINFOMD_SSYM, sizeof(ssym), &ssym); file_addmetadata(fp, MODINFOMD_ESYM, sizeof(esym), &esym); nosyms: printf("\n"); ret = lastaddr - firstaddr; fp->f_addr = firstaddr; php = NULL; for (i = 0; i < ehdr->e_phnum; i++) { if (phdr[i].p_type == PT_DYNAMIC) { php = phdr + i; adp = php->p_vaddr; file_addmetadata(fp, MODINFOMD_DYNAMIC, sizeof(adp), &adp); break; } } if (php == NULL) /* this is bad, we cannot get to symbols or _DYNAMIC */ goto out; ndp = php->p_filesz / sizeof(Elf_Dyn); if (ndp == 0) goto out; dp = malloc(php->p_filesz); if (dp == NULL) goto out; archsw.arch_copyout(php->p_vaddr + off, dp, php->p_filesz); ef->strsz = 0; for (i = 0; i < ndp; i++) { if (dp[i].d_tag == 0) break; switch (dp[i].d_tag) { case DT_HASH: ef->hashtab = (Elf_Hashelt*)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_STRTAB: ef->strtab = (char *)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_STRSZ: ef->strsz = dp[i].d_un.d_val; break; case DT_SYMTAB: ef->symtab = (Elf_Sym*)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_REL: ef->rel = (Elf_Rel *)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_RELSZ: ef->relsz = dp[i].d_un.d_val; break; case DT_RELA: ef->rela = (Elf_Rela *)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_RELASZ: ef->relasz = dp[i].d_un.d_val; break; default: break; } } if (ef->hashtab == NULL || ef->symtab == NULL || ef->strtab == NULL || ef->strsz == 0) goto out; COPYOUT(ef->hashtab, &ef->nbuckets, sizeof(ef->nbuckets)); COPYOUT(ef->hashtab + 1, &ef->nchains, sizeof(ef->nchains)); ef->buckets = ef->hashtab + 2; ef->chains = ef->buckets + ef->nbuckets; if (__elfN(lookup_symbol)(fp, ef, "__start_set_modmetadata_set", &sym) != 0) return 0; p_start = sym.st_value + ef->off; if (__elfN(lookup_symbol)(fp, ef, "__stop_set_modmetadata_set", &sym) != 0) return ENOENT; p_end = sym.st_value + ef->off; if (__elfN(parse_modmetadata)(fp, ef, p_start, p_end) == 0) goto out; if (ef->kernel) /* kernel must not depend on anything */ goto out; out: if (dp) free(dp); if (shdr) free(shdr); return ret; } static char invalid_name[] = "bad"; char * fake_modname(const char *name) { const char *sp, *ep; char *fp; size_t len; sp = strrchr(name, '/'); if (sp) sp++; else sp = name; ep = strrchr(name, '.'); if (ep) { if (ep == name) { sp = invalid_name; ep = invalid_name + sizeof(invalid_name) - 1; } } else ep = name + strlen(name); len = ep - sp; fp = malloc(len + 1); if (fp == NULL) return NULL; memcpy(fp, sp, len); fp[len] = '\0'; return fp; } #if (defined(__i386__) || defined(__powerpc__)) && __ELF_WORD_SIZE == 64 struct mod_metadata64 { int md_version; /* structure version MDTV_* */ int md_type; /* type of entry MDT_* */ u_int64_t md_data; /* specific data */ u_int64_t md_cval; /* common string label */ }; #endif #if defined(__amd64__) && __ELF_WORD_SIZE == 32 struct mod_metadata32 { int md_version; /* structure version MDTV_* */ int md_type; /* type of entry MDT_* */ u_int32_t md_data; /* specific data */ u_int32_t md_cval; /* common string label */ }; #endif int __elfN(load_modmetadata)(struct preloaded_file *fp, u_int64_t dest) { struct elf_file ef; int err, i, j; Elf_Shdr *sh_meta, *shdr = NULL; Elf_Shdr *sh_data[2]; char *shstrtab = NULL; size_t size; Elf_Addr p_start, p_end; bzero(&ef, sizeof(struct elf_file)); ef.fd = -1; err = __elfN(load_elf_header)(fp->f_name, &ef); if (err != 0) goto out; if (ef.ehdr->e_type == ET_EXEC) { ef.kernel = 1; } else if (ef.ehdr->e_type != ET_DYN) { err = EFTYPE; goto out; } size = ef.ehdr->e_shnum * ef.ehdr->e_shentsize; shdr = alloc_pread(ef.fd, ef.ehdr->e_shoff, size); if (shdr == NULL) { err = ENOMEM; goto out; } /* Load shstrtab. */ shstrtab = alloc_pread(ef.fd, shdr[ef.ehdr->e_shstrndx].sh_offset, shdr[ef.ehdr->e_shstrndx].sh_size); if (shstrtab == NULL) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load shstrtab\n"); err = EFTYPE; goto out; } /* Find set_modmetadata_set and data sections. */ sh_data[0] = sh_data[1] = sh_meta = NULL; for (i = 0, j = 0; i < ef.ehdr->e_shnum; i++) { if (strcmp(&shstrtab[shdr[i].sh_name], "set_modmetadata_set") == 0) { sh_meta = &shdr[i]; } if ((strcmp(&shstrtab[shdr[i].sh_name], ".data") == 0) || (strcmp(&shstrtab[shdr[i].sh_name], ".rodata") == 0)) { sh_data[j++] = &shdr[i]; } } if (sh_meta == NULL || sh_data[0] == NULL || sh_data[1] == NULL) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to find set_modmetadata_set or data sections\n"); err = EFTYPE; goto out; } /* Load set_modmetadata_set into memory */ err = kern_pread(ef.fd, dest, sh_meta->sh_size, sh_meta->sh_offset); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load set_modmetadata_set: %d\n", err); goto out; } p_start = dest; p_end = dest + sh_meta->sh_size; dest += sh_meta->sh_size; /* Load data sections into memory. */ err = kern_pread(ef.fd, dest, sh_data[0]->sh_size, sh_data[0]->sh_offset); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load data: %d\n", err); goto out; } /* * We have to increment the dest, so that the offset is the same into * both the .rodata and .data sections. */ ef.off = -(sh_data[0]->sh_addr - dest); dest += (sh_data[1]->sh_addr - sh_data[0]->sh_addr); err = kern_pread(ef.fd, dest, sh_data[1]->sh_size, sh_data[1]->sh_offset); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load data: %d\n", err); goto out; } err = __elfN(parse_modmetadata)(fp, &ef, p_start, p_end); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to parse metadata: %d\n", err); goto out; } out: if (shstrtab != NULL) free(shstrtab); if (shdr != NULL) free(shdr); if (ef.firstpage != NULL) free(ef.firstpage); if (ef.fd != -1) close(ef.fd); return (err); } int __elfN(parse_modmetadata)(struct preloaded_file *fp, elf_file_t ef, Elf_Addr p_start, Elf_Addr p_end) { struct mod_metadata md; #if (defined(__i386__) || defined(__powerpc__)) && __ELF_WORD_SIZE == 64 struct mod_metadata64 md64; #elif defined(__amd64__) && __ELF_WORD_SIZE == 32 struct mod_metadata32 md32; #endif struct mod_depend *mdepend; struct mod_version mver; char *s; int error, modcnt, minfolen; Elf_Addr v, p; modcnt = 0; p = p_start; while (p < p_end) { COPYOUT(p, &v, sizeof(v)); error = __elfN(reloc_ptr)(fp, ef, p, &v, sizeof(v)); if (error == EOPNOTSUPP) v += ef->off; else if (error != 0) return (error); #if (defined(__i386__) || defined(__powerpc__)) && __ELF_WORD_SIZE == 64 COPYOUT(v, &md64, sizeof(md64)); error = __elfN(reloc_ptr)(fp, ef, v, &md64, sizeof(md64)); if (error == EOPNOTSUPP) { md64.md_cval += ef->off; md64.md_data += ef->off; } else if (error != 0) return (error); md.md_version = md64.md_version; md.md_type = md64.md_type; md.md_cval = (const char *)(uintptr_t)md64.md_cval; md.md_data = (void *)(uintptr_t)md64.md_data; #elif defined(__amd64__) && __ELF_WORD_SIZE == 32 COPYOUT(v, &md32, sizeof(md32)); error = __elfN(reloc_ptr)(fp, ef, v, &md32, sizeof(md32)); if (error == EOPNOTSUPP) { md32.md_cval += ef->off; md32.md_data += ef->off; } else if (error != 0) return (error); md.md_version = md32.md_version; md.md_type = md32.md_type; md.md_cval = (const char *)(uintptr_t)md32.md_cval; md.md_data = (void *)(uintptr_t)md32.md_data; #else COPYOUT(v, &md, sizeof(md)); error = __elfN(reloc_ptr)(fp, ef, v, &md, sizeof(md)); if (error == EOPNOTSUPP) { md.md_cval += ef->off; md.md_data += ef->off; } else if (error != 0) return (error); #endif p += sizeof(Elf_Addr); switch(md.md_type) { case MDT_DEPEND: if (ef->kernel) /* kernel must not depend on anything */ break; s = strdupout((vm_offset_t)md.md_cval); minfolen = sizeof(*mdepend) + strlen(s) + 1; mdepend = malloc(minfolen); if (mdepend == NULL) return ENOMEM; COPYOUT((vm_offset_t)md.md_data, mdepend, sizeof(*mdepend)); strcpy((char*)(mdepend + 1), s); free(s); file_addmetadata(fp, MODINFOMD_DEPLIST, minfolen, mdepend); free(mdepend); break; case MDT_VERSION: s = strdupout((vm_offset_t)md.md_cval); COPYOUT((vm_offset_t)md.md_data, &mver, sizeof(mver)); file_addmodule(fp, s, mver.mv_version, NULL); free(s); modcnt++; break; } } if (modcnt == 0) { s = fake_modname(fp->f_name); file_addmodule(fp, s, 1, NULL); free(s); } return 0; } static unsigned long elf_hash(const char *name) { const unsigned char *p = (const unsigned char *) name; unsigned long h = 0; unsigned long g; while (*p != '\0') { h = (h << 4) + *p++; if ((g = h & 0xf0000000) != 0) h ^= g >> 24; h &= ~g; } return h; } static const char __elfN(bad_symtable)[] = "elf" __XSTRING(__ELF_WORD_SIZE) "_lookup_symbol: corrupt symbol table\n"; int __elfN(lookup_symbol)(struct preloaded_file *fp, elf_file_t ef, const char* name, Elf_Sym *symp) { Elf_Hashelt symnum; Elf_Sym sym; char *strp; unsigned long hash; hash = elf_hash(name); COPYOUT(&ef->buckets[hash % ef->nbuckets], &symnum, sizeof(symnum)); while (symnum != STN_UNDEF) { if (symnum >= ef->nchains) { printf(__elfN(bad_symtable)); return ENOENT; } COPYOUT(ef->symtab + symnum, &sym, sizeof(sym)); if (sym.st_name == 0) { printf(__elfN(bad_symtable)); return ENOENT; } strp = strdupout((vm_offset_t)(ef->strtab + sym.st_name)); if (strcmp(name, strp) == 0) { free(strp); if (sym.st_shndx != SHN_UNDEF || (sym.st_value != 0 && ELF_ST_TYPE(sym.st_info) == STT_FUNC)) { *symp = sym; return 0; } return ENOENT; } free(strp); COPYOUT(&ef->chains[symnum], &symnum, sizeof(symnum)); } return ENOENT; } /* * Apply any intra-module relocations to the value. p is the load address * of the value and val/len is the value to be modified. This does NOT modify * the image in-place, because this is done by kern_linker later on. * * Returns EOPNOTSUPP if no relocation method is supplied. */ static int __elfN(reloc_ptr)(struct preloaded_file *mp, elf_file_t ef, Elf_Addr p, void *val, size_t len) { size_t n; Elf_Rela a; Elf_Rel r; int error; /* * The kernel is already relocated, but we still want to apply * offset adjustments. */ if (ef->kernel) return (EOPNOTSUPP); for (n = 0; n < ef->relsz / sizeof(r); n++) { COPYOUT(ef->rel + n, &r, sizeof(r)); error = __elfN(reloc)(ef, __elfN(symaddr), &r, ELF_RELOC_REL, ef->off, p, val, len); if (error != 0) return (error); } for (n = 0; n < ef->relasz / sizeof(a); n++) { COPYOUT(ef->rela + n, &a, sizeof(a)); error = __elfN(reloc)(ef, __elfN(symaddr), &a, ELF_RELOC_RELA, ef->off, p, val, len); if (error != 0) return (error); } return (0); } static Elf_Addr __elfN(symaddr)(struct elf_file *ef, Elf_Size symidx) { /* Symbol lookup by index not required here. */ return (0); } diff --git a/sys/boot/i386/libi386/smbios.c b/sys/boot/i386/libi386/smbios.c index 0d5eb7b04ec0..570111ccb79f 100644 --- a/sys/boot/i386/libi386/smbios.c +++ b/sys/boot/i386/libi386/smbios.c @@ -1,440 +1,440 @@ /*- * Copyright (c) 2005-2009 Jung-uk Kim * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include "btxv86.h" #include "libi386.h" /* * Detect SMBIOS and export information about the SMBIOS into the * environment. * * System Management BIOS Reference Specification, v2.6 Final * http://www.dmtf.org/standards/published_documents/DSP0134_2.6.0.pdf */ /* * 2.1.1 SMBIOS Structure Table Entry Point * * "On non-EFI systems, the SMBIOS Entry Point structure, described below, can * be located by application software by searching for the anchor-string on * paragraph (16-byte) boundaries within the physical memory address range * 000F0000h to 000FFFFFh. This entry point encapsulates an intermediate anchor * string that is used by some existing DMI browsers." */ #define SMBIOS_START 0xf0000 #define SMBIOS_LENGTH 0x10000 #define SMBIOS_STEP 0x10 #define SMBIOS_SIG "_SM_" #define SMBIOS_DMI_SIG "_DMI_" #define SMBIOS_GET8(base, off) (*(uint8_t *)((base) + (off))) #define SMBIOS_GET16(base, off) (*(uint16_t *)((base) + (off))) #define SMBIOS_GET32(base, off) (*(uint32_t *)((base) + (off))) #define SMBIOS_GETLEN(base) SMBIOS_GET8(base, 0x01) #define SMBIOS_GETSTR(base) ((base) + SMBIOS_GETLEN(base)) struct smbios_attr { int probed; caddr_t addr; size_t length; size_t count; int major; int minor; int ver; const char* bios_vendor; const char* maker; const char* product; uint32_t enabled_memory; uint32_t old_enabled_memory; uint8_t enabled_sockets; uint8_t populated_sockets; }; static struct smbios_attr smbios; static uint8_t smbios_checksum(const caddr_t addr, const uint8_t len) { uint8_t sum; int i; for (sum = 0, i = 0; i < len; i++) sum += SMBIOS_GET8(addr, i); return (sum); } static caddr_t smbios_sigsearch(const caddr_t addr, const uint32_t len) { caddr_t cp; /* Search on 16-byte boundaries. */ for (cp = addr; cp < addr + len; cp += SMBIOS_STEP) if (strncmp(cp, SMBIOS_SIG, 4) == 0 && smbios_checksum(cp, SMBIOS_GET8(cp, 0x05)) == 0 && strncmp(cp + 0x10, SMBIOS_DMI_SIG, 5) == 0 && smbios_checksum(cp + 0x10, 0x0f) == 0) return (cp); return (NULL); } static const char* smbios_getstring(caddr_t addr, const int offset) { caddr_t cp; int i, idx; idx = SMBIOS_GET8(addr, offset); if (idx != 0) { cp = SMBIOS_GETSTR(addr); for (i = 1; i < idx; i++) cp += strlen(cp) + 1; return cp; } return (NULL); } static void smbios_setenv(const char *name, caddr_t addr, const int offset) { const char* val; val = smbios_getstring(addr, offset); if (val != NULL) setenv(name, val, 1); } #ifdef SMBIOS_SERIAL_NUMBERS #define UUID_SIZE 16 #define UUID_TYPE uint32_t #define UUID_STEP sizeof(UUID_TYPE) #define UUID_ALL_BITS (UUID_SIZE / UUID_STEP) #define UUID_GET(base, off) (*(UUID_TYPE *)((base) + (off))) static void smbios_setuuid(const char *name, const caddr_t addr, const int ver) { char uuid[37]; int byteorder, i, ones, zeros; UUID_TYPE n; uint32_t f1; uint16_t f2, f3; for (i = 0, ones = 0, zeros = 0; i < UUID_SIZE; i += UUID_STEP) { n = UUID_GET(addr, i) + 1; if (zeros == 0 && n == 0) ones++; else if (ones == 0 && n == 1) zeros++; else break; } if (ones != UUID_ALL_BITS && zeros != UUID_ALL_BITS) { /* * 3.3.2.1 System UUID * * "Although RFC 4122 recommends network byte order for all * fields, the PC industry (including the ACPI, UEFI, and * Microsoft specifications) has consistently used * little-endian byte encoding for the first three fields: * time_low, time_mid, time_hi_and_version. The same encoding, * also known as wire format, should also be used for the * SMBIOS representation of the UUID." * * Note: We use network byte order for backward compatibility * unless SMBIOS version is 2.6+ or little-endian is forced. */ #if defined(SMBIOS_LITTLE_ENDIAN_UUID) byteorder = LITTLE_ENDIAN; #elif defined(SMBIOS_NETWORK_ENDIAN_UUID) byteorder = BIG_ENDIAN; #else byteorder = ver < 0x0206 ? BIG_ENDIAN : LITTLE_ENDIAN; #endif if (byteorder != LITTLE_ENDIAN) { f1 = ntohl(SMBIOS_GET32(addr, 0)); f2 = ntohs(SMBIOS_GET16(addr, 4)); f3 = ntohs(SMBIOS_GET16(addr, 6)); } else { f1 = le32toh(SMBIOS_GET32(addr, 0)); f2 = le16toh(SMBIOS_GET16(addr, 4)); f3 = le16toh(SMBIOS_GET16(addr, 6)); } sprintf(uuid, "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", f1, f2, f3, SMBIOS_GET8(addr, 8), SMBIOS_GET8(addr, 9), SMBIOS_GET8(addr, 10), SMBIOS_GET8(addr, 11), SMBIOS_GET8(addr, 12), SMBIOS_GET8(addr, 13), SMBIOS_GET8(addr, 14), SMBIOS_GET8(addr, 15)); setenv(name, uuid, 1); } } #undef UUID_SIZE #undef UUID_TYPE #undef UUID_STEP #undef UUID_ALL_BITS #undef UUID_GET #endif static caddr_t smbios_parse_table(const caddr_t addr) { caddr_t cp; int proc, size, osize, type; type = SMBIOS_GET8(addr, 0); /* 3.1.2 Structure Header Format */ switch(type) { case 0: /* 3.3.1 BIOS Information (Type 0) */ smbios_setenv("smbios.bios.vendor", addr, 0x04); smbios_setenv("smbios.bios.version", addr, 0x05); smbios_setenv("smbios.bios.reldate", addr, 0x08); break; case 1: /* 3.3.2 System Information (Type 1) */ smbios_setenv("smbios.system.maker", addr, 0x04); smbios_setenv("smbios.system.product", addr, 0x05); smbios_setenv("smbios.system.version", addr, 0x06); #ifdef SMBIOS_SERIAL_NUMBERS smbios_setenv("smbios.system.serial", addr, 0x07); smbios_setuuid("smbios.system.uuid", addr + 0x08, smbios.ver); #endif break; case 2: /* 3.3.3 Base Board (or Module) Information (Type 2) */ smbios_setenv("smbios.planar.maker", addr, 0x04); smbios_setenv("smbios.planar.product", addr, 0x05); smbios_setenv("smbios.planar.version", addr, 0x06); #ifdef SMBIOS_SERIAL_NUMBERS smbios_setenv("smbios.planar.serial", addr, 0x07); #endif break; case 3: /* 3.3.4 System Enclosure or Chassis (Type 3) */ smbios_setenv("smbios.chassis.maker", addr, 0x04); smbios_setenv("smbios.chassis.version", addr, 0x06); #ifdef SMBIOS_SERIAL_NUMBERS smbios_setenv("smbios.chassis.serial", addr, 0x07); smbios_setenv("smbios.chassis.tag", addr, 0x08); #endif break; case 4: /* 3.3.5 Processor Information (Type 4) */ /* * Offset 18h: Processor Status * * Bit 7 Reserved, must be 0 * Bit 6 CPU Socket Populated * 1 - CPU Socket Populated * 0 - CPU Socket Unpopulated * Bit 5:3 Reserved, must be zero * Bit 2:0 CPU Status * 0h - Unknown * 1h - CPU Enabled * 2h - CPU Disabled by User via BIOS Setup * 3h - CPU Disabled by BIOS (POST Error) * 4h - CPU is Idle, waiting to be enabled * 5-6h - Reserved * 7h - Other */ proc = SMBIOS_GET8(addr, 0x18); if ((proc & 0x07) == 1) smbios.enabled_sockets++; if ((proc & 0x40) != 0) smbios.populated_sockets++; break; case 6: /* 3.3.7 Memory Module Information (Type 6, Obsolete) */ /* * Offset 0Ah: Enabled Size * * Bit 7 Bank connection * 1 - Double-bank connection * 0 - Single-bank connection * Bit 6:0 Size (n), where 2**n is the size in MB * 7Dh - Not determinable (Installed Size only) * 7Eh - Module is installed, but no memory * has been enabled * 7Fh - Not installed */ osize = SMBIOS_GET8(addr, 0x0a) & 0x7f; if (osize > 0 && osize < 22) smbios.old_enabled_memory += 1 << (osize + 10); break; case 17: /* 3.3.18 Memory Device (Type 17) */ /* * Offset 0Ch: Size * * Bit 15 Granularity * 1 - Value is in kilobytes units * 0 - Value is in megabytes units * Bit 14:0 Size */ size = SMBIOS_GET16(addr, 0x0c); if (size != 0 && size != 0xffff) smbios.enabled_memory += (size & 0x8000) != 0 ? (size & 0x7fff) : (size << 10); break; default: /* skip other types */ break; } /* Find structure terminator. */ cp = SMBIOS_GETSTR(addr); while (SMBIOS_GET16(cp, 0) != 0) cp++; return (cp + 2); } +static caddr_t +smbios_find_struct(int type) +{ + caddr_t dmi; + int i; + + if (smbios.addr == NULL) + return (NULL); + + for (dmi = smbios.addr, i = 0; + dmi < smbios.addr + smbios.length && i < smbios.count; i++) { + if (SMBIOS_GET8(dmi, 0) == type) + return dmi; + /* Find structure terminator. */ + dmi = SMBIOS_GETSTR(dmi); + while (SMBIOS_GET16(dmi, 0) != 0) + dmi++; + dmi += 2; + } + + return (NULL); +} + static void smbios_probe(void) { caddr_t saddr, info; u_int32_t paddr; if (smbios.probed) return; smbios.probed = 1; /* Search signatures and validate checksums. */ saddr = smbios_sigsearch(PTOV(SMBIOS_START), SMBIOS_LENGTH); if (saddr == NULL) return; smbios.length = SMBIOS_GET16(saddr, 0x16); /* Structure Table Length */ paddr = SMBIOS_GET32(saddr, 0x18); /* Structure Table Address */ smbios.count = SMBIOS_GET16(saddr, 0x1c); /* No of SMBIOS Structures */ smbios.ver = SMBIOS_GET8(saddr, 0x1e); /* SMBIOS BCD Revision */ if (smbios.ver != 0) { smbios.major = smbios.ver >> 4; smbios.minor = smbios.ver & 0x0f; if (smbios.major > 9 || smbios.minor > 9) smbios.ver = 0; } if (smbios.ver == 0) { smbios.major = SMBIOS_GET8(saddr, 0x06);/* SMBIOS Major Version */ smbios.minor = SMBIOS_GET8(saddr, 0x07);/* SMBIOS Minor Version */ } smbios.ver = (smbios.major << 8) | smbios.minor; smbios.addr = PTOV(paddr); /* Get system information from SMBIOS */ info = smbios_find_struct(0x00); if (info != NULL) { smbios.bios_vendor = smbios_getstring(info, 0x04); } info = smbios_find_struct(0x01); if (info != NULL) { smbios.maker = smbios_getstring(info, 0x04); smbios.product = smbios_getstring(info, 0x05); } } -static caddr_t -smbios_find_struct(int type) -{ - caddr_t dmi; - int i; - - if (smbios.addr == NULL) - return (NULL); - - for (dmi = smbios.addr, i = 0; - dmi < smbios.addr + smbios.length && i < smbios.count; i++) { - if (SMBIOS_GET8(dmi, 0) == type) - return dmi; - /* Find structure terminator. */ - dmi = SMBIOS_GETSTR(dmi); - while (SMBIOS_GET16(dmi, 0) != 0) - dmi++; - dmi += 2; - } - - return (NULL); -} - void smbios_detect(void) { char buf[16]; caddr_t dmi; int i; smbios_probe(); if (smbios.addr == NULL) return; for (dmi = smbios.addr, i = 0; dmi < smbios.addr + smbios.length && i < smbios.count; i++) dmi = smbios_parse_table(dmi); sprintf(buf, "%d.%d", smbios.major, smbios.minor); setenv("smbios.version", buf, 1); if (smbios.enabled_memory > 0 || smbios.old_enabled_memory > 0) { sprintf(buf, "%u", smbios.enabled_memory > 0 ? smbios.enabled_memory : smbios.old_enabled_memory); setenv("smbios.memory.enabled", buf, 1); } if (smbios.enabled_sockets > 0) { sprintf(buf, "%u", smbios.enabled_sockets); setenv("smbios.socket.enabled", buf, 1); } if (smbios.populated_sockets > 0) { sprintf(buf, "%u", smbios.populated_sockets); setenv("smbios.socket.populated", buf, 1); } } static int smbios_match_str(const char* s1, const char* s2) { return (s1 == NULL || (s2 != NULL && !strcmp(s1, s2))); } int smbios_match(const char* bios_vendor, const char* maker, const char* product) { smbios_probe(); return (smbios_match_str(bios_vendor, smbios.bios_vendor) && smbios_match_str(maker, smbios.maker) && smbios_match_str(product, smbios.product)); } diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c index 0114b57b8247..99d698b245a3 100644 --- a/sys/cam/ctl/ctl_frontend_iscsi.c +++ b/sys/cam/ctl/ctl_frontend_iscsi.c @@ -1,2979 +1,2981 @@ /*- * Copyright (c) 2012 The FreeBSD Foundation * All rights reserved. * * This software was developed by Edward Tomasz Napierala under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * CTL frontend for the iSCSI protocol. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #ifdef ICL_KERNEL_PROXY #include #endif #ifdef ICL_KERNEL_PROXY FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY"); #endif static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend"); static uma_zone_t cfiscsi_data_wait_zone; SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0, "CAM Target Layer iSCSI Frontend"); static int debug = 1; SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN, &debug, 1, "Enable debug messages"); static int ping_timeout = 5; SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds"); static int login_timeout = 60; SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds"); static int maxcmdsn_delta = 256; SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RWTUN, &maxcmdsn_delta, 256, "Number of commands the initiator can send " "without confirmation"); #define CFISCSI_DEBUG(X, ...) \ do { \ if (debug > 1) { \ printf("%s: " X "\n", \ __func__, ## __VA_ARGS__); \ } \ } while (0) #define CFISCSI_WARN(X, ...) \ do { \ if (debug > 0) { \ printf("WARNING: %s: " X "\n", \ __func__, ## __VA_ARGS__); \ } \ } while (0) #define CFISCSI_SESSION_DEBUG(S, X, ...) \ do { \ if (debug > 1) { \ printf("%s: %s (%s): " X "\n", \ __func__, S->cs_initiator_addr, \ S->cs_initiator_name, ## __VA_ARGS__); \ } \ } while (0) #define CFISCSI_SESSION_WARN(S, X, ...) \ do { \ if (debug > 0) { \ printf("WARNING: %s (%s): " X "\n", \ S->cs_initiator_addr, \ S->cs_initiator_name, ## __VA_ARGS__); \ } \ } while (0) #define CFISCSI_SESSION_LOCK(X) mtx_lock(&X->cs_lock) #define CFISCSI_SESSION_UNLOCK(X) mtx_unlock(&X->cs_lock) #define CFISCSI_SESSION_LOCK_ASSERT(X) mtx_assert(&X->cs_lock, MA_OWNED) #define CONN_SESSION(X) ((struct cfiscsi_session *)(X)->ic_prv0) #define PDU_SESSION(X) CONN_SESSION((X)->ip_conn) #define PDU_EXPDATASN(X) (X)->ip_prv0 #define PDU_TOTAL_TRANSFER_LEN(X) (X)->ip_prv1 #define PDU_R2TSN(X) (X)->ip_prv2 int cfiscsi_init(void); static void cfiscsi_online(void *arg); static void cfiscsi_offline(void *arg); static int cfiscsi_info(void *arg, struct sbuf *sb); static int cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id); static int cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id); static uint32_t cfiscsi_lun_map(void *arg, uint32_t lun); static int cfiscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td); static void cfiscsi_datamove(union ctl_io *io); static void cfiscsi_datamove_in(union ctl_io *io); static void cfiscsi_datamove_out(union ctl_io *io); static void cfiscsi_done(union ctl_io *io); static bool cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request); static void cfiscsi_pdu_handle_nop_out(struct icl_pdu *request); static void cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request); static void cfiscsi_pdu_handle_task_request(struct icl_pdu *request); static void cfiscsi_pdu_handle_data_out(struct icl_pdu *request); static void cfiscsi_pdu_handle_logout_request(struct icl_pdu *request); static void cfiscsi_session_terminate(struct cfiscsi_session *cs); static struct cfiscsi_target *cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name); static struct cfiscsi_target *cfiscsi_target_find_or_create( struct cfiscsi_softc *softc, const char *name, const char *alias); static void cfiscsi_target_release(struct cfiscsi_target *ct); static void cfiscsi_session_delete(struct cfiscsi_session *cs); static struct cfiscsi_softc cfiscsi_softc; extern struct ctl_softc *control_softc; static struct ctl_frontend cfiscsi_frontend = { .name = "iscsi", .init = cfiscsi_init, .ioctl = cfiscsi_ioctl, }; CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend); MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1); static struct icl_pdu * cfiscsi_pdu_new_response(struct icl_pdu *request, int flags) { return (icl_pdu_new(request->ip_conn, flags)); } static bool cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request) { const struct iscsi_bhs_scsi_command *bhssc; struct cfiscsi_session *cs; uint32_t cmdsn, expstatsn; cs = PDU_SESSION(request); /* * Every incoming PDU - not just NOP-Out - resets the ping timer. * The purpose of the timeout is to reset the connection when it stalls; * we don't want this to happen when NOP-In or NOP-Out ends up delayed * in some queue. * * XXX: Locking? */ cs->cs_timeout = 0; /* * Data-Out PDUs don't contain CmdSN. */ if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_DATA_OUT) return (false); /* * We're only using fields common for all the request * (initiator -> target) PDUs. */ bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs; cmdsn = ntohl(bhssc->bhssc_cmdsn); expstatsn = ntohl(bhssc->bhssc_expstatsn); CFISCSI_SESSION_LOCK(cs); #if 0 if (expstatsn != cs->cs_statsn) { CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, " "while current StatSN is %d", expstatsn, cs->cs_statsn); } #endif if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) { /* * The target MUST silently ignore any non-immediate command * outside of this range. */ if (ISCSI_SNLT(cmdsn, cs->cs_cmdsn) || ISCSI_SNGT(cmdsn, cs->cs_cmdsn + maxcmdsn_delta)) { CFISCSI_SESSION_UNLOCK(cs); CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, " "while expected %u", cmdsn, cs->cs_cmdsn); return (true); } /* * We don't support multiple connections now, so any * discontinuity in CmdSN means lost PDUs. Since we don't * support PDU retransmission -- terminate the connection. */ if (cmdsn != cs->cs_cmdsn) { CFISCSI_SESSION_UNLOCK(cs); CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, " "while expected %u; dropping connection", cmdsn, cs->cs_cmdsn); cfiscsi_session_terminate(cs); return (true); } cs->cs_cmdsn++; } CFISCSI_SESSION_UNLOCK(cs); return (false); } static void cfiscsi_pdu_handle(struct icl_pdu *request) { struct cfiscsi_session *cs; bool ignore; cs = PDU_SESSION(request); ignore = cfiscsi_pdu_update_cmdsn(request); if (ignore) { icl_pdu_free(request); return; } /* * Handle the PDU; this includes e.g. receiving the remaining * part of PDU and submitting the SCSI command to CTL * or queueing a reply. The handling routine is responsible * for freeing the PDU when it's no longer needed. */ switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) { case ISCSI_BHS_OPCODE_NOP_OUT: cfiscsi_pdu_handle_nop_out(request); break; case ISCSI_BHS_OPCODE_SCSI_COMMAND: cfiscsi_pdu_handle_scsi_command(request); break; case ISCSI_BHS_OPCODE_TASK_REQUEST: cfiscsi_pdu_handle_task_request(request); break; case ISCSI_BHS_OPCODE_SCSI_DATA_OUT: cfiscsi_pdu_handle_data_out(request); break; case ISCSI_BHS_OPCODE_LOGOUT_REQUEST: cfiscsi_pdu_handle_logout_request(request); break; default: CFISCSI_SESSION_WARN(cs, "received PDU with unsupported " "opcode 0x%x; dropping connection", request->ip_bhs->bhs_opcode); icl_pdu_free(request); cfiscsi_session_terminate(cs); } } static void cfiscsi_receive_callback(struct icl_pdu *request) { struct cfiscsi_session *cs; cs = PDU_SESSION(request); #ifdef ICL_KERNEL_PROXY if (cs->cs_waiting_for_ctld || cs->cs_login_phase) { if (cs->cs_login_pdu == NULL) cs->cs_login_pdu = request; else icl_pdu_free(request); cv_signal(&cs->cs_login_cv); return; } #endif cfiscsi_pdu_handle(request); } static void cfiscsi_error_callback(struct icl_conn *ic) { struct cfiscsi_session *cs; cs = CONN_SESSION(ic); CFISCSI_SESSION_WARN(cs, "connection error; dropping connection"); cfiscsi_session_terminate(cs); } static int cfiscsi_pdu_prepare(struct icl_pdu *response) { struct cfiscsi_session *cs; struct iscsi_bhs_scsi_response *bhssr; bool advance_statsn = true; cs = PDU_SESSION(response); CFISCSI_SESSION_LOCK_ASSERT(cs); /* * We're only using fields common for all the response * (target -> initiator) PDUs. */ bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs; /* * 10.8.3: "The StatSN for this connection is not advanced * after this PDU is sent." */ if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T) advance_statsn = false; /* * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff, * StatSN for the connection is not advanced after this PDU is sent." */ if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN && bhssr->bhssr_initiator_task_tag == 0xffffffff) advance_statsn = false; /* * See the comment below - StatSN is not meaningful and must * not be advanced. */ if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN && (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0) advance_statsn = false; /* * 10.7.3: "The fields StatSN, Status, and Residual Count * only have meaningful content if the S bit is set to 1." */ if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN || (bhssr->bhssr_flags & BHSDI_FLAGS_S)) bhssr->bhssr_statsn = htonl(cs->cs_statsn); bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn); bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn + maxcmdsn_delta); if (advance_statsn) cs->cs_statsn++; return (0); } static void cfiscsi_pdu_queue(struct icl_pdu *response) { struct cfiscsi_session *cs; cs = PDU_SESSION(response); CFISCSI_SESSION_LOCK(cs); cfiscsi_pdu_prepare(response); icl_pdu_queue(response); CFISCSI_SESSION_UNLOCK(cs); } static uint32_t cfiscsi_decode_lun(uint64_t encoded) { uint8_t lun[8]; uint32_t result; /* * The LUN field in iSCSI PDUs may look like an ordinary 64 bit number, * but is in fact an evil, multidimensional structure defined * in SCSI Architecture Model 5 (SAM-5), section 4.6. */ memcpy(lun, &encoded, sizeof(lun)); switch (lun[0] & 0xC0) { case 0x00: if ((lun[0] & 0x3f) != 0 || lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) { CFISCSI_WARN("malformed LUN " "(peripheral device addressing method): 0x%jx", (uintmax_t)encoded); result = 0xffffffff; break; } result = lun[1]; break; case 0x40: if (lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) { CFISCSI_WARN("malformed LUN " "(flat address space addressing method): 0x%jx", (uintmax_t)encoded); result = 0xffffffff; break; } result = ((lun[0] & 0x3f) << 8) + lun[1]; break; case 0xC0: if (lun[0] != 0xD2 || lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) { CFISCSI_WARN("malformed LUN (extended flat " "address space addressing method): 0x%jx", (uintmax_t)encoded); result = 0xffffffff; break; } result = (lun[1] << 16) + (lun[2] << 8) + lun[3]; default: CFISCSI_WARN("unsupported LUN format 0x%jx", (uintmax_t)encoded); result = 0xffffffff; break; } return (result); } static void cfiscsi_pdu_handle_nop_out(struct icl_pdu *request) { struct cfiscsi_session *cs; struct iscsi_bhs_nop_out *bhsno; struct iscsi_bhs_nop_in *bhsni; struct icl_pdu *response; void *data = NULL; size_t datasize; int error; cs = PDU_SESSION(request); bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs; if (bhsno->bhsno_initiator_task_tag == 0xffffffff) { /* * Nothing to do, iscsi_pdu_update_statsn() already * zeroed the timeout. */ icl_pdu_free(request); return; } datasize = icl_pdu_data_segment_length(request); if (datasize > 0) { data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO); if (data == NULL) { CFISCSI_SESSION_WARN(cs, "failed to allocate memory; " "dropping connection"); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } icl_pdu_get_data(request, 0, data, datasize); } response = cfiscsi_pdu_new_response(request, M_NOWAIT); if (response == NULL) { CFISCSI_SESSION_WARN(cs, "failed to allocate memory; " "droppping connection"); free(data, M_CFISCSI); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs; bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN; bhsni->bhsni_flags = 0x80; bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag; bhsni->bhsni_target_transfer_tag = 0xffffffff; if (datasize > 0) { error = icl_pdu_append_data(response, data, datasize, M_NOWAIT); if (error != 0) { CFISCSI_SESSION_WARN(cs, "failed to allocate memory; " "dropping connection"); free(data, M_CFISCSI); icl_pdu_free(request); icl_pdu_free(response); cfiscsi_session_terminate(cs); return; } free(data, M_CFISCSI); } icl_pdu_free(request); cfiscsi_pdu_queue(response); } static void cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request) { struct iscsi_bhs_scsi_command *bhssc; struct cfiscsi_session *cs; union ctl_io *io; int error; cs = PDU_SESSION(request); bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs; //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x", // bhssc->bhssc_initiator_task_tag); if (request->ip_data_len > 0 && cs->cs_immediate_data == false) { CFISCSI_SESSION_WARN(cs, "unsolicited data with " "ImmediateData=No; dropping connection"); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref); ctl_zero_io(io); io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request; io->io_hdr.io_type = CTL_IO_SCSI; io->io_hdr.nexus.initid.id = cs->cs_ctl_initid; io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port; io->io_hdr.nexus.targ_target.id = 0; io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhssc->bhssc_lun); io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag; switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) { case BHSSC_FLAGS_ATTR_UNTAGGED: io->scsiio.tag_type = CTL_TAG_UNTAGGED; break; case BHSSC_FLAGS_ATTR_SIMPLE: io->scsiio.tag_type = CTL_TAG_SIMPLE; break; case BHSSC_FLAGS_ATTR_ORDERED: io->scsiio.tag_type = CTL_TAG_ORDERED; break; case BHSSC_FLAGS_ATTR_HOQ: io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE; break; case BHSSC_FLAGS_ATTR_ACA: io->scsiio.tag_type = CTL_TAG_ACA; break; default: io->scsiio.tag_type = CTL_TAG_UNTAGGED; CFISCSI_SESSION_WARN(cs, "unhandled tag type %d", bhssc->bhssc_flags & BHSSC_FLAGS_ATTR); break; } io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */ memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb)); refcount_acquire(&cs->cs_outstanding_ctl_pdus); error = ctl_queue(io); if (error != CTL_RETVAL_COMPLETE) { CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; " "dropping connection", error); ctl_free_io(io); refcount_release(&cs->cs_outstanding_ctl_pdus); icl_pdu_free(request); cfiscsi_session_terminate(cs); } } static void cfiscsi_pdu_handle_task_request(struct icl_pdu *request) { struct iscsi_bhs_task_management_request *bhstmr; struct iscsi_bhs_task_management_response *bhstmr2; struct icl_pdu *response; struct cfiscsi_session *cs; union ctl_io *io; int error; cs = PDU_SESSION(request); bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs; io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref); ctl_zero_io(io); io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request; io->io_hdr.io_type = CTL_IO_TASK; io->io_hdr.nexus.initid.id = cs->cs_ctl_initid; io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port; io->io_hdr.nexus.targ_target.id = 0; io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhstmr->bhstmr_lun); io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */ switch (bhstmr->bhstmr_function & ~0x80) { case BHSTMR_FUNCTION_ABORT_TASK: #if 0 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK"); #endif io->taskio.task_action = CTL_TASK_ABORT_TASK; io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag; break; case BHSTMR_FUNCTION_ABORT_TASK_SET: #if 0 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET"); #endif io->taskio.task_action = CTL_TASK_ABORT_TASK_SET; break; case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET: #if 0 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET"); #endif io->taskio.task_action = CTL_TASK_LUN_RESET; break; case BHSTMR_FUNCTION_TARGET_WARM_RESET: #if 0 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET"); #endif io->taskio.task_action = CTL_TASK_TARGET_RESET; break; default: CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x", bhstmr->bhstmr_function & ~0x80); ctl_free_io(io); response = cfiscsi_pdu_new_response(request, M_NOWAIT); if (response == NULL) { CFISCSI_SESSION_WARN(cs, "failed to allocate memory; " "dropping connection"); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } bhstmr2 = (struct iscsi_bhs_task_management_response *) response->ip_bhs; bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE; bhstmr2->bhstmr_flags = 0x80; bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED; bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag; icl_pdu_free(request); cfiscsi_pdu_queue(response); return; } refcount_acquire(&cs->cs_outstanding_ctl_pdus); error = ctl_queue(io); if (error != CTL_RETVAL_COMPLETE) { CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; " "dropping connection", error); ctl_free_io(io); refcount_release(&cs->cs_outstanding_ctl_pdus); icl_pdu_free(request); cfiscsi_session_terminate(cs); } } static bool cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw) { struct iscsi_bhs_data_out *bhsdo; struct cfiscsi_session *cs; struct ctl_sg_entry ctl_sg_entry, *ctl_sglist; size_t copy_len, len, off, buffer_offset; int ctl_sg_count; union ctl_io *io; cs = PDU_SESSION(request); KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_DATA_OUT || (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_COMMAND, ("bad opcode 0x%x", request->ip_bhs->bhs_opcode)); /* * We're only using fields common for Data-Out and SCSI Command PDUs. */ bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs; io = cdw->cdw_ctl_io; KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN, ("CTL_FLAG_DATA_IN")); #if 0 CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d", request->ip_data_len, io->scsiio.kern_total_len); #endif if (io->scsiio.kern_sg_entries > 0) { ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr; ctl_sg_count = io->scsiio.kern_sg_entries; } else { ctl_sglist = &ctl_sg_entry; ctl_sglist->addr = io->scsiio.kern_data_ptr; ctl_sglist->len = io->scsiio.kern_data_len; ctl_sg_count = 1; } if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_DATA_OUT) buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset); else buffer_offset = 0; len = icl_pdu_data_segment_length(request); /* * Make sure the offset, as sent by the initiator, matches the offset * we're supposed to be at in the scatter-gather list. */ if (buffer_offset > io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled || buffer_offset + len <= io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) { CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, " "expected %zd; dropping connection", buffer_offset, (size_t)io->scsiio.kern_rel_offset + (size_t)io->scsiio.ext_data_filled); ctl_set_data_phase_error(&io->scsiio); cfiscsi_session_terminate(cs); return (true); } /* * This is the offset within the PDU data segment, as opposed * to buffer_offset, which is the offset within the task (SCSI * command). */ off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled - buffer_offset; /* * Iterate over the scatter/gather segments, filling them with data * from the PDU data segment. Note that this can get called multiple * times for one SCSI command; the cdw structure holds state for the * scatter/gather list. */ for (;;) { KASSERT(cdw->cdw_sg_index < ctl_sg_count, ("cdw->cdw_sg_index >= ctl_sg_count")); if (cdw->cdw_sg_len == 0) { cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr; cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len; } KASSERT(off <= len, ("len > off")); copy_len = len - off; if (copy_len > cdw->cdw_sg_len) copy_len = cdw->cdw_sg_len; icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len); cdw->cdw_sg_addr += copy_len; cdw->cdw_sg_len -= copy_len; off += copy_len; io->scsiio.ext_data_filled += copy_len; if (cdw->cdw_sg_len == 0) { /* * End of current segment. */ if (cdw->cdw_sg_index == ctl_sg_count - 1) { /* * Last segment in scatter/gather list. */ break; } cdw->cdw_sg_index++; } if (off == len) { /* * End of PDU payload. */ break; } } if (len > off) { /* * In case of unsolicited data, it's possible that the buffer * provided by CTL is smaller than negotiated FirstBurstLength. * Just ignore the superfluous data; will ask for them with R2T * on next call to cfiscsi_datamove(). * * This obviously can only happen with SCSI Command PDU. */ if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_COMMAND) return (true); CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, " "expected %zd; dropping connection", icl_pdu_data_segment_length(request), off); ctl_set_data_phase_error(&io->scsiio); cfiscsi_session_terminate(cs); return (true); } if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end && (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) { CFISCSI_SESSION_WARN(cs, "got the final packet without " "the F flag; flags = 0x%x; dropping connection", bhsdo->bhsdo_flags); ctl_set_data_phase_error(&io->scsiio); cfiscsi_session_terminate(cs); return (true); } if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end && (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) { if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_DATA_OUT) { CFISCSI_SESSION_WARN(cs, "got the final packet, but the " "transmitted size was %zd bytes instead of %d; " "dropping connection", (size_t)io->scsiio.ext_data_filled, cdw->cdw_r2t_end); ctl_set_data_phase_error(&io->scsiio); cfiscsi_session_terminate(cs); return (true); } else { /* * For SCSI Command PDU, this just means we need to * solicit more data by sending R2T. */ return (false); } } if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) { #if 0 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target " "transfer tag 0x%x", cdw->cdw_target_transfer_tag); #endif return (true); } return (false); } static void cfiscsi_pdu_handle_data_out(struct icl_pdu *request) { struct iscsi_bhs_data_out *bhsdo; struct cfiscsi_session *cs; struct cfiscsi_data_wait *cdw = NULL; union ctl_io *io; bool done; cs = PDU_SESSION(request); bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs; CFISCSI_SESSION_LOCK(cs); TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) { #if 0 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for " "ttt 0x%x, itt 0x%x", bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag, cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag)); #endif if (bhsdo->bhsdo_target_transfer_tag == cdw->cdw_target_transfer_tag) break; } CFISCSI_SESSION_UNLOCK(cs); if (cdw == NULL) { CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag " "0x%x, not found; dropping connection", bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) { CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with " "DataSN %u, while expected %u; dropping connection", ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } cdw->cdw_datasn++; io = cdw->cdw_ctl_io; KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN, ("CTL_FLAG_DATA_IN")); done = cfiscsi_handle_data_segment(request, cdw); if (done) { CFISCSI_SESSION_LOCK(cs); TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next); CFISCSI_SESSION_UNLOCK(cs); done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end || io->scsiio.ext_data_filled == io->scsiio.kern_data_len); uma_zfree(cfiscsi_data_wait_zone, cdw); if (done) io->scsiio.be_move_done(io); else cfiscsi_datamove_out(io); } icl_pdu_free(request); } static void cfiscsi_pdu_handle_logout_request(struct icl_pdu *request) { struct iscsi_bhs_logout_request *bhslr; struct iscsi_bhs_logout_response *bhslr2; struct icl_pdu *response; struct cfiscsi_session *cs; cs = PDU_SESSION(request); bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs; switch (bhslr->bhslr_reason & 0x7f) { case BHSLR_REASON_CLOSE_SESSION: case BHSLR_REASON_CLOSE_CONNECTION: response = cfiscsi_pdu_new_response(request, M_NOWAIT); if (response == NULL) { CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory"); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs; bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE; bhslr2->bhslr_flags = 0x80; bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY; bhslr2->bhslr_initiator_task_tag = bhslr->bhslr_initiator_task_tag; icl_pdu_free(request); cfiscsi_pdu_queue(response); cfiscsi_session_terminate(cs); break; case BHSLR_REASON_REMOVE_FOR_RECOVERY: response = cfiscsi_pdu_new_response(request, M_NOWAIT); if (response == NULL) { CFISCSI_SESSION_WARN(cs, "failed to allocate memory; dropping connection"); icl_pdu_free(request); cfiscsi_session_terminate(cs); return; } bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs; bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE; bhslr2->bhslr_flags = 0x80; bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED; bhslr2->bhslr_initiator_task_tag = bhslr->bhslr_initiator_task_tag; icl_pdu_free(request); cfiscsi_pdu_queue(response); break; default: CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection", bhslr->bhslr_reason); icl_pdu_free(request); cfiscsi_session_terminate(cs); break; } } static void cfiscsi_callout(void *context) { struct icl_pdu *cp; struct iscsi_bhs_nop_in *bhsni; struct cfiscsi_session *cs; cs = context; if (cs->cs_terminating) return; callout_schedule(&cs->cs_callout, 1 * hz); atomic_add_int(&cs->cs_timeout, 1); #ifdef ICL_KERNEL_PROXY if (cs->cs_waiting_for_ctld || cs->cs_login_phase) { if (login_timeout > 0 && cs->cs_timeout > login_timeout) { CFISCSI_SESSION_WARN(cs, "login timed out after " "%d seconds; dropping connection", cs->cs_timeout); cfiscsi_session_terminate(cs); } return; } #endif if (ping_timeout <= 0) { /* * Pings are disabled. Don't send NOP-In in this case; * user might have disabled pings to work around problems * with certain initiators that can't properly handle * NOP-In, such as iPXE. Reset the timeout, to avoid * triggering reconnection, should the user decide to * reenable them. */ cs->cs_timeout = 0; return; } if (cs->cs_timeout >= ping_timeout) { CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; " "dropping connection", ping_timeout); cfiscsi_session_terminate(cs); return; } /* * If the ping was reset less than one second ago - which means * that we've received some PDU during the last second - assume * the traffic flows correctly and don't bother sending a NOP-Out. * * (It's 2 - one for one second, and one for incrementing is_timeout * earlier in this routine.) */ if (cs->cs_timeout < 2) return; cp = icl_pdu_new(cs->cs_conn, M_NOWAIT); if (cp == NULL) { CFISCSI_SESSION_WARN(cs, "failed to allocate memory"); return; } bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs; bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN; bhsni->bhsni_flags = 0x80; bhsni->bhsni_initiator_task_tag = 0xffffffff; cfiscsi_pdu_queue(cp); } static void cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs) { struct cfiscsi_data_wait *cdw; union ctl_io *io; int error, last, wait; if (cs->cs_target == NULL) return; /* No target yet, so nothing to do. */ io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref); ctl_zero_io(io); io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs; io->io_hdr.io_type = CTL_IO_TASK; io->io_hdr.nexus.initid.id = cs->cs_ctl_initid; io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port; io->io_hdr.nexus.targ_target.id = 0; io->io_hdr.nexus.targ_lun = 0; io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */ io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; wait = cs->cs_outstanding_ctl_pdus; refcount_acquire(&cs->cs_outstanding_ctl_pdus); error = ctl_queue(io); if (error != CTL_RETVAL_COMPLETE) { CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error); refcount_release(&cs->cs_outstanding_ctl_pdus); ctl_free_io(io); } CFISCSI_SESSION_LOCK(cs); while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) { TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next); CFISCSI_SESSION_UNLOCK(cs); /* * Set nonzero port status; this prevents backends from * assuming that the data transfer actually succeeded * and writing uninitialized data to disk. */ cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42; cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io); uma_zfree(cfiscsi_data_wait_zone, cdw); CFISCSI_SESSION_LOCK(cs); } CFISCSI_SESSION_UNLOCK(cs); /* * Wait for CTL to terminate all the tasks. */ if (wait > 0) CFISCSI_SESSION_WARN(cs, "waiting for CTL to terminate %d tasks", wait); for (;;) { refcount_acquire(&cs->cs_outstanding_ctl_pdus); last = refcount_release(&cs->cs_outstanding_ctl_pdus); if (last != 0) break; tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus), 0, "cfiscsi_terminate", hz / 100); } if (wait > 0) CFISCSI_SESSION_WARN(cs, "tasks terminated"); } static void cfiscsi_maintenance_thread(void *arg) { struct cfiscsi_session *cs; cs = arg; for (;;) { CFISCSI_SESSION_LOCK(cs); if (cs->cs_terminating == false) cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock); CFISCSI_SESSION_UNLOCK(cs); if (cs->cs_terminating) { /* * We used to wait up to 30 seconds to deliver queued * PDUs to the initiator. We also tried hard to deliver * SCSI Responses for the aborted PDUs. We don't do * that anymore. We might need to revisit that. */ callout_drain(&cs->cs_callout); icl_conn_close(cs->cs_conn); /* * At this point ICL receive thread is no longer * running; no new tasks can be queued. */ cfiscsi_session_terminate_tasks(cs); cfiscsi_session_delete(cs); kthread_exit(); return; } CFISCSI_SESSION_DEBUG(cs, "nothing to do"); } } static void cfiscsi_session_terminate(struct cfiscsi_session *cs) { if (cs->cs_terminating) return; cs->cs_terminating = true; cv_signal(&cs->cs_maintenance_cv); #ifdef ICL_KERNEL_PROXY cv_signal(&cs->cs_login_cv); #endif } static int cfiscsi_session_register_initiator(struct cfiscsi_session *cs) { struct cfiscsi_target *ct; char *name; int i; KASSERT(cs->cs_ctl_initid == -1, ("already registered")); ct = cs->cs_target; name = strdup(cs->cs_initiator_id, M_CTL); i = ctl_add_initiator(&ct->ct_port, -1, 0, name); if (i < 0) { CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d", i); cs->cs_ctl_initid = -1; return (1); } cs->cs_ctl_initid = i; #if 0 CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i); #endif return (0); } static void cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs) { int error; if (cs->cs_ctl_initid == -1) return; error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid); if (error != 0) { CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d", error); } cs->cs_ctl_initid = -1; } static struct cfiscsi_session * cfiscsi_session_new(struct cfiscsi_softc *softc) { struct cfiscsi_session *cs; int error; cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO); if (cs == NULL) { CFISCSI_WARN("malloc failed"); return (NULL); } cs->cs_ctl_initid = -1; refcount_init(&cs->cs_outstanding_ctl_pdus, 0); TAILQ_INIT(&cs->cs_waiting_for_data_out); mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF); cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt"); #ifdef ICL_KERNEL_PROXY cv_init(&cs->cs_login_cv, "cfiscsi_login"); #endif - cs->cs_conn = icl_conn_new("cfiscsi", &cs->cs_lock); + cs->cs_conn = icl_new_conn(NULL, "cfiscsi", &cs->cs_lock); cs->cs_conn->ic_receive = cfiscsi_receive_callback; cs->cs_conn->ic_error = cfiscsi_error_callback; cs->cs_conn->ic_prv0 = cs; error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt"); if (error != 0) { CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error); free(cs, M_CFISCSI); return (NULL); } mtx_lock(&softc->lock); cs->cs_id = ++softc->last_session_id; TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next); mtx_unlock(&softc->lock); /* * Start pinging the initiator. */ callout_init(&cs->cs_callout, 1); callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs); return (cs); } static void cfiscsi_session_delete(struct cfiscsi_session *cs) { struct cfiscsi_softc *softc; softc = &cfiscsi_softc; KASSERT(cs->cs_outstanding_ctl_pdus == 0, ("destroying session with outstanding CTL pdus")); KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out), ("destroying session with non-empty queue")); cfiscsi_session_unregister_initiator(cs); if (cs->cs_target != NULL) cfiscsi_target_release(cs->cs_target); icl_conn_close(cs->cs_conn); icl_conn_free(cs->cs_conn); mtx_lock(&softc->lock); TAILQ_REMOVE(&softc->sessions, cs, cs_next); cv_signal(&softc->sessions_cv); mtx_unlock(&softc->lock); free(cs, M_CFISCSI); } int cfiscsi_init(void) { struct cfiscsi_softc *softc; int retval; softc = &cfiscsi_softc; retval = 0; bzero(softc, sizeof(*softc)); mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF); cv_init(&softc->sessions_cv, "cfiscsi_sessions"); #ifdef ICL_KERNEL_PROXY cv_init(&softc->accept_cv, "cfiscsi_accept"); #endif TAILQ_INIT(&softc->sessions); TAILQ_INIT(&softc->targets); cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait", sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); return (0); } #ifdef ICL_KERNEL_PROXY static void cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id) { struct cfiscsi_session *cs; cs = cfiscsi_session_new(&cfiscsi_softc); if (cs == NULL) { CFISCSI_WARN("failed to create session"); return; } icl_conn_handoff_sock(cs->cs_conn, so); cs->cs_initiator_sa = sa; cs->cs_portal_id = portal_id; cs->cs_waiting_for_ctld = true; cv_signal(&cfiscsi_softc.accept_cv); } #endif static void cfiscsi_online(void *arg) { struct cfiscsi_softc *softc; struct cfiscsi_target *ct; int online; ct = (struct cfiscsi_target *)arg; softc = ct->ct_softc; mtx_lock(&softc->lock); if (ct->ct_online) { mtx_unlock(&softc->lock); return; } ct->ct_online = 1; online = softc->online++; mtx_unlock(&softc->lock); if (online > 0) return; #ifdef ICL_KERNEL_PROXY if (softc->listener != NULL) icl_listen_free(softc->listener); softc->listener = icl_listen_new(cfiscsi_accept); #endif } static void cfiscsi_offline(void *arg) { struct cfiscsi_softc *softc; struct cfiscsi_target *ct; struct cfiscsi_session *cs; int online; ct = (struct cfiscsi_target *)arg; softc = ct->ct_softc; mtx_lock(&softc->lock); if (!ct->ct_online) { mtx_unlock(&softc->lock); return; } ct->ct_online = 0; online = --softc->online; TAILQ_FOREACH(cs, &softc->sessions, cs_next) { if (cs->cs_target == ct) cfiscsi_session_terminate(cs); } do { TAILQ_FOREACH(cs, &softc->sessions, cs_next) { if (cs->cs_target == ct) break; } if (cs != NULL) cv_wait(&softc->sessions_cv, &softc->lock); } while (cs != NULL && ct->ct_online == 0); mtx_unlock(&softc->lock); if (online > 0) return; #ifdef ICL_KERNEL_PROXY icl_listen_free(softc->listener); softc->listener = NULL; #endif } static int cfiscsi_info(void *arg, struct sbuf *sb) { struct cfiscsi_target *ct = (struct cfiscsi_target *)arg; int retval; retval = sbuf_printf(sb, "\t%d\n", ct->ct_state); return (retval); } static void cfiscsi_ioctl_handoff(struct ctl_iscsi *ci) { struct cfiscsi_softc *softc; struct cfiscsi_session *cs, *cs2; struct cfiscsi_target *ct; struct ctl_iscsi_handoff_params *cihp; int error; cihp = (struct ctl_iscsi_handoff_params *)&(ci->data); softc = &cfiscsi_softc; CFISCSI_DEBUG("new connection from %s (%s) to %s", cihp->initiator_name, cihp->initiator_addr, cihp->target_name); ct = cfiscsi_target_find(softc, cihp->target_name); if (ct == NULL) { ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "%s: target not found", __func__); return; } #ifdef ICL_KERNEL_PROXY if (cihp->socket > 0 && cihp->connection_id > 0) { snprintf(ci->error_str, sizeof(ci->error_str), "both socket and connection_id set"); ci->status = CTL_ISCSI_ERROR; cfiscsi_target_release(ct); return; } if (cihp->socket == 0) { mtx_lock(&cfiscsi_softc.lock); TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) { if (cs->cs_id == cihp->connection_id) break; } if (cs == NULL) { mtx_unlock(&cfiscsi_softc.lock); snprintf(ci->error_str, sizeof(ci->error_str), "connection not found"); ci->status = CTL_ISCSI_ERROR; cfiscsi_target_release(ct); return; } mtx_unlock(&cfiscsi_softc.lock); } else { #endif cs = cfiscsi_session_new(softc); if (cs == NULL) { ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "%s: cfiscsi_session_new failed", __func__); cfiscsi_target_release(ct); return; } #ifdef ICL_KERNEL_PROXY } #endif /* * First PDU of Full Feature phase has the same CmdSN as the last * PDU from the Login Phase received from the initiator. Thus, * the -1 below. */ cs->cs_portal_group_tag = cihp->portal_group_tag; cs->cs_cmdsn = cihp->cmdsn; cs->cs_statsn = cihp->statsn; cs->cs_max_data_segment_length = cihp->max_recv_data_segment_length; cs->cs_max_burst_length = cihp->max_burst_length; cs->cs_immediate_data = !!cihp->immediate_data; if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C) cs->cs_conn->ic_header_crc32c = true; if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C) cs->cs_conn->ic_data_crc32c = true; strlcpy(cs->cs_initiator_name, cihp->initiator_name, sizeof(cs->cs_initiator_name)); strlcpy(cs->cs_initiator_addr, cihp->initiator_addr, sizeof(cs->cs_initiator_addr)); strlcpy(cs->cs_initiator_alias, cihp->initiator_alias, sizeof(cs->cs_initiator_alias)); memcpy(cs->cs_initiator_isid, cihp->initiator_isid, sizeof(cs->cs_initiator_isid)); snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id), "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name, cihp->initiator_isid[0], cihp->initiator_isid[1], cihp->initiator_isid[2], cihp->initiator_isid[3], cihp->initiator_isid[4], cihp->initiator_isid[5]); mtx_lock(&softc->lock); if (ct->ct_online == 0) { mtx_unlock(&softc->lock); cfiscsi_session_terminate(cs); cfiscsi_target_release(ct); ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "%s: port offline", __func__); return; } cs->cs_target = ct; mtx_unlock(&softc->lock); refcount_acquire(&cs->cs_outstanding_ctl_pdus); restart: if (!cs->cs_terminating) { mtx_lock(&softc->lock); TAILQ_FOREACH(cs2, &softc->sessions, cs_next) { if (cs2 != cs && cs2->cs_tasks_aborted == false && cs->cs_target == cs2->cs_target && cs->cs_portal_group_tag == cs2->cs_portal_group_tag && strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) { cfiscsi_session_terminate(cs2); mtx_unlock(&softc->lock); pause("cfiscsi_reinstate", 1); goto restart; } } mtx_unlock(&softc->lock); } /* * Register initiator with CTL. */ cfiscsi_session_register_initiator(cs); #ifdef ICL_KERNEL_PROXY if (cihp->socket > 0) { #endif error = icl_conn_handoff(cs->cs_conn, cihp->socket); if (error != 0) { cfiscsi_session_terminate(cs); refcount_release(&cs->cs_outstanding_ctl_pdus); ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "%s: icl_conn_handoff failed with error %d", __func__, error); return; } #ifdef ICL_KERNEL_PROXY } #endif #ifdef ICL_KERNEL_PROXY cs->cs_login_phase = false; /* * First PDU of the Full Feature phase has likely already arrived. * We have to pick it up and execute properly. */ if (cs->cs_login_pdu != NULL) { CFISCSI_SESSION_DEBUG(cs, "picking up first PDU"); cfiscsi_pdu_handle(cs->cs_login_pdu); cs->cs_login_pdu = NULL; } #endif refcount_release(&cs->cs_outstanding_ctl_pdus); ci->status = CTL_ISCSI_OK; } static void cfiscsi_ioctl_list(struct ctl_iscsi *ci) { struct ctl_iscsi_list_params *cilp; struct cfiscsi_session *cs; struct cfiscsi_softc *softc; struct sbuf *sb; int error; cilp = (struct ctl_iscsi_list_params *)&(ci->data); softc = &cfiscsi_softc; sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN); if (sb == NULL) { ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "Unable to allocate %d bytes for iSCSI session list", cilp->alloc_len); return; } sbuf_printf(sb, "\n"); mtx_lock(&softc->lock); TAILQ_FOREACH(cs, &softc->sessions, cs_next) { #ifdef ICL_KERNEL_PROXY if (cs->cs_target == NULL) continue; #endif error = sbuf_printf(sb, "" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%zd" "%d" "%d" "\n", cs->cs_id, cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias, cs->cs_target->ct_name, cs->cs_target->ct_alias, cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None", cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None", cs->cs_max_data_segment_length, cs->cs_immediate_data, cs->cs_conn->ic_iser); if (error != 0) break; } mtx_unlock(&softc->lock); error = sbuf_printf(sb, "\n"); if (error != 0) { sbuf_delete(sb); ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE; snprintf(ci->error_str, sizeof(ci->error_str), "Out of space, %d bytes is too small", cilp->alloc_len); return; } sbuf_finish(sb); error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1); cilp->fill_len = sbuf_len(sb) + 1; ci->status = CTL_ISCSI_OK; sbuf_delete(sb); } static void cfiscsi_ioctl_terminate(struct ctl_iscsi *ci) { struct icl_pdu *response; struct iscsi_bhs_asynchronous_message *bhsam; struct ctl_iscsi_terminate_params *citp; struct cfiscsi_session *cs; struct cfiscsi_softc *softc; int found = 0; citp = (struct ctl_iscsi_terminate_params *)&(ci->data); softc = &cfiscsi_softc; mtx_lock(&softc->lock); TAILQ_FOREACH(cs, &softc->sessions, cs_next) { if (citp->all == 0 && cs->cs_id != citp->connection_id && strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 && strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0) continue; response = icl_pdu_new(cs->cs_conn, M_NOWAIT); if (response == NULL) { /* * Oh well. Just terminate the connection. */ } else { bhsam = (struct iscsi_bhs_asynchronous_message *) response->ip_bhs; bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE; bhsam->bhsam_flags = 0x80; bhsam->bhsam_0xffffffff = 0xffffffff; bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_TERMINATES_SESSION; cfiscsi_pdu_queue(response); } cfiscsi_session_terminate(cs); found++; } mtx_unlock(&softc->lock); if (found == 0) { ci->status = CTL_ISCSI_SESSION_NOT_FOUND; snprintf(ci->error_str, sizeof(ci->error_str), "No matching connections found"); return; } ci->status = CTL_ISCSI_OK; } static void cfiscsi_ioctl_logout(struct ctl_iscsi *ci) { struct icl_pdu *response; struct iscsi_bhs_asynchronous_message *bhsam; struct ctl_iscsi_logout_params *cilp; struct cfiscsi_session *cs; struct cfiscsi_softc *softc; int found = 0; cilp = (struct ctl_iscsi_logout_params *)&(ci->data); softc = &cfiscsi_softc; mtx_lock(&softc->lock); TAILQ_FOREACH(cs, &softc->sessions, cs_next) { if (cilp->all == 0 && cs->cs_id != cilp->connection_id && strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 && strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0) continue; response = icl_pdu_new(cs->cs_conn, M_NOWAIT); if (response == NULL) { ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "Unable to allocate memory"); mtx_unlock(&softc->lock); return; } bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs; bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE; bhsam->bhsam_flags = 0x80; bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT; bhsam->bhsam_parameter3 = htons(10); cfiscsi_pdu_queue(response); found++; } mtx_unlock(&softc->lock); if (found == 0) { ci->status = CTL_ISCSI_SESSION_NOT_FOUND; snprintf(ci->error_str, sizeof(ci->error_str), "No matching connections found"); return; } ci->status = CTL_ISCSI_OK; } #ifdef ICL_KERNEL_PROXY static void cfiscsi_ioctl_listen(struct ctl_iscsi *ci) { struct ctl_iscsi_listen_params *cilp; struct sockaddr *sa; int error; cilp = (struct ctl_iscsi_listen_params *)&(ci->data); if (cfiscsi_softc.listener == NULL) { CFISCSI_DEBUG("no listener"); snprintf(ci->error_str, sizeof(ci->error_str), "no listener"); ci->status = CTL_ISCSI_ERROR; return; } error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen); if (error != 0) { CFISCSI_DEBUG("getsockaddr, error %d", error); snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed"); ci->status = CTL_ISCSI_ERROR; return; } error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain, cilp->socktype, cilp->protocol, sa, cilp->portal_id); if (error != 0) { free(sa, M_SONAME); CFISCSI_DEBUG("icl_listen_add, error %d", error); snprintf(ci->error_str, sizeof(ci->error_str), "icl_listen_add failed, error %d", error); ci->status = CTL_ISCSI_ERROR; return; } ci->status = CTL_ISCSI_OK; } static void cfiscsi_ioctl_accept(struct ctl_iscsi *ci) { struct ctl_iscsi_accept_params *ciap; struct cfiscsi_session *cs; int error; ciap = (struct ctl_iscsi_accept_params *)&(ci->data); mtx_lock(&cfiscsi_softc.lock); for (;;) { TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) { if (cs->cs_waiting_for_ctld) break; } if (cs != NULL) break; error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock); if (error != 0) { mtx_unlock(&cfiscsi_softc.lock); snprintf(ci->error_str, sizeof(ci->error_str), "interrupted"); ci->status = CTL_ISCSI_ERROR; return; } } mtx_unlock(&cfiscsi_softc.lock); cs->cs_waiting_for_ctld = false; cs->cs_login_phase = true; ciap->connection_id = cs->cs_id; ciap->portal_id = cs->cs_portal_id; ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len; error = copyout(cs->cs_initiator_sa, ciap->initiator_addr, cs->cs_initiator_sa->sa_len); if (error != 0) { snprintf(ci->error_str, sizeof(ci->error_str), "copyout failed with error %d", error); ci->status = CTL_ISCSI_ERROR; return; } ci->status = CTL_ISCSI_OK; } static void cfiscsi_ioctl_send(struct ctl_iscsi *ci) { struct ctl_iscsi_send_params *cisp; struct cfiscsi_session *cs; struct icl_pdu *ip; size_t datalen; void *data; int error; cisp = (struct ctl_iscsi_send_params *)&(ci->data); mtx_lock(&cfiscsi_softc.lock); TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) { if (cs->cs_id == cisp->connection_id) break; } if (cs == NULL) { mtx_unlock(&cfiscsi_softc.lock); snprintf(ci->error_str, sizeof(ci->error_str), "connection not found"); ci->status = CTL_ISCSI_ERROR; return; } mtx_unlock(&cfiscsi_softc.lock); #if 0 if (cs->cs_login_phase == false) return (EBUSY); #endif if (cs->cs_terminating) { snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating"); ci->status = CTL_ISCSI_ERROR; return; } datalen = cisp->data_segment_len; /* * XXX */ //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) { if (datalen > 65535) { snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big"); ci->status = CTL_ISCSI_ERROR; return; } if (datalen > 0) { data = malloc(datalen, M_CFISCSI, M_WAITOK); error = copyin(cisp->data_segment, data, datalen); if (error != 0) { free(data, M_CFISCSI); snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error); ci->status = CTL_ISCSI_ERROR; return; } } ip = icl_pdu_new(cs->cs_conn, M_WAITOK); memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs)); if (datalen > 0) { icl_pdu_append_data(ip, data, datalen, M_WAITOK); free(data, M_CFISCSI); } CFISCSI_SESSION_LOCK(cs); icl_pdu_queue(ip); CFISCSI_SESSION_UNLOCK(cs); ci->status = CTL_ISCSI_OK; } static void cfiscsi_ioctl_receive(struct ctl_iscsi *ci) { struct ctl_iscsi_receive_params *cirp; struct cfiscsi_session *cs; struct icl_pdu *ip; void *data; int error; cirp = (struct ctl_iscsi_receive_params *)&(ci->data); mtx_lock(&cfiscsi_softc.lock); TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) { if (cs->cs_id == cirp->connection_id) break; } if (cs == NULL) { mtx_unlock(&cfiscsi_softc.lock); snprintf(ci->error_str, sizeof(ci->error_str), "connection not found"); ci->status = CTL_ISCSI_ERROR; return; } mtx_unlock(&cfiscsi_softc.lock); #if 0 if (is->is_login_phase == false) return (EBUSY); #endif CFISCSI_SESSION_LOCK(cs); while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) { error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock); if (error != 0) { CFISCSI_SESSION_UNLOCK(cs); snprintf(ci->error_str, sizeof(ci->error_str), "interrupted by signal"); ci->status = CTL_ISCSI_ERROR; return; } } if (cs->cs_terminating) { CFISCSI_SESSION_UNLOCK(cs); snprintf(ci->error_str, sizeof(ci->error_str), "connection terminating"); ci->status = CTL_ISCSI_ERROR; return; } ip = cs->cs_login_pdu; cs->cs_login_pdu = NULL; CFISCSI_SESSION_UNLOCK(cs); if (ip->ip_data_len > cirp->data_segment_len) { icl_pdu_free(ip); snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big"); ci->status = CTL_ISCSI_ERROR; return; } copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs)); if (ip->ip_data_len > 0) { data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK); icl_pdu_get_data(ip, 0, data, ip->ip_data_len); copyout(data, cirp->data_segment, ip->ip_data_len); free(data, M_CFISCSI); } icl_pdu_free(ip); ci->status = CTL_ISCSI_OK; } #endif /* !ICL_KERNEL_PROXY */ static void cfiscsi_ioctl_port_create(struct ctl_req *req) { struct cfiscsi_target *ct; struct ctl_port *port; const char *target, *alias, *tag; struct scsi_vpd_id_descriptor *desc; ctl_options_t opts; int retval, len, idlen; ctl_init_opts(&opts, req->num_args, req->kern_args); target = ctl_get_opt(&opts, "cfiscsi_target"); alias = ctl_get_opt(&opts, "cfiscsi_target_alias"); tag = ctl_get_opt(&opts, "cfiscsi_portal_group_tag"); if (target == NULL || tag == NULL) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Missing required argument"); ctl_free_opts(&opts); return; } ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias); if (ct == NULL) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "failed to create target \"%s\"", target); ctl_free_opts(&opts); return; } if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "target \"%s\" already exists", target); cfiscsi_target_release(ct); ctl_free_opts(&opts); return; } port = &ct->ct_port; + // WAT if (ct->ct_state == CFISCSI_TARGET_STATE_DYING) goto done; port->frontend = &cfiscsi_frontend; port->port_type = CTL_PORT_ISCSI; /* XXX KDM what should the real number be here? */ port->num_requested_ctl_io = 4096; port->port_name = "iscsi"; port->physical_port = strtoul(tag, NULL, 0); port->virtual_port = ct->ct_target_id; port->port_online = cfiscsi_online; port->port_offline = cfiscsi_offline; port->port_info = cfiscsi_info; port->onoff_arg = ct; port->lun_enable = cfiscsi_lun_enable; port->lun_disable = cfiscsi_lun_disable; port->lun_map = cfiscsi_lun_map; port->targ_lun_arg = ct; port->fe_datamove = cfiscsi_datamove; port->fe_done = cfiscsi_done; /* XXX KDM what should we report here? */ /* XXX These should probably be fetched from CTL. */ port->max_targets = 1; port->max_target_id = 15; port->options = opts; STAILQ_INIT(&opts); /* Generate Port ID. */ idlen = strlen(target) + strlen(",t,0x0001") + 1; idlen = roundup2(idlen, 4); len = sizeof(struct scsi_vpd_device_id) + idlen; port->port_devid = malloc(sizeof(struct ctl_devid) + len, M_CTL, M_WAITOK | M_ZERO); port->port_devid->len = len; desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data; desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8; desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT | SVPD_ID_TYPE_SCSI_NAME; desc->length = idlen; snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, port->physical_port); /* Generate Target ID. */ idlen = strlen(target) + 1; idlen = roundup2(idlen, 4); len = sizeof(struct scsi_vpd_device_id) + idlen; port->target_devid = malloc(sizeof(struct ctl_devid) + len, M_CTL, M_WAITOK | M_ZERO); port->target_devid->len = len; desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data; desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8; desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET | SVPD_ID_TYPE_SCSI_NAME; desc->length = idlen; strlcpy(desc->identifier, target, idlen); retval = ctl_port_register(port); if (retval != 0) { ctl_free_opts(&port->options); cfiscsi_target_release(ct); free(port->port_devid, M_CFISCSI); free(port->target_devid, M_CFISCSI); req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "ctl_frontend_register() failed with error %d", retval); return; } done: ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE; req->status = CTL_LUN_OK; memcpy(req->kern_args[0].kvalue, &port->targ_port, sizeof(port->targ_port)); //XXX } static void cfiscsi_ioctl_port_remove(struct ctl_req *req) { struct cfiscsi_target *ct; const char *target; ctl_options_t opts; ctl_init_opts(&opts, req->num_args, req->kern_args); target = ctl_get_opt(&opts, "cfiscsi_target"); if (target == NULL) { ctl_free_opts(&opts); req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Missing required argument"); return; } ct = cfiscsi_target_find(&cfiscsi_softc, target); if (ct == NULL) { ctl_free_opts(&opts); req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "can't find target \"%s\"", target); return; } if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) { ctl_free_opts(&opts); req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "target \"%s\" is already dying", target); return; } ctl_free_opts(&opts); ct->ct_state = CFISCSI_TARGET_STATE_DYING; ctl_port_offline(&ct->ct_port); cfiscsi_target_release(ct); cfiscsi_target_release(ct); } static int cfiscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td) { struct ctl_iscsi *ci; struct ctl_req *req; if (cmd == CTL_PORT_REQ) { req = (struct ctl_req *)addr; switch (req->reqtype) { case CTL_REQ_CREATE: cfiscsi_ioctl_port_create(req); break; case CTL_REQ_REMOVE: cfiscsi_ioctl_port_remove(req); break; default: req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Unsupported request type %d", req->reqtype); } return (0); } if (cmd != CTL_ISCSI) return (ENOTTY); ci = (struct ctl_iscsi *)addr; switch (ci->type) { case CTL_ISCSI_HANDOFF: cfiscsi_ioctl_handoff(ci); break; case CTL_ISCSI_LIST: cfiscsi_ioctl_list(ci); break; case CTL_ISCSI_TERMINATE: cfiscsi_ioctl_terminate(ci); break; case CTL_ISCSI_LOGOUT: cfiscsi_ioctl_logout(ci); break; #ifdef ICL_KERNEL_PROXY case CTL_ISCSI_LISTEN: cfiscsi_ioctl_listen(ci); break; case CTL_ISCSI_ACCEPT: cfiscsi_ioctl_accept(ci); break; case CTL_ISCSI_SEND: cfiscsi_ioctl_send(ci); break; case CTL_ISCSI_RECEIVE: cfiscsi_ioctl_receive(ci); break; #else case CTL_ISCSI_LISTEN: case CTL_ISCSI_ACCEPT: case CTL_ISCSI_SEND: case CTL_ISCSI_RECEIVE: ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "%s: CTL compiled without ICL_KERNEL_PROXY", __func__); break; #endif /* !ICL_KERNEL_PROXY */ default: ci->status = CTL_ISCSI_ERROR; snprintf(ci->error_str, sizeof(ci->error_str), "%s: invalid iSCSI request type %d", __func__, ci->type); break; } return (0); } static void cfiscsi_target_hold(struct cfiscsi_target *ct) { refcount_acquire(&ct->ct_refcount); } static void cfiscsi_target_release(struct cfiscsi_target *ct) { struct cfiscsi_softc *softc; softc = ct->ct_softc; mtx_lock(&softc->lock); if (refcount_release(&ct->ct_refcount)) { TAILQ_REMOVE(&softc->targets, ct, ct_next); mtx_unlock(&softc->lock); if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) { ct->ct_state = CFISCSI_TARGET_STATE_INVALID; if (ctl_port_deregister(&ct->ct_port) != 0) printf("%s: ctl_port_deregister() failed\n", __func__); } free(ct, M_CFISCSI); return; } mtx_unlock(&softc->lock); } static struct cfiscsi_target * cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name) { struct cfiscsi_target *ct; mtx_lock(&softc->lock); TAILQ_FOREACH(ct, &softc->targets, ct_next) { if (strcmp(name, ct->ct_name) != 0 || ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) continue; cfiscsi_target_hold(ct); mtx_unlock(&softc->lock); return (ct); } mtx_unlock(&softc->lock); return (NULL); } static struct cfiscsi_target * cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name, const char *alias) { struct cfiscsi_target *ct, *newct; int i; if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN) return (NULL); newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO); mtx_lock(&softc->lock); TAILQ_FOREACH(ct, &softc->targets, ct_next) { if (strcmp(name, ct->ct_name) != 0 || ct->ct_state == CFISCSI_TARGET_STATE_INVALID) continue; cfiscsi_target_hold(ct); mtx_unlock(&softc->lock); free(newct, M_CFISCSI); return (ct); } for (i = 0; i < CTL_MAX_LUNS; i++) newct->ct_luns[i] = UINT32_MAX; strlcpy(newct->ct_name, name, sizeof(newct->ct_name)); if (alias != NULL) strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias)); refcount_init(&newct->ct_refcount, 1); newct->ct_softc = softc; if (TAILQ_EMPTY(&softc->targets)) softc->last_target_id = 0; newct->ct_target_id = ++softc->last_target_id; TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next); mtx_unlock(&softc->lock); return (newct); } /* * Takes LUN from the target space and returns LUN from the CTL space. */ static uint32_t cfiscsi_lun_map(void *arg, uint32_t lun) { struct cfiscsi_target *ct = arg; if (lun >= CTL_MAX_LUNS) { CFISCSI_DEBUG("requested lun number %d is higher " "than maximum %d", lun, CTL_MAX_LUNS - 1); return (UINT32_MAX); } return (ct->ct_luns[lun]); } static int cfiscsi_target_set_lun(struct cfiscsi_target *ct, unsigned long lun_id, unsigned long ctl_lun_id) { if (lun_id >= CTL_MAX_LUNS) { CFISCSI_WARN("requested lun number %ld is higher " "than maximum %d", lun_id, CTL_MAX_LUNS - 1); return (-1); } if (ct->ct_luns[lun_id] < CTL_MAX_LUNS) { /* * CTL calls cfiscsi_lun_enable() twice for each LUN - once * when the LUN is created, and a second time just before * the port is brought online; don't emit warnings * for that case. */ if (ct->ct_luns[lun_id] == ctl_lun_id) return (0); CFISCSI_WARN("lun %ld already allocated", lun_id); return (-1); } #if 0 CFISCSI_DEBUG("adding mapping for lun %ld, target %s " "to ctl lun %ld", lun_id, ct->ct_name, ctl_lun_id); #endif ct->ct_luns[lun_id] = ctl_lun_id; return (0); } static int cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id) { struct cfiscsi_softc *softc; struct cfiscsi_target *ct; const char *target = NULL; const char *lun = NULL; unsigned long tmp; ct = (struct cfiscsi_target *)arg; softc = ct->ct_softc; target = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options, "cfiscsi_target"); lun = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options, "cfiscsi_lun"); if (target == NULL && lun == NULL) return (0); if (target == NULL || lun == NULL) { CFISCSI_WARN("lun added with cfiscsi_target, but without " "cfiscsi_lun, or the other way around; ignoring"); return (0); } if (strcmp(target, ct->ct_name) != 0) return (0); tmp = strtoul(lun, NULL, 10); cfiscsi_target_set_lun(ct, tmp, lun_id); return (0); } static int cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id) { struct cfiscsi_softc *softc; struct cfiscsi_target *ct; int i; ct = (struct cfiscsi_target *)arg; softc = ct->ct_softc; mtx_lock(&softc->lock); for (i = 0; i < CTL_MAX_LUNS; i++) { if (ct->ct_luns[i] != lun_id) continue; ct->ct_luns[i] = UINT32_MAX; break; } mtx_unlock(&softc->lock); return (0); } static void cfiscsi_datamove_in(union ctl_io *io) { struct cfiscsi_session *cs; struct icl_pdu *request, *response; const struct iscsi_bhs_scsi_command *bhssc; struct iscsi_bhs_data_in *bhsdi; struct ctl_sg_entry ctl_sg_entry, *ctl_sglist; size_t len, expected_len, sg_len, buffer_offset; const char *sg_addr; int ctl_sg_count, error, i; request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; cs = PDU_SESSION(request); bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs; KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_COMMAND, ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND")); if (io->scsiio.kern_sg_entries > 0) { ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr; ctl_sg_count = io->scsiio.kern_sg_entries; } else { ctl_sglist = &ctl_sg_entry; ctl_sglist->addr = io->scsiio.kern_data_ptr; ctl_sglist->len = io->scsiio.kern_data_len; ctl_sg_count = 1; } /* * This is the total amount of data to be transferred within the current * SCSI command. We need to record it so that we can properly report * underflow/underflow. */ PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len; /* * This is the offset within the current SCSI command; for the first * call to cfiscsi_datamove() it will be 0, and for subsequent ones * it will be the sum of lengths of previous ones. */ buffer_offset = io->scsiio.kern_rel_offset; /* * This is the transfer length expected by the initiator. In theory, * it could be different from the correct amount of data from the SCSI * point of view, even if that doesn't make any sense. */ expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length); #if 0 if (expected_len != io->scsiio.kern_total_len) { CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, " "actual length %zd", expected_len, (size_t)io->scsiio.kern_total_len); } #endif if (buffer_offset >= expected_len) { #if 0 CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, " "already sent the expected len", buffer_offset); #endif io->scsiio.be_move_done(io); return; } i = 0; sg_addr = NULL; sg_len = 0; response = NULL; bhsdi = NULL; for (;;) { if (response == NULL) { response = cfiscsi_pdu_new_response(request, M_NOWAIT); if (response == NULL) { CFISCSI_SESSION_WARN(cs, "failed to " "allocate memory; dropping connection"); ctl_set_busy(&io->scsiio); io->scsiio.be_move_done(io); cfiscsi_session_terminate(cs); return; } bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs; bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN; bhsdi->bhsdi_initiator_task_tag = bhssc->bhssc_initiator_task_tag; bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request)); PDU_EXPDATASN(request)++; bhsdi->bhsdi_buffer_offset = htonl(buffer_offset); } KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count")); if (sg_len == 0) { sg_addr = ctl_sglist[i].addr; sg_len = ctl_sglist[i].len; KASSERT(sg_len > 0, ("sg_len <= 0")); } len = sg_len; /* * Truncate to maximum data segment length. */ KASSERT(response->ip_data_len < cs->cs_max_data_segment_length, ("ip_data_len %zd >= max_data_segment_length %zd", response->ip_data_len, cs->cs_max_data_segment_length)); if (response->ip_data_len + len > cs->cs_max_data_segment_length) { len = cs->cs_max_data_segment_length - response->ip_data_len; KASSERT(len <= sg_len, ("len %zd > sg_len %zd", len, sg_len)); } /* * Truncate to expected data transfer length. */ KASSERT(buffer_offset + response->ip_data_len < expected_len, ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd", buffer_offset, response->ip_data_len, expected_len)); if (buffer_offset + response->ip_data_len + len > expected_len) { CFISCSI_SESSION_DEBUG(cs, "truncating from %zd " "to expected data transfer length %zd", buffer_offset + response->ip_data_len + len, expected_len); len = expected_len - (buffer_offset + response->ip_data_len); KASSERT(len <= sg_len, ("len %zd > sg_len %zd", len, sg_len)); } error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT); if (error != 0) { CFISCSI_SESSION_WARN(cs, "failed to " "allocate memory; dropping connection"); icl_pdu_free(response); ctl_set_busy(&io->scsiio); io->scsiio.be_move_done(io); cfiscsi_session_terminate(cs); return; } sg_addr += len; sg_len -= len; KASSERT(buffer_offset + response->ip_data_len <= expected_len, ("buffer_offset %zd + ip_data_len %zd > expected_len %zd", buffer_offset, response->ip_data_len, expected_len)); if (buffer_offset + response->ip_data_len == expected_len) { /* * Already have the amount of data the initiator wanted. */ break; } if (sg_len == 0) { /* * End of scatter-gather segment; * proceed to the next one... */ if (i == ctl_sg_count - 1) { /* * ... unless this was the last one. */ break; } i++; } if (response->ip_data_len == cs->cs_max_data_segment_length) { /* * Can't stuff more data into the current PDU; * queue it. Note that's not enough to check * for kern_data_resid == 0 instead; there * may be several Data-In PDUs for the final * call to cfiscsi_datamove(), and we want * to set the F flag only on the last of them. */ buffer_offset += response->ip_data_len; if (buffer_offset == io->scsiio.kern_total_len || buffer_offset == expected_len) { buffer_offset -= response->ip_data_len; break; } cfiscsi_pdu_queue(response); response = NULL; bhsdi = NULL; } } if (response != NULL) { buffer_offset += response->ip_data_len; if (buffer_offset == io->scsiio.kern_total_len || buffer_offset == expected_len) { bhsdi->bhsdi_flags |= BHSDI_FLAGS_F; if (io->io_hdr.status == CTL_SUCCESS) { bhsdi->bhsdi_flags |= BHSDI_FLAGS_S; if (PDU_TOTAL_TRANSFER_LEN(request) < ntohl(bhssc->bhssc_expected_data_transfer_length)) { bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW; bhsdi->bhsdi_residual_count = htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) - PDU_TOTAL_TRANSFER_LEN(request)); } else if (PDU_TOTAL_TRANSFER_LEN(request) > ntohl(bhssc->bhssc_expected_data_transfer_length)) { bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW; bhsdi->bhsdi_residual_count = htonl(PDU_TOTAL_TRANSFER_LEN(request) - ntohl(bhssc->bhssc_expected_data_transfer_length)); } bhsdi->bhsdi_status = io->scsiio.scsi_status; io->io_hdr.flags |= CTL_FLAG_STATUS_SENT; } } KASSERT(response->ip_data_len > 0, ("sending empty Data-In")); cfiscsi_pdu_queue(response); } io->scsiio.be_move_done(io); } static void cfiscsi_datamove_out(union ctl_io *io) { struct cfiscsi_session *cs; struct icl_pdu *request, *response; const struct iscsi_bhs_scsi_command *bhssc; struct iscsi_bhs_r2t *bhsr2t; struct cfiscsi_data_wait *cdw; struct ctl_sg_entry ctl_sg_entry, *ctl_sglist; uint32_t expected_len, r2t_off, r2t_len; uint32_t target_transfer_tag; bool done; request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; cs = PDU_SESSION(request); bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs; KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_COMMAND, ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND")); /* * We need to record it so that we can properly report * underflow/underflow. */ PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len; /* * Report write underflow as error since CTL and backends don't * really support it, and SCSI does not tell how to do it right. */ expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length); if (io->scsiio.kern_rel_offset + io->scsiio.kern_data_len > expected_len) { io->scsiio.io_hdr.port_status = 43; io->scsiio.be_move_done(io); return; } target_transfer_tag = atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1); #if 0 CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator " "task tag 0x%x, target transfer tag 0x%x", bhssc->bhssc_initiator_task_tag, target_transfer_tag); #endif cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO); if (cdw == NULL) { CFISCSI_SESSION_WARN(cs, "failed to " "allocate memory; dropping connection"); ctl_set_busy(&io->scsiio); io->scsiio.be_move_done(io); cfiscsi_session_terminate(cs); return; } cdw->cdw_ctl_io = io; cdw->cdw_target_transfer_tag = target_transfer_tag; cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag; cdw->cdw_r2t_end = io->scsiio.kern_data_len; cdw->cdw_datasn = 0; /* Set initial data pointer for the CDW respecting ext_data_filled. */ if (io->scsiio.kern_sg_entries > 0) { ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr; } else { ctl_sglist = &ctl_sg_entry; ctl_sglist->addr = io->scsiio.kern_data_ptr; ctl_sglist->len = io->scsiio.kern_data_len; } cdw->cdw_sg_index = 0; cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr; cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len; r2t_off = io->scsiio.ext_data_filled; while (r2t_off > 0) { if (r2t_off >= cdw->cdw_sg_len) { r2t_off -= cdw->cdw_sg_len; cdw->cdw_sg_index++; cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr; cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len; continue; } cdw->cdw_sg_addr += r2t_off; cdw->cdw_sg_len -= r2t_off; r2t_off = 0; } if (cs->cs_immediate_data && io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled < icl_pdu_data_segment_length(request)) { done = cfiscsi_handle_data_segment(request, cdw); if (done) { uma_zfree(cfiscsi_data_wait_zone, cdw); io->scsiio.be_move_done(io); return; } } r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled; r2t_len = MIN(io->scsiio.kern_data_len - io->scsiio.ext_data_filled, cs->cs_max_burst_length); cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len; CFISCSI_SESSION_LOCK(cs); TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next); CFISCSI_SESSION_UNLOCK(cs); /* * XXX: We should limit the number of outstanding R2T PDUs * per task to MaxOutstandingR2T. */ response = cfiscsi_pdu_new_response(request, M_NOWAIT); if (response == NULL) { CFISCSI_SESSION_WARN(cs, "failed to " "allocate memory; dropping connection"); ctl_set_busy(&io->scsiio); io->scsiio.be_move_done(io); cfiscsi_session_terminate(cs); return; } bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs; bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T; bhsr2t->bhsr2t_flags = 0x80; bhsr2t->bhsr2t_lun = bhssc->bhssc_lun; bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag; bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag; /* * XXX: Here we assume that cfiscsi_datamove() won't ever * be running concurrently on several CPUs for a given * command. */ bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request)); PDU_R2TSN(request)++; /* * This is the offset within the current SCSI command; * i.e. for the first call of datamove(), it will be 0, * and for subsequent ones it will be the sum of lengths * of previous ones. * * The ext_data_filled is to account for unsolicited * (immediate) data that might have already arrived. */ bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off); /* * This is the total length (sum of S/G lengths) this call * to cfiscsi_datamove() is supposed to handle, limited by * MaxBurstLength. */ bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len); cfiscsi_pdu_queue(response); } static void cfiscsi_datamove(union ctl_io *io) { if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) cfiscsi_datamove_in(io); else { /* We hadn't received anything during this datamove yet. */ io->scsiio.ext_data_filled = 0; cfiscsi_datamove_out(io); } } static void cfiscsi_scsi_command_done(union ctl_io *io) { struct icl_pdu *request, *response; struct iscsi_bhs_scsi_command *bhssc; struct iscsi_bhs_scsi_response *bhssr; #ifdef DIAGNOSTIC struct cfiscsi_data_wait *cdw; #endif struct cfiscsi_session *cs; uint16_t sense_length; request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; cs = PDU_SESSION(request); bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs; KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_SCSI_COMMAND, ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode)); //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x", // bhssc->bhssc_initiator_task_tag); #ifdef DIAGNOSTIC CFISCSI_SESSION_LOCK(cs); TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) KASSERT(bhssc->bhssc_initiator_task_tag != cdw->cdw_initiator_task_tag, ("dangling cdw")); CFISCSI_SESSION_UNLOCK(cs); #endif /* * Do not return status for aborted commands. * There are exceptions, but none supported by CTL yet. */ if (((io->io_hdr.flags & CTL_FLAG_ABORT) && (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) || (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) { ctl_free_io(io); icl_pdu_free(request); return; } response = cfiscsi_pdu_new_response(request, M_WAITOK); bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs; bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE; bhssr->bhssr_flags = 0x80; /* * XXX: We don't deal with bidirectional under/overflows; * does anything actually support those? */ if (PDU_TOTAL_TRANSFER_LEN(request) < ntohl(bhssc->bhssc_expected_data_transfer_length)) { bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW; bhssr->bhssr_residual_count = htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) - PDU_TOTAL_TRANSFER_LEN(request)); //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d", // ntohl(bhssr->bhssr_residual_count)); } else if (PDU_TOTAL_TRANSFER_LEN(request) > ntohl(bhssc->bhssc_expected_data_transfer_length)) { bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW; bhssr->bhssr_residual_count = htonl(PDU_TOTAL_TRANSFER_LEN(request) - ntohl(bhssc->bhssc_expected_data_transfer_length)); //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d", // ntohl(bhssr->bhssr_residual_count)); } bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED; bhssr->bhssr_status = io->scsiio.scsi_status; bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag; bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request)); if (io->scsiio.sense_len > 0) { #if 0 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data", io->scsiio.sense_len); #endif sense_length = htons(io->scsiio.sense_len); icl_pdu_append_data(response, &sense_length, sizeof(sense_length), M_WAITOK); icl_pdu_append_data(response, &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK); } ctl_free_io(io); icl_pdu_free(request); cfiscsi_pdu_queue(response); } static void cfiscsi_task_management_done(union ctl_io *io) { struct icl_pdu *request, *response; struct iscsi_bhs_task_management_request *bhstmr; struct iscsi_bhs_task_management_response *bhstmr2; struct cfiscsi_data_wait *cdw, *tmpcdw; struct cfiscsi_session *cs; request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; cs = PDU_SESSION(request); bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs; KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) == ISCSI_BHS_OPCODE_TASK_REQUEST, ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode)); #if 0 CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x", bhstmr->bhstmr_initiator_task_tag, bhstmr->bhstmr_referenced_task_tag); #endif if ((bhstmr->bhstmr_function & ~0x80) == BHSTMR_FUNCTION_ABORT_TASK) { /* * Make sure we no longer wait for Data-Out for this command. */ CFISCSI_SESSION_LOCK(cs); TAILQ_FOREACH_SAFE(cdw, &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) { if (bhstmr->bhstmr_referenced_task_tag != cdw->cdw_initiator_task_tag) continue; #if 0 CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task " "tag 0x%x", bhstmr->bhstmr_initiator_task_tag); #endif TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next); cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io); uma_zfree(cfiscsi_data_wait_zone, cdw); } CFISCSI_SESSION_UNLOCK(cs); } response = cfiscsi_pdu_new_response(request, M_WAITOK); bhstmr2 = (struct iscsi_bhs_task_management_response *) response->ip_bhs; bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE; bhstmr2->bhstmr_flags = 0x80; if (io->io_hdr.status == CTL_SUCCESS) { bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE; } else { /* * XXX: How to figure out what exactly went wrong? iSCSI spec * expects us to provide detailed error, e.g. "Task does * not exist" or "LUN does not exist". */ CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED"); bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED; } bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag; ctl_free_io(io); icl_pdu_free(request); cfiscsi_pdu_queue(response); } static void cfiscsi_done(union ctl_io *io) { struct icl_pdu *request; struct cfiscsi_session *cs; KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE), ("invalid CTL status %#x", io->io_hdr.status)); if (io->io_hdr.io_type == CTL_IO_TASK && io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) { /* * Implicit task termination has just completed; nothing to do. */ cs = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; cs->cs_tasks_aborted = true; refcount_release(&cs->cs_outstanding_ctl_pdus); wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus)); ctl_free_io(io); return; } request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; cs = PDU_SESSION(request); refcount_release(&cs->cs_outstanding_ctl_pdus); switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) { case ISCSI_BHS_OPCODE_SCSI_COMMAND: cfiscsi_scsi_command_done(io); break; case ISCSI_BHS_OPCODE_TASK_REQUEST: cfiscsi_task_management_done(io); break; default: panic("cfiscsi_done called with wrong opcode 0x%x", request->ip_bhs->bhs_opcode); } } diff --git a/sys/conf/files b/sys/conf/files index ee8a5381be9c..38f96a29e4b4 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1,4013 +1,4014 @@ # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # acpi_quirks.h optional acpi \ dependency "$S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ compile-with "${AWK} -f $S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ no-obj no-implicit-rule before-depend \ clean "acpi_quirks.h" # # The 'fdt_dtb_file' target covers an actual DTB file name, which is derived # from the specified source (DTS) file: .dts -> .dtb # fdt_dtb_file optional fdt fdt_dtb_static \ compile-with "sh -c 'MACHINE=${MACHINE} $S/tools/fdt/make_dtb.sh $S ${FDT_DTS_FILE} ${.CURDIR}'" \ no-obj no-implicit-rule before-depend \ clean "${FDT_DTS_FILE:R}.dtb" fdt_static_dtb.h optional fdt fdt_dtb_static \ compile-with "sh -c 'MACHINE=${MACHINE} $S/tools/fdt/make_dtbh.sh ${FDT_DTS_FILE} ${.CURDIR}'" \ dependency "fdt_dtb_file" \ no-obj no-implicit-rule before-depend \ clean "fdt_static_dtb.h" feeder_eq_gen.h optional sound \ dependency "$S/tools/sound/feeder_eq_mkfilter.awk" \ compile-with "${AWK} -f $S/tools/sound/feeder_eq_mkfilter.awk -- ${FEEDER_EQ_PRESETS} > feeder_eq_gen.h" \ no-obj no-implicit-rule before-depend \ clean "feeder_eq_gen.h" feeder_rate_gen.h optional sound \ dependency "$S/tools/sound/feeder_rate_mkfilter.awk" \ compile-with "${AWK} -f $S/tools/sound/feeder_rate_mkfilter.awk -- ${FEEDER_RATE_PRESETS} > feeder_rate_gen.h" \ no-obj no-implicit-rule before-depend \ clean "feeder_rate_gen.h" snd_fxdiv_gen.h optional sound \ dependency "$S/tools/sound/snd_fxdiv_gen.awk" \ compile-with "${AWK} -f $S/tools/sound/snd_fxdiv_gen.awk -- > snd_fxdiv_gen.h" \ no-obj no-implicit-rule before-depend \ clean "snd_fxdiv_gen.h" miidevs.h optional miibus | mii \ dependency "$S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ compile-with "${AWK} -f $S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ no-obj no-implicit-rule before-depend \ clean "miidevs.h" pccarddevs.h standard \ dependency "$S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ compile-with "${AWK} -f $S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ no-obj no-implicit-rule before-depend \ clean "pccarddevs.h" teken_state.h optional sc | vt \ dependency "$S/teken/gensequences $S/teken/sequences" \ compile-with "${AWK} -f $S/teken/gensequences $S/teken/sequences > teken_state.h" \ no-obj no-implicit-rule before-depend \ clean "teken_state.h" usbdevs.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -h" \ no-obj no-implicit-rule before-depend \ clean "usbdevs.h" usbdevs_data.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -d" \ no-obj no-implicit-rule before-depend \ clean "usbdevs_data.h" cam/cam.c optional scbus cam/cam_compat.c optional scbus cam/cam_periph.c optional scbus cam/cam_queue.c optional scbus cam/cam_sim.c optional scbus cam/cam_xpt.c optional scbus cam/ata/ata_all.c optional scbus cam/ata/ata_xpt.c optional scbus cam/ata/ata_pmp.c optional scbus cam/scsi/scsi_xpt.c optional scbus cam/scsi/scsi_all.c optional scbus cam/scsi/scsi_cd.c optional cd cam/scsi/scsi_ch.c optional ch cam/ata/ata_da.c optional ada | da cam/ctl/ctl.c optional ctl cam/ctl/ctl_backend.c optional ctl cam/ctl/ctl_backend_block.c optional ctl cam/ctl/ctl_backend_ramdisk.c optional ctl cam/ctl/ctl_cmd_table.c optional ctl cam/ctl/ctl_frontend.c optional ctl cam/ctl/ctl_frontend_cam_sim.c optional ctl cam/ctl/ctl_frontend_internal.c optional ctl cam/ctl/ctl_frontend_iscsi.c optional ctl cam/ctl/ctl_scsi_all.c optional ctl cam/ctl/ctl_tpc.c optional ctl cam/ctl/ctl_tpc_local.c optional ctl cam/ctl/ctl_error.c optional ctl cam/ctl/ctl_util.c optional ctl cam/ctl/scsi_ctl.c optional ctl cam/scsi/scsi_da.c optional da cam/scsi/scsi_low.c optional ct | ncv | nsp | stg cam/scsi/scsi_pass.c optional pass cam/scsi/scsi_pt.c optional pt cam/scsi/scsi_sa.c optional sa cam/scsi/scsi_enc.c optional ses cam/scsi/scsi_enc_ses.c optional ses cam/scsi/scsi_enc_safte.c optional ses cam/scsi/scsi_sg.c optional sg cam/scsi/scsi_targ_bh.c optional targbh cam/scsi/scsi_target.c optional targ cam/scsi/smp_all.c optional scbus # shared between zfs and dtrace cddl/compat/opensolaris/kern/opensolaris.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_cmn_err.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_kmem.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_misc.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_sunddi.c optional zfs compile-with "${ZFS_C}" # zfs specific cddl/compat/opensolaris/kern/opensolaris_acl.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_dtrace.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_kobj.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_kstat.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_lookup.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_policy.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_string.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_sysevent.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_taskq.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_uio.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_vfs.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_vm.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_zone.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/acl/acl_common.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/avl/avl.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/nvpair/fnvpair.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/nvpair/nvpair.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/nvpair/nvpair_alloc_fixed.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/unicode/u8_textprep.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfeature_common.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_comutil.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_deleg.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_fletcher.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_namecheck.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_prop.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zpool_prop.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zprop_common.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/gfs.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/vnode.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/blkptr.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/bptree.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_diff.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_object.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_zfetch.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c optional zfs compile-with "${ZFS_C}" \ warning "kernel contains CDDL licensed ZFS filesystem" cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_bookmark.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deadlist.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deleg.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_userhold.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_synctask.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/gzip.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lzjb.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/sha256.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_errlog.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_history.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/space_map.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/uberblock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/unique.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_cache.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_root.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zap_micro.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_byteswap.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_debug.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fm.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fuid.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_onexit.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_sa.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio_inject.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zle.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/callb.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/fm.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/list.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/adler32.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/deflate.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/inffast.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/inflate.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/inftrees.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/opensolaris_crc32.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/trees.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/zmod.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/zmod_subr.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/zutil.c optional zfs compile-with "${ZFS_C}" compat/freebsd32/freebsd32_capability.c optional compat_freebsd32 compat/freebsd32/freebsd32_ioctl.c optional compat_freebsd32 compat/freebsd32/freebsd32_misc.c optional compat_freebsd32 compat/freebsd32/freebsd32_syscalls.c optional compat_freebsd32 compat/freebsd32/freebsd32_sysent.c optional compat_freebsd32 contrib/altq/altq/altq_cbq.c optional altq contrib/altq/altq/altq_cdnr.c optional altq contrib/altq/altq/altq_hfsc.c optional altq contrib/altq/altq/altq_priq.c optional altq contrib/altq/altq/altq_red.c optional altq contrib/altq/altq/altq_rio.c optional altq contrib/altq/altq/altq_rmclass.c optional altq contrib/altq/altq/altq_subr.c optional altq contrib/dev/acpica/common/ahids.c optional acpi acpi_debug contrib/dev/acpica/common/ahuuids.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbcmds.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbconvert.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbdisply.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbexec.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbfileio.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbhistry.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbinput.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbmethod.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbnames.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbstats.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbtest.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbutils.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbxface.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmbuffer.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmdeferred.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmnames.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmopcode.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmobject.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrc.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrcl.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrcl2.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrcs.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmutils.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmwalk.c optional acpi acpi_debug contrib/dev/acpica/components/dispatcher/dsargs.c optional acpi contrib/dev/acpica/components/dispatcher/dscontrol.c optional acpi contrib/dev/acpica/components/dispatcher/dsfield.c optional acpi contrib/dev/acpica/components/dispatcher/dsinit.c optional acpi contrib/dev/acpica/components/dispatcher/dsmethod.c optional acpi contrib/dev/acpica/components/dispatcher/dsmthdat.c optional acpi contrib/dev/acpica/components/dispatcher/dsobject.c optional acpi contrib/dev/acpica/components/dispatcher/dsopcode.c optional acpi contrib/dev/acpica/components/dispatcher/dsutils.c optional acpi contrib/dev/acpica/components/dispatcher/dswexec.c optional acpi contrib/dev/acpica/components/dispatcher/dswload.c optional acpi contrib/dev/acpica/components/dispatcher/dswload2.c optional acpi contrib/dev/acpica/components/dispatcher/dswscope.c optional acpi contrib/dev/acpica/components/dispatcher/dswstate.c optional acpi contrib/dev/acpica/components/events/evevent.c optional acpi contrib/dev/acpica/components/events/evglock.c optional acpi contrib/dev/acpica/components/events/evgpe.c optional acpi contrib/dev/acpica/components/events/evgpeblk.c optional acpi contrib/dev/acpica/components/events/evgpeinit.c optional acpi contrib/dev/acpica/components/events/evgpeutil.c optional acpi contrib/dev/acpica/components/events/evhandler.c optional acpi contrib/dev/acpica/components/events/evmisc.c optional acpi contrib/dev/acpica/components/events/evregion.c optional acpi contrib/dev/acpica/components/events/evrgnini.c optional acpi contrib/dev/acpica/components/events/evsci.c optional acpi contrib/dev/acpica/components/events/evxface.c optional acpi contrib/dev/acpica/components/events/evxfevnt.c optional acpi contrib/dev/acpica/components/events/evxfgpe.c optional acpi contrib/dev/acpica/components/events/evxfregn.c optional acpi contrib/dev/acpica/components/executer/exconfig.c optional acpi contrib/dev/acpica/components/executer/exconvrt.c optional acpi contrib/dev/acpica/components/executer/excreate.c optional acpi contrib/dev/acpica/components/executer/exdebug.c optional acpi contrib/dev/acpica/components/executer/exdump.c optional acpi contrib/dev/acpica/components/executer/exfield.c optional acpi contrib/dev/acpica/components/executer/exfldio.c optional acpi contrib/dev/acpica/components/executer/exmisc.c optional acpi contrib/dev/acpica/components/executer/exmutex.c optional acpi contrib/dev/acpica/components/executer/exnames.c optional acpi contrib/dev/acpica/components/executer/exoparg1.c optional acpi contrib/dev/acpica/components/executer/exoparg2.c optional acpi contrib/dev/acpica/components/executer/exoparg3.c optional acpi contrib/dev/acpica/components/executer/exoparg6.c optional acpi contrib/dev/acpica/components/executer/exprep.c optional acpi contrib/dev/acpica/components/executer/exregion.c optional acpi contrib/dev/acpica/components/executer/exresnte.c optional acpi contrib/dev/acpica/components/executer/exresolv.c optional acpi contrib/dev/acpica/components/executer/exresop.c optional acpi contrib/dev/acpica/components/executer/exstore.c optional acpi contrib/dev/acpica/components/executer/exstoren.c optional acpi contrib/dev/acpica/components/executer/exstorob.c optional acpi contrib/dev/acpica/components/executer/exsystem.c optional acpi contrib/dev/acpica/components/executer/exutils.c optional acpi contrib/dev/acpica/components/hardware/hwacpi.c optional acpi contrib/dev/acpica/components/hardware/hwesleep.c optional acpi contrib/dev/acpica/components/hardware/hwgpe.c optional acpi contrib/dev/acpica/components/hardware/hwpci.c optional acpi contrib/dev/acpica/components/hardware/hwregs.c optional acpi contrib/dev/acpica/components/hardware/hwsleep.c optional acpi contrib/dev/acpica/components/hardware/hwtimer.c optional acpi contrib/dev/acpica/components/hardware/hwvalid.c optional acpi contrib/dev/acpica/components/hardware/hwxface.c optional acpi contrib/dev/acpica/components/hardware/hwxfsleep.c optional acpi contrib/dev/acpica/components/namespace/nsaccess.c optional acpi contrib/dev/acpica/components/namespace/nsalloc.c optional acpi contrib/dev/acpica/components/namespace/nsarguments.c optional acpi contrib/dev/acpica/components/namespace/nsconvert.c optional acpi contrib/dev/acpica/components/namespace/nsdump.c optional acpi contrib/dev/acpica/components/namespace/nseval.c optional acpi contrib/dev/acpica/components/namespace/nsinit.c optional acpi contrib/dev/acpica/components/namespace/nsload.c optional acpi contrib/dev/acpica/components/namespace/nsnames.c optional acpi contrib/dev/acpica/components/namespace/nsobject.c optional acpi contrib/dev/acpica/components/namespace/nsparse.c optional acpi contrib/dev/acpica/components/namespace/nspredef.c optional acpi contrib/dev/acpica/components/namespace/nsprepkg.c optional acpi contrib/dev/acpica/components/namespace/nsrepair.c optional acpi contrib/dev/acpica/components/namespace/nsrepair2.c optional acpi contrib/dev/acpica/components/namespace/nssearch.c optional acpi contrib/dev/acpica/components/namespace/nsutils.c optional acpi contrib/dev/acpica/components/namespace/nswalk.c optional acpi contrib/dev/acpica/components/namespace/nsxfeval.c optional acpi contrib/dev/acpica/components/namespace/nsxfname.c optional acpi contrib/dev/acpica/components/namespace/nsxfobj.c optional acpi contrib/dev/acpica/components/parser/psargs.c optional acpi contrib/dev/acpica/components/parser/psloop.c optional acpi contrib/dev/acpica/components/parser/psobject.c optional acpi contrib/dev/acpica/components/parser/psopcode.c optional acpi contrib/dev/acpica/components/parser/psopinfo.c optional acpi contrib/dev/acpica/components/parser/psparse.c optional acpi contrib/dev/acpica/components/parser/psscope.c optional acpi contrib/dev/acpica/components/parser/pstree.c optional acpi contrib/dev/acpica/components/parser/psutils.c optional acpi contrib/dev/acpica/components/parser/pswalk.c optional acpi contrib/dev/acpica/components/parser/psxface.c optional acpi contrib/dev/acpica/components/resources/rsaddr.c optional acpi contrib/dev/acpica/components/resources/rscalc.c optional acpi contrib/dev/acpica/components/resources/rscreate.c optional acpi contrib/dev/acpica/components/resources/rsdump.c optional acpi contrib/dev/acpica/components/resources/rsdumpinfo.c optional acpi contrib/dev/acpica/components/resources/rsinfo.c optional acpi contrib/dev/acpica/components/resources/rsio.c optional acpi contrib/dev/acpica/components/resources/rsirq.c optional acpi contrib/dev/acpica/components/resources/rslist.c optional acpi contrib/dev/acpica/components/resources/rsmemory.c optional acpi contrib/dev/acpica/components/resources/rsmisc.c optional acpi contrib/dev/acpica/components/resources/rsserial.c optional acpi contrib/dev/acpica/components/resources/rsutils.c optional acpi contrib/dev/acpica/components/resources/rsxface.c optional acpi contrib/dev/acpica/components/tables/tbdata.c optional acpi contrib/dev/acpica/components/tables/tbfadt.c optional acpi contrib/dev/acpica/components/tables/tbfind.c optional acpi contrib/dev/acpica/components/tables/tbinstal.c optional acpi contrib/dev/acpica/components/tables/tbprint.c optional acpi contrib/dev/acpica/components/tables/tbutils.c optional acpi contrib/dev/acpica/components/tables/tbxface.c optional acpi contrib/dev/acpica/components/tables/tbxfload.c optional acpi contrib/dev/acpica/components/tables/tbxfroot.c optional acpi contrib/dev/acpica/components/utilities/utaddress.c optional acpi contrib/dev/acpica/components/utilities/utalloc.c optional acpi contrib/dev/acpica/components/utilities/utbuffer.c optional acpi contrib/dev/acpica/components/utilities/utcache.c optional acpi contrib/dev/acpica/components/utilities/utcopy.c optional acpi contrib/dev/acpica/components/utilities/utdebug.c optional acpi contrib/dev/acpica/components/utilities/utdecode.c optional acpi contrib/dev/acpica/components/utilities/utdelete.c optional acpi contrib/dev/acpica/components/utilities/uterror.c optional acpi contrib/dev/acpica/components/utilities/uteval.c optional acpi contrib/dev/acpica/components/utilities/utexcep.c optional acpi contrib/dev/acpica/components/utilities/utglobal.c optional acpi contrib/dev/acpica/components/utilities/uthex.c optional acpi contrib/dev/acpica/components/utilities/utids.c optional acpi contrib/dev/acpica/components/utilities/utinit.c optional acpi contrib/dev/acpica/components/utilities/utlock.c optional acpi contrib/dev/acpica/components/utilities/utmath.c optional acpi contrib/dev/acpica/components/utilities/utmisc.c optional acpi contrib/dev/acpica/components/utilities/utmutex.c optional acpi contrib/dev/acpica/components/utilities/utobject.c optional acpi contrib/dev/acpica/components/utilities/utosi.c optional acpi contrib/dev/acpica/components/utilities/utownerid.c optional acpi contrib/dev/acpica/components/utilities/utpredef.c optional acpi contrib/dev/acpica/components/utilities/utresrc.c optional acpi contrib/dev/acpica/components/utilities/utstate.c optional acpi contrib/dev/acpica/components/utilities/utstring.c optional acpi contrib/dev/acpica/components/utilities/utuuid.c optional acpi acpi_debug contrib/dev/acpica/components/utilities/utxface.c optional acpi contrib/dev/acpica/components/utilities/utxferror.c optional acpi contrib/dev/acpica/components/utilities/utxfinit.c optional acpi #contrib/dev/acpica/components/utilities/utxfmutex.c optional acpi contrib/ipfilter/netinet/fil.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_auth.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_fil_freebsd.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_frag.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_log.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_nat.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_proxy.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_state.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_lookup.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -Wno-error -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_pool.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_htable.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_sync.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/mlfk_ipl.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_nat6.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_rules.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_scan.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_dstlist.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/radix_ipf.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/libfdt/fdt.c optional fdt contrib/libfdt/fdt_ro.c optional fdt contrib/libfdt/fdt_rw.c optional fdt contrib/libfdt/fdt_strerror.c optional fdt contrib/libfdt/fdt_sw.c optional fdt contrib/libfdt/fdt_wip.c optional fdt contrib/ngatm/netnatm/api/cc_conn.c optional ngatm_ccatm \ compile-with "${NORMAL_C_NOWERROR} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_data.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_dump.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_port.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_sig.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_user.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/unisap.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/straddr.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/unimsg_common.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/traffic.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_ie.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_msg.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_call.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_coord.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_party.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_print.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_reset.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_unimsgcpy.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_verify.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" crypto/blowfish/bf_ecb.c optional ipsec crypto/blowfish/bf_skey.c optional crypto | ipsec crypto/camellia/camellia.c optional crypto | ipsec crypto/camellia/camellia-api.c optional crypto | ipsec crypto/des/des_ecb.c optional crypto | ipsec | netsmb crypto/des/des_setkey.c optional crypto | ipsec | netsmb crypto/rc4/rc4.c optional netgraph_mppc_encryption | kgssapi crypto/rijndael/rijndael-alg-fst.c optional crypto | geom_bde | \ ipsec | random | wlan_ccmp crypto/rijndael/rijndael-api-fst.c optional geom_bde | random crypto/rijndael/rijndael-api.c optional crypto | ipsec | wlan_ccmp crypto/sha1.c optional carp | crypto | ipsec | \ netgraph_mppc_encryption | sctp crypto/sha2/sha2.c optional crypto | geom_bde | ipsec | random | \ sctp | zfs crypto/sha2/sha256c.c optional crypto | geom_bde | ipsec | random | \ sctp | zfs crypto/siphash/siphash.c optional inet | inet6 crypto/siphash/siphash_test.c optional inet | inet6 ddb/db_access.c optional ddb ddb/db_break.c optional ddb ddb/db_capture.c optional ddb ddb/db_command.c optional ddb ddb/db_examine.c optional ddb ddb/db_expr.c optional ddb ddb/db_input.c optional ddb ddb/db_lex.c optional ddb ddb/db_main.c optional ddb ddb/db_output.c optional ddb ddb/db_print.c optional ddb ddb/db_ps.c optional ddb ddb/db_run.c optional ddb ddb/db_script.c optional ddb ddb/db_sym.c optional ddb ddb/db_thread.c optional ddb ddb/db_textdump.c optional ddb ddb/db_variables.c optional ddb ddb/db_watch.c optional ddb ddb/db_write_cmd.c optional ddb dev/aac/aac.c optional aac dev/aac/aac_cam.c optional aacp aac dev/aac/aac_debug.c optional aac dev/aac/aac_disk.c optional aac dev/aac/aac_linux.c optional aac compat_linux dev/aac/aac_pci.c optional aac pci dev/aacraid/aacraid.c optional aacraid dev/aacraid/aacraid_cam.c optional aacraid scbus dev/aacraid/aacraid_debug.c optional aacraid dev/aacraid/aacraid_linux.c optional aacraid compat_linux dev/aacraid/aacraid_pci.c optional aacraid pci dev/acpi_support/acpi_wmi.c optional acpi_wmi acpi dev/acpi_support/acpi_asus.c optional acpi_asus acpi dev/acpi_support/acpi_asus_wmi.c optional acpi_asus_wmi acpi dev/acpi_support/acpi_fujitsu.c optional acpi_fujitsu acpi dev/acpi_support/acpi_hp.c optional acpi_hp acpi dev/acpi_support/acpi_ibm.c optional acpi_ibm acpi dev/acpi_support/acpi_panasonic.c optional acpi_panasonic acpi dev/acpi_support/acpi_sony.c optional acpi_sony acpi dev/acpi_support/acpi_toshiba.c optional acpi_toshiba acpi dev/acpi_support/atk0110.c optional aibs acpi dev/acpica/Osd/OsdDebug.c optional acpi dev/acpica/Osd/OsdHardware.c optional acpi dev/acpica/Osd/OsdInterrupt.c optional acpi dev/acpica/Osd/OsdMemory.c optional acpi dev/acpica/Osd/OsdSchedule.c optional acpi dev/acpica/Osd/OsdStream.c optional acpi dev/acpica/Osd/OsdSynch.c optional acpi dev/acpica/Osd/OsdTable.c optional acpi dev/acpica/acpi.c optional acpi dev/acpica/acpi_acad.c optional acpi dev/acpica/acpi_battery.c optional acpi dev/acpica/acpi_button.c optional acpi dev/acpica/acpi_cmbat.c optional acpi dev/acpica/acpi_cpu.c optional acpi dev/acpica/acpi_ec.c optional acpi dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_isab.c optional acpi isa dev/acpica/acpi_lid.c optional acpi dev/acpica/acpi_package.c optional acpi dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_perf.c optional acpi dev/acpica/acpi_powerres.c optional acpi dev/acpica/acpi_quirk.c optional acpi dev/acpica/acpi_resource.c optional acpi dev/acpica/acpi_smbat.c optional acpi dev/acpica/acpi_thermal.c optional acpi dev/acpica/acpi_throttle.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/acpica/acpi_video.c optional acpi_video acpi dev/acpica/acpi_dock.c optional acpi_dock acpi dev/adlink/adlink.c optional adlink dev/advansys/adv_eisa.c optional adv eisa dev/advansys/adv_pci.c optional adv pci dev/advansys/advansys.c optional adv dev/advansys/advlib.c optional adv dev/advansys/advmcode.c optional adv dev/advansys/adw_pci.c optional adw pci dev/advansys/adwcam.c optional adw dev/advansys/adwlib.c optional adw dev/advansys/adwmcode.c optional adw dev/ae/if_ae.c optional ae pci dev/age/if_age.c optional age pci dev/agp/agp.c optional agp pci dev/agp/agp_if.m optional agp pci dev/aha/aha.c optional aha dev/aha/aha_isa.c optional aha isa dev/aha/aha_mca.c optional aha mca dev/ahb/ahb.c optional ahb eisa dev/ahci/ahci.c optional ahci dev/ahci/ahciem.c optional ahci dev/ahci/ahci_pci.c optional ahci pci dev/aic/aic.c optional aic dev/aic/aic_pccard.c optional aic pccard dev/aic7xxx/ahc_eisa.c optional ahc eisa dev/aic7xxx/ahc_isa.c optional ahc isa dev/aic7xxx/ahc_pci.c optional ahc pci \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/aic7xxx/ahd_pci.c optional ahd pci \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/aic7xxx/aic7770.c optional ahc dev/aic7xxx/aic79xx.c optional ahd pci dev/aic7xxx/aic79xx_osm.c optional ahd pci dev/aic7xxx/aic79xx_pci.c optional ahd pci dev/aic7xxx/aic79xx_reg_print.c optional ahd pci ahd_reg_pretty_print dev/aic7xxx/aic7xxx.c optional ahc dev/aic7xxx/aic7xxx_93cx6.c optional ahc dev/aic7xxx/aic7xxx_osm.c optional ahc dev/aic7xxx/aic7xxx_pci.c optional ahc pci dev/aic7xxx/aic7xxx_reg_print.c optional ahc ahc_reg_pretty_print dev/alc/if_alc.c optional alc pci dev/ale/if_ale.c optional ale pci dev/alpm/alpm.c optional alpm pci dev/altera/avgen/altera_avgen.c optional altera_avgen dev/altera/avgen/altera_avgen_fdt.c optional altera_avgen fdt dev/altera/avgen/altera_avgen_nexus.c optional altera_avgen dev/altera/sdcard/altera_sdcard.c optional altera_sdcard dev/altera/sdcard/altera_sdcard_disk.c optional altera_sdcard dev/altera/sdcard/altera_sdcard_io.c optional altera_sdcard dev/altera/sdcard/altera_sdcard_fdt.c optional altera_sdcard fdt dev/altera/sdcard/altera_sdcard_nexus.c optional altera_sdcard dev/altera/pio/pio.c optional altera_pio dev/altera/pio/pio_if.m optional altera_pio dev/amdpm/amdpm.c optional amdpm pci | nfpm pci dev/amdsmb/amdsmb.c optional amdsmb pci dev/amr/amr.c optional amr dev/amr/amr_cam.c optional amrp amr dev/amr/amr_disk.c optional amr dev/amr/amr_linux.c optional amr compat_linux dev/amr/amr_pci.c optional amr pci dev/an/if_an.c optional an dev/an/if_an_isa.c optional an isa dev/an/if_an_pccard.c optional an pccard dev/an/if_an_pci.c optional an pci # dev/ata/ata_if.m optional ata | atacore dev/ata/ata-all.c optional ata | atacore dev/ata/ata-dma.c optional ata | atacore dev/ata/ata-lowlevel.c optional ata | atacore dev/ata/ata-sata.c optional ata | atacore dev/ata/ata-card.c optional ata pccard | atapccard dev/ata/ata-cbus.c optional ata pc98 | atapc98 dev/ata/ata-isa.c optional ata isa | ataisa dev/ata/ata-pci.c optional ata pci | atapci dev/ata/chipsets/ata-ahci.c optional ata pci | ataahci | ataacerlabs | \ ataati | ataintel | atajmicron | \ atavia | atanvidia dev/ata/chipsets/ata-acard.c optional ata pci | ataacard dev/ata/chipsets/ata-acerlabs.c optional ata pci | ataacerlabs dev/ata/chipsets/ata-adaptec.c optional ata pci | ataadaptec dev/ata/chipsets/ata-amd.c optional ata pci | ataamd dev/ata/chipsets/ata-ati.c optional ata pci | ataati dev/ata/chipsets/ata-cenatek.c optional ata pci | atacenatek dev/ata/chipsets/ata-cypress.c optional ata pci | atacypress dev/ata/chipsets/ata-cyrix.c optional ata pci | atacyrix dev/ata/chipsets/ata-highpoint.c optional ata pci | atahighpoint dev/ata/chipsets/ata-intel.c optional ata pci | ataintel dev/ata/chipsets/ata-ite.c optional ata pci | ataite dev/ata/chipsets/ata-jmicron.c optional ata pci | atajmicron dev/ata/chipsets/ata-marvell.c optional ata pci | atamarvell | ataadaptec dev/ata/chipsets/ata-micron.c optional ata pci | atamicron dev/ata/chipsets/ata-national.c optional ata pci | atanational dev/ata/chipsets/ata-netcell.c optional ata pci | atanetcell dev/ata/chipsets/ata-nvidia.c optional ata pci | atanvidia dev/ata/chipsets/ata-promise.c optional ata pci | atapromise dev/ata/chipsets/ata-serverworks.c optional ata pci | ataserverworks dev/ata/chipsets/ata-siliconimage.c optional ata pci | atasiliconimage | ataati dev/ata/chipsets/ata-sis.c optional ata pci | atasis dev/ata/chipsets/ata-via.c optional ata pci | atavia # dev/ath/if_ath_pci.c optional ath_pci pci \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/if_ath_ahb.c optional ath_ahb \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/if_ath.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_alq.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_beacon.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_btcoex.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_debug.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_keycache.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_led.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_lna_div.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx_edma.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx_ht.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tdma.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_sysctl.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_rx.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_rx_edma.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_spectral.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ah_osdep.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/ath_hal/ah.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v1.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v3.c optional ath_hal | ath_ar5211 | ath_ar5212 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v14.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v4k.c \ optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_9287.c \ optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_regdomain.c optional ath \ compile-with "${NORMAL_C} ${NO_WSHIFT_COUNT_NEGATIVE} ${NO_WSHIFT_COUNT_OVERFLOW} -I$S/dev/ath" # ar5210 dev/ath/ath_hal/ar5210/ar5210_attach.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_beacon.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_interrupts.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_keycache.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_misc.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_phy.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_power.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_recv.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_reset.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_xmit.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5211 dev/ath/ath_hal/ar5211/ar5211_attach.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_beacon.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_interrupts.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_keycache.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_misc.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_phy.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_power.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_recv.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_reset.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_xmit.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5212 dev/ath/ath_hal/ar5212/ar5212_ani.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_attach.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_beacon.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_eeprom.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_gpio.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_interrupts.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_keycache.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_misc.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_phy.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_power.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_recv.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_reset.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_rfgain.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_xmit.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5416 (depends on ar5212) dev/ath/ath_hal/ar5416/ar5416_ani.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_attach.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_beacon.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_btcoex.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_iq.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_adcgain.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_adcdc.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_eeprom.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_gpio.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_interrupts.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_keycache.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_misc.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_phy.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_power.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_radar.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_recv.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_reset.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_spectral.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_xmit.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9130 (depends upon ar5416) - also requires AH_SUPPORT_AR9130 # # Since this is an embedded MAC SoC, there's no need to compile it into the # default HAL. dev/ath/ath_hal/ar9001/ar9130_attach.c optional ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9001/ar9130_phy.c optional ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9001/ar9130_eeprom.c optional ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9160 (depends on ar5416) dev/ath/ath_hal/ar9001/ar9160_attach.c optional ath_hal | ath_ar9160 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9280 (depends on ar5416) dev/ath/ath_hal/ar9002/ar9280_attach.c optional ath_hal | ath_ar9280 | \ ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9280_olc.c optional ath_hal | ath_ar9280 | \ ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9285 (depends on ar5416 and ar9280) dev/ath/ath_hal/ar9002/ar9285_attach.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_btcoex.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_reset.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_cal.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_phy.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_diversity.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9287 (depends on ar5416) dev/ath/ath_hal/ar9002/ar9287_attach.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_reset.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_cal.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_olc.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9300 contrib/dev/ath/ath_hal/ar9300/ar9300_ani.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_attach.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_beacon.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_eeprom.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WCONSTANT_CONVERSION}" contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_gpio.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_interrupts.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_keycache.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_mci.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_misc.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_paprd.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_phy.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_power.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_radar.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_radio.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_recv.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_recv_ds.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_reset.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WSOMETIMES_UNINITIALIZED} -Wno-unused-function" contrib/dev/ath/ath_hal/ar9300/ar9300_stub.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_stub_funcs.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_timer.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_xmit.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_xmit_ds.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" # rf backends dev/ath/ath_hal/ar5212/ar2316.c optional ath_rf2316 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2317.c optional ath_rf2317 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2413.c optional ath_hal | ath_rf2413 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2425.c optional ath_hal | ath_rf2425 | ath_rf2417 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5111.c optional ath_hal | ath_rf5111 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5112.c optional ath_hal | ath_rf5112 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5413.c optional ath_hal | ath_rf5413 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar2133.c optional ath_hal | ath_ar5416 | \ ath_ar9130 | ath_ar9160 | ath_ar9280 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9280.c optional ath_hal | ath_ar9280 | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ath rate control algorithms dev/ath/ath_rate/amrr/amrr.c optional ath_rate_amrr \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/onoe/onoe.c optional ath_rate_onoe \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/sample/sample.c optional ath_rate_sample \ compile-with "${NORMAL_C} -I$S/dev/ath" # ath DFS modules dev/ath/ath_dfs/null/dfs_null.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/bce/if_bce.c optional bce dev/bfe/if_bfe.c optional bfe dev/bge/if_bge.c optional bge dev/bktr/bktr_audio.c optional bktr pci dev/bktr/bktr_card.c optional bktr pci dev/bktr/bktr_core.c optional bktr pci dev/bktr/bktr_i2c.c optional bktr pci smbus dev/bktr/bktr_os.c optional bktr pci dev/bktr/bktr_tuner.c optional bktr pci dev/bktr/msp34xx.c optional bktr pci dev/buslogic/bt.c optional bt dev/buslogic/bt_eisa.c optional bt eisa dev/buslogic/bt_isa.c optional bt isa dev/buslogic/bt_mca.c optional bt mca dev/buslogic/bt_pci.c optional bt pci dev/bwi/bwimac.c optional bwi dev/bwi/bwiphy.c optional bwi dev/bwi/bwirf.c optional bwi dev/bwi/if_bwi.c optional bwi dev/bwi/if_bwi_pci.c optional bwi pci # XXX Work around clang warning, until maintainer approves fix. dev/bwn/if_bwn.c optional bwn siba_bwn \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/cardbus/cardbus.c optional cardbus dev/cardbus/cardbus_cis.c optional cardbus dev/cardbus/cardbus_device.c optional cardbus dev/cas/if_cas.c optional cas dev/cfi/cfi_bus_fdt.c optional cfi fdt dev/cfi/cfi_bus_nexus.c optional cfi dev/cfi/cfi_core.c optional cfi dev/cfi/cfi_dev.c optional cfi dev/cfi/cfi_disk.c optional cfid dev/ciss/ciss.c optional ciss dev/cm/smc90cx6.c optional cm dev/cmx/cmx.c optional cmx dev/cmx/cmx_pccard.c optional cmx pccard dev/cpufreq/ichss.c optional cpufreq dev/cs/if_cs.c optional cs dev/cs/if_cs_isa.c optional cs isa dev/cs/if_cs_pccard.c optional cs pccard dev/cxgb/cxgb_main.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_sge.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_mc5.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_vsc7323.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_vsc8211.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_ael1002.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_aq100x.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_mv88e1xxx.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_xgmac.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_t3_hw.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_tn1010.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/sys/uipc_mvec.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_t3fw.c optional cxgb cxgb_t3fw \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgbe/t4_mp_ring.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe ${GCC_MS_EXTENSIONS}" dev/cxgbe/t4_main.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_netmap.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_sge.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_l2t.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_tracer.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/common/t4_hw.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" t4fw_cfg.c optional cxgbe \ compile-with "${AWK} -f $S/tools/fw_stub.awk t4fw_cfg.fw:t4fw_cfg t4fw_cfg_uwire.fw:t4fw_cfg_uwire t4fw.fw:t4fw -mt4fw_cfg -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "t4fw_cfg.c" t4fw_cfg.fwo optional cxgbe \ dependency "t4fw_cfg.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw_cfg.fwo" t4fw_cfg.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw_cfg.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t4fw_cfg.fw" t4fw_cfg_uwire.fwo optional cxgbe \ dependency "t4fw_cfg_uwire.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw_cfg_uwire.fwo" t4fw_cfg_uwire.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw_cfg_uwire.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t4fw_cfg_uwire.fw" t4fw.fwo optional cxgbe \ dependency "t4fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw.fwo" t4fw.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw-1.11.27.0.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "t4fw.fw" t5fw_cfg.c optional cxgbe \ compile-with "${AWK} -f $S/tools/fw_stub.awk t5fw_cfg.fw:t5fw_cfg t5fw.fw:t5fw -mt5fw_cfg -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "t5fw_cfg.c" t5fw_cfg.fwo optional cxgbe \ dependency "t5fw_cfg.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t5fw_cfg.fwo" t5fw_cfg.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t5fw_cfg.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t5fw_cfg.fw" t5fw.fwo optional cxgbe \ dependency "t5fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t5fw.fwo" t5fw.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t5fw-1.11.27.0.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "t5fw.fw" dev/cy/cy.c optional cy dev/cy/cy_isa.c optional cy isa dev/cy/cy_pci.c optional cy pci dev/dc/if_dc.c optional dc pci dev/dc/dcphy.c optional dc pci dev/dc/pnphy.c optional dc pci dev/dcons/dcons.c optional dcons dev/dcons/dcons_crom.c optional dcons_crom dev/dcons/dcons_os.c optional dcons dev/de/if_de.c optional de pci dev/digi/CX.c optional digi_CX dev/digi/CX_PCI.c optional digi_CX_PCI dev/digi/EPCX.c optional digi_EPCX dev/digi/EPCX_PCI.c optional digi_EPCX_PCI dev/digi/Xe.c optional digi_Xe dev/digi/Xem.c optional digi_Xem dev/digi/Xr.c optional digi_Xr dev/digi/digi.c optional digi dev/digi/digi_isa.c optional digi isa dev/digi/digi_pci.c optional digi pci dev/dpt/dpt_eisa.c optional dpt eisa dev/dpt/dpt_pci.c optional dpt pci dev/dpt/dpt_scsi.c optional dpt dev/drm/ati_pcigart.c optional drm dev/drm/drm_agpsupport.c optional drm dev/drm/drm_auth.c optional drm dev/drm/drm_bufs.c optional drm dev/drm/drm_context.c optional drm dev/drm/drm_dma.c optional drm dev/drm/drm_drawable.c optional drm dev/drm/drm_drv.c optional drm dev/drm/drm_fops.c optional drm dev/drm/drm_hashtab.c optional drm dev/drm/drm_ioctl.c optional drm dev/drm/drm_irq.c optional drm dev/drm/drm_lock.c optional drm dev/drm/drm_memory.c optional drm dev/drm/drm_mm.c optional drm dev/drm/drm_pci.c optional drm dev/drm/drm_scatter.c optional drm dev/drm/drm_sman.c optional drm dev/drm/drm_sysctl.c optional drm dev/drm/drm_vm.c optional drm dev/drm/i915_dma.c optional i915drm dev/drm/i915_drv.c optional i915drm dev/drm/i915_irq.c optional i915drm dev/drm/i915_mem.c optional i915drm dev/drm/i915_suspend.c optional i915drm dev/drm/mach64_dma.c optional mach64drm dev/drm/mach64_drv.c optional mach64drm dev/drm/mach64_irq.c optional mach64drm dev/drm/mach64_state.c optional mach64drm dev/drm/mga_dma.c optional mgadrm dev/drm/mga_drv.c optional mgadrm dev/drm/mga_irq.c optional mgadrm dev/drm/mga_state.c optional mgadrm dev/drm/mga_warp.c optional mgadrm dev/drm/r128_cce.c optional r128drm \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/drm/r128_drv.c optional r128drm dev/drm/r128_irq.c optional r128drm dev/drm/r128_state.c optional r128drm dev/drm/r300_cmdbuf.c optional radeondrm dev/drm/r600_blit.c optional radeondrm dev/drm/r600_cp.c optional radeondrm \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/drm/radeon_cp.c optional radeondrm \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/drm/radeon_cs.c optional radeondrm dev/drm/radeon_drv.c optional radeondrm dev/drm/radeon_irq.c optional radeondrm dev/drm/radeon_mem.c optional radeondrm dev/drm/radeon_state.c optional radeondrm dev/drm/savage_bci.c optional savagedrm dev/drm/savage_drv.c optional savagedrm dev/drm/savage_state.c optional savagedrm dev/drm/sis_drv.c optional sisdrm dev/drm/sis_ds.c optional sisdrm dev/drm/sis_mm.c optional sisdrm dev/drm/tdfx_drv.c optional tdfxdrm dev/drm/via_dma.c optional viadrm dev/drm/via_dmablit.c optional viadrm dev/drm/via_drv.c optional viadrm dev/drm/via_irq.c optional viadrm dev/drm/via_map.c optional viadrm dev/drm/via_mm.c optional viadrm dev/drm/via_verifier.c optional viadrm dev/drm/via_video.c optional viadrm dev/ed/if_ed.c optional ed dev/ed/if_ed_novell.c optional ed dev/ed/if_ed_rtl80x9.c optional ed dev/ed/if_ed_pccard.c optional ed pccard dev/ed/if_ed_pci.c optional ed pci dev/eisa/eisa_if.m standard dev/eisa/eisaconf.c optional eisa dev/e1000/if_em.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/if_lem.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/if_igb.c optional igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_80003es2lan.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82540.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82541.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82542.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82543.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82571.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82575.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_ich8lan.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_i210.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_api.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_mac.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_manage.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_nvm.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_phy.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_vf.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_mbx.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_osdep.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/et/if_et.c optional et dev/en/if_en_pci.c optional en pci dev/en/midway.c optional en dev/ep/if_ep.c optional ep dev/ep/if_ep_eisa.c optional ep eisa dev/ep/if_ep_isa.c optional ep isa dev/ep/if_ep_mca.c optional ep mca dev/ep/if_ep_pccard.c optional ep pccard dev/esp/esp_pci.c optional esp pci dev/esp/ncr53c9x.c optional esp dev/etherswitch/arswitch/arswitch.c optional arswitch dev/etherswitch/arswitch/arswitch_reg.c optional arswitch dev/etherswitch/arswitch/arswitch_phy.c optional arswitch dev/etherswitch/arswitch/arswitch_8216.c optional arswitch dev/etherswitch/arswitch/arswitch_8226.c optional arswitch dev/etherswitch/arswitch/arswitch_8316.c optional arswitch dev/etherswitch/arswitch/arswitch_8327.c optional arswitch dev/etherswitch/arswitch/arswitch_7240.c optional arswitch dev/etherswitch/arswitch/arswitch_9340.c optional arswitch dev/etherswitch/arswitch/arswitch_vlans.c optional arswitch dev/etherswitch/etherswitch.c optional etherswitch dev/etherswitch/etherswitch_if.m optional etherswitch dev/etherswitch/ip17x/ip17x.c optional ip17x dev/etherswitch/ip17x/ip175c.c optional ip17x dev/etherswitch/ip17x/ip175d.c optional ip17x dev/etherswitch/ip17x/ip17x_phy.c optional ip17x dev/etherswitch/ip17x/ip17x_vlans.c optional ip17x dev/etherswitch/mdio_if.m optional miiproxy dev/etherswitch/mdio.c optional miiproxy dev/etherswitch/miiproxy.c optional miiproxy dev/etherswitch/rtl8366/rtl8366rb.c optional rtl8366rb dev/etherswitch/ukswitch/ukswitch.c optional ukswitch dev/ex/if_ex.c optional ex dev/ex/if_ex_isa.c optional ex isa dev/ex/if_ex_pccard.c optional ex pccard dev/exca/exca.c optional cbb dev/fatm/if_fatm.c optional fatm pci dev/fb/fbd.c optional fbd | vt dev/fb/fb_if.m standard dev/fb/splash.c optional sc splash dev/fdt/fdt_clock.c optional fdt fdt_clock dev/fdt/fdt_clock_if.m optional fdt fdt_clock dev/fdt/fdt_common.c optional fdt dev/fdt/fdt_pinctrl.c optional fdt fdt_pinctrl dev/fdt/fdt_pinctrl_if.m optional fdt fdt_pinctrl dev/fdt/fdt_slicer.c optional fdt cfi | fdt nand dev/fdt/fdt_static_dtb.S optional fdt fdt_dtb_static \ dependency "$S/boot/fdt/dts/${MACHINE}/${FDT_DTS_FILE}" dev/fdt/simplebus.c optional fdt dev/fe/if_fe.c optional fe dev/fe/if_fe_pccard.c optional fe pccard dev/filemon/filemon.c optional filemon dev/firewire/firewire.c optional firewire dev/firewire/fwcrom.c optional firewire dev/firewire/fwdev.c optional firewire dev/firewire/fwdma.c optional firewire dev/firewire/fwmem.c optional firewire dev/firewire/fwohci.c optional firewire dev/firewire/fwohci_pci.c optional firewire pci dev/firewire/if_fwe.c optional fwe dev/firewire/if_fwip.c optional fwip dev/firewire/sbp.c optional sbp dev/firewire/sbp_targ.c optional sbp_targ dev/flash/at45d.c optional at45d dev/flash/mx25l.c optional mx25l dev/fxp/if_fxp.c optional fxp dev/fxp/inphy.c optional fxp dev/gem/if_gem.c optional gem dev/gem/if_gem_pci.c optional gem pci dev/gem/if_gem_sbus.c optional gem sbus dev/gpio/gpiobus.c optional gpio \ dependency "gpiobus_if.h" dev/gpio/gpioc.c optional gpio \ dependency "gpio_if.h" dev/gpio/gpioiic.c optional gpioiic dev/gpio/gpioled.c optional gpioled dev/gpio/gpio_if.m optional gpio dev/gpio/gpiobus_if.m optional gpio dev/gpio/ofw_gpiobus.c optional fdt gpio dev/hatm/if_hatm.c optional hatm pci dev/hatm/if_hatm_intr.c optional hatm pci dev/hatm/if_hatm_ioctl.c optional hatm pci dev/hatm/if_hatm_rx.c optional hatm pci dev/hatm/if_hatm_tx.c optional hatm pci dev/hifn/hifn7751.c optional hifn dev/hme/if_hme.c optional hme dev/hme/if_hme_pci.c optional hme pci dev/hme/if_hme_sbus.c optional hme sbus dev/hptiop/hptiop.c optional hptiop scbus dev/hwpmc/hwpmc_logging.c optional hwpmc dev/hwpmc/hwpmc_mod.c optional hwpmc dev/hwpmc/hwpmc_soft.c optional hwpmc dev/ichsmb/ichsmb.c optional ichsmb dev/ichsmb/ichsmb_pci.c optional ichsmb pci dev/ida/ida.c optional ida dev/ida/ida_disk.c optional ida dev/ida/ida_eisa.c optional ida eisa dev/ida/ida_pci.c optional ida pci dev/ie/if_ie.c optional ie isa nowerror dev/ie/if_ie_isa.c optional ie isa dev/iicbus/ad7418.c optional ad7418 dev/iicbus/ds133x.c optional ds133x dev/iicbus/ds1374.c optional ds1374 dev/iicbus/ds1672.c optional ds1672 dev/iicbus/icee.c optional icee dev/iicbus/if_ic.c optional ic dev/iicbus/iic.c optional iic dev/iicbus/iicbb.c optional iicbb dev/iicbus/iicbb_if.m optional iicbb dev/iicbus/iicbus.c optional iicbus dev/iicbus/iicbus_if.m optional iicbus dev/iicbus/iiconf.c optional iicbus dev/iicbus/iicsmb.c optional iicsmb \ dependency "iicbus_if.h" dev/iicbus/iicoc.c optional iicoc dev/iicbus/lm75.c optional lm75 dev/iicbus/pcf8563.c optional pcf8563 dev/iicbus/s35390a.c optional s35390a dev/iir/iir.c optional iir dev/iir/iir_ctrl.c optional iir dev/iir/iir_pci.c optional iir pci dev/intpm/intpm.c optional intpm pci # XXX Work around clang warning, until maintainer approves fix. dev/ips/ips.c optional ips \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/ips/ips_commands.c optional ips dev/ips/ips_disk.c optional ips dev/ips/ips_ioctl.c optional ips dev/ips/ips_pci.c optional ips pci dev/ipw/if_ipw.c optional ipw ipwbssfw.c optional ipwbssfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_bss.fw:ipw_bss:130 -lintel_ipw -mipw_bss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwbssfw.c" ipw_bss.fwo optional ipwbssfw | ipwfw \ dependency "ipw_bss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_bss.fwo" ipw_bss.fw optional ipwbssfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_bss.fw" ipwibssfw.c optional ipwibssfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_ibss.fw:ipw_ibss:130 -lintel_ipw -mipw_ibss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwibssfw.c" ipw_ibss.fwo optional ipwibssfw | ipwfw \ dependency "ipw_ibss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_ibss.fwo" ipw_ibss.fw optional ipwibssfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3-i.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_ibss.fw" ipwmonitorfw.c optional ipwmonitorfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_monitor.fw:ipw_monitor:130 -lintel_ipw -mipw_monitor -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwmonitorfw.c" ipw_monitor.fwo optional ipwmonitorfw | ipwfw \ dependency "ipw_monitor.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_monitor.fwo" ipw_monitor.fw optional ipwmonitorfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3-p.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_monitor.fw" dev/iscsi/icl.c optional iscsi | ctl dev/iscsi/icl_proxy.c optional iscsi | ctl +dev/iscsi/icl_soft.c optional iscsi | ctl dev/iscsi/iscsi.c optional iscsi scbus dev/iscsi_initiator/iscsi.c optional iscsi_initiator scbus dev/iscsi_initiator/iscsi_subr.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_cam.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_soc.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_sm.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_subr.c optional iscsi_initiator scbus dev/ismt/ismt.c optional ismt dev/isp/isp.c optional isp dev/isp/isp_freebsd.c optional isp dev/isp/isp_library.c optional isp dev/isp/isp_pci.c optional isp pci dev/isp/isp_sbus.c optional isp sbus dev/isp/isp_target.c optional isp dev/ispfw/ispfw.c optional ispfw dev/iwi/if_iwi.c optional iwi iwibssfw.c optional iwibssfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_bss.fw:iwi_bss:300 -lintel_iwi -miwi_bss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwibssfw.c" iwi_bss.fwo optional iwibssfw | iwifw \ dependency "iwi_bss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_bss.fwo" iwi_bss.fw optional iwibssfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-bss.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_bss.fw" iwiibssfw.c optional iwiibssfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_ibss.fw:iwi_ibss:300 -lintel_iwi -miwi_ibss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwiibssfw.c" iwi_ibss.fwo optional iwiibssfw | iwifw \ dependency "iwi_ibss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_ibss.fwo" iwi_ibss.fw optional iwiibssfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-ibss.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_ibss.fw" iwimonitorfw.c optional iwimonitorfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_monitor.fw:iwi_monitor:300 -lintel_iwi -miwi_monitor -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwimonitorfw.c" iwi_monitor.fwo optional iwimonitorfw | iwifw \ dependency "iwi_monitor.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_monitor.fwo" iwi_monitor.fw optional iwimonitorfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-sniffer.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_monitor.fw" dev/iwn/if_iwn.c optional iwn iwn1000fw.c optional iwn1000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn1000.fw:iwn1000fw -miwn1000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn1000fw.c" iwn1000fw.fwo optional iwn1000fw | iwnfw \ dependency "iwn1000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn1000fw.fwo" iwn1000.fw optional iwn1000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-1000-39.31.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn1000.fw" iwn100fw.c optional iwn100fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn100.fw:iwn100fw -miwn100fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn100fw.c" iwn100fw.fwo optional iwn100fw | iwnfw \ dependency "iwn100.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn100fw.fwo" iwn100.fw optional iwn100fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-100-39.31.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn100.fw" iwn105fw.c optional iwn105fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn105.fw:iwn105fw -miwn105fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn105fw.c" iwn105fw.fwo optional iwn105fw | iwnfw \ dependency "iwn105.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn105fw.fwo" iwn105.fw optional iwn105fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-105-6-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn105.fw" iwn135fw.c optional iwn135fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn135.fw:iwn135fw -miwn135fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn135fw.c" iwn135fw.fwo optional iwn135fw | iwnfw \ dependency "iwn135.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn135fw.fwo" iwn135.fw optional iwn135fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-135-6-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn135.fw" iwn2000fw.c optional iwn2000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn2000.fw:iwn2000fw -miwn2000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn2000fw.c" iwn2000fw.fwo optional iwn2000fw | iwnfw \ dependency "iwn2000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn2000fw.fwo" iwn2000.fw optional iwn2000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-2000-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn2000.fw" iwn2030fw.c optional iwn2030fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn2030.fw:iwn2030fw -miwn2030fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn2030fw.c" iwn2030fw.fwo optional iwn2030fw | iwnfw \ dependency "iwn2030.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn2030fw.fwo" iwn2030.fw optional iwn2030fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwnwifi-2030-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn2030.fw" iwn4965fw.c optional iwn4965fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn4965.fw:iwn4965fw -miwn4965fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn4965fw.c" iwn4965fw.fwo optional iwn4965fw | iwnfw \ dependency "iwn4965.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn4965fw.fwo" iwn4965.fw optional iwn4965fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-4965-228.61.2.24.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn4965.fw" iwn5000fw.c optional iwn5000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5000.fw:iwn5000fw -miwn5000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn5000fw.c" iwn5000fw.fwo optional iwn5000fw | iwnfw \ dependency "iwn5000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn5000fw.fwo" iwn5000.fw optional iwn5000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-5000-8.83.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn5000.fw" iwn5150fw.c optional iwn5150fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5150.fw:iwn5150fw -miwn5150fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn5150fw.c" iwn5150fw.fwo optional iwn5150fw | iwnfw \ dependency "iwn5150.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn5150fw.fwo" iwn5150.fw optional iwn5150fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-5150-8.24.2.2.fw.uu"\ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn5150.fw" iwn6000fw.c optional iwn6000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000.fw:iwn6000fw -miwn6000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000fw.c" iwn6000fw.fwo optional iwn6000fw | iwnfw \ dependency "iwn6000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000fw.fwo" iwn6000.fw optional iwn6000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000-9.221.4.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000.fw" iwn6000g2afw.c optional iwn6000g2afw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2a.fw:iwn6000g2afw -miwn6000g2afw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000g2afw.c" iwn6000g2afw.fwo optional iwn6000g2afw | iwnfw \ dependency "iwn6000g2a.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000g2afw.fwo" iwn6000g2a.fw optional iwn6000g2afw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000g2a-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000g2a.fw" iwn6000g2bfw.c optional iwn6000g2bfw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2b.fw:iwn6000g2bfw -miwn6000g2bfw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000g2bfw.c" iwn6000g2bfw.fwo optional iwn6000g2bfw | iwnfw \ dependency "iwn6000g2b.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000g2bfw.fwo" iwn6000g2b.fw optional iwn6000g2bfw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000g2b-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000g2b.fw" iwn6050fw.c optional iwn6050fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6050.fw:iwn6050fw -miwn6050fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6050fw.c" iwn6050fw.fwo optional iwn6050fw | iwnfw \ dependency "iwn6050.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6050fw.fwo" iwn6050.fw optional iwn6050fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6050-41.28.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6050.fw" dev/ixgb/if_ixgb.c optional ixgb dev/ixgb/ixgb_ee.c optional ixgb dev/ixgb/ixgb_hw.c optional ixgb dev/ixgbe/ixgbe.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe -DSMP" dev/ixgbe/ixv.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_phy.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_api.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_common.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_mbx.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_vf.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_82598.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_82599.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_x540.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb_82598.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb_82599.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/jme/if_jme.c optional jme pci dev/joy/joy.c optional joy dev/joy/joy_isa.c optional joy isa dev/kbdmux/kbdmux.c optional kbdmux dev/ksyms/ksyms.c optional ksyms dev/le/am7990.c optional le dev/le/am79900.c optional le dev/le/if_le_pci.c optional le pci dev/le/lance.c optional le dev/led/led.c standard dev/lge/if_lge.c optional lge dev/lmc/if_lmc.c optional lmc dev/malo/if_malo.c optional malo dev/malo/if_malohal.c optional malo dev/malo/if_malo_pci.c optional malo pci dev/mc146818/mc146818.c optional mc146818 dev/mca/mca_bus.c optional mca dev/mcd/mcd.c optional mcd isa nowerror dev/mcd/mcd_isa.c optional mcd isa nowerror dev/md/md.c optional md dev/mem/memdev.c optional mem dev/mem/memutil.c optional mem dev/mfi/mfi.c optional mfi dev/mfi/mfi_debug.c optional mfi dev/mfi/mfi_pci.c optional mfi pci dev/mfi/mfi_disk.c optional mfi dev/mfi/mfi_syspd.c optional mfi dev/mfi/mfi_tbolt.c optional mfi dev/mfi/mfi_linux.c optional mfi compat_linux dev/mfi/mfi_cam.c optional mfip scbus dev/mii/acphy.c optional miibus | acphy dev/mii/amphy.c optional miibus | amphy dev/mii/atphy.c optional miibus | atphy dev/mii/axphy.c optional miibus | axphy dev/mii/bmtphy.c optional miibus | bmtphy dev/mii/brgphy.c optional miibus | brgphy dev/mii/ciphy.c optional miibus | ciphy dev/mii/e1000phy.c optional miibus | e1000phy dev/mii/gentbi.c optional miibus | gentbi dev/mii/icsphy.c optional miibus | icsphy dev/mii/ip1000phy.c optional miibus | ip1000phy dev/mii/jmphy.c optional miibus | jmphy dev/mii/lxtphy.c optional miibus | lxtphy dev/mii/mii.c optional miibus | mii dev/mii/mii_bitbang.c optional miibus | mii_bitbang dev/mii/mii_physubr.c optional miibus | mii dev/mii/miibus_if.m optional miibus | mii dev/mii/mlphy.c optional miibus | mlphy dev/mii/nsgphy.c optional miibus | nsgphy dev/mii/nsphy.c optional miibus | nsphy dev/mii/nsphyter.c optional miibus | nsphyter dev/mii/pnaphy.c optional miibus | pnaphy dev/mii/qsphy.c optional miibus | qsphy dev/mii/rdcphy.c optional miibus | rdcphy dev/mii/rgephy.c optional miibus | rgephy dev/mii/rlphy.c optional miibus | rlphy dev/mii/rlswitch.c optional rlswitch dev/mii/smcphy.c optional miibus | smcphy dev/mii/smscphy.c optional miibus | smscphy dev/mii/tdkphy.c optional miibus | tdkphy dev/mii/tlphy.c optional miibus | tlphy dev/mii/truephy.c optional miibus | truephy dev/mii/ukphy.c optional miibus | mii dev/mii/ukphy_subr.c optional miibus | mii dev/mii/xmphy.c optional miibus | xmphy dev/mk48txx/mk48txx.c optional mk48txx dev/mlx/mlx.c optional mlx dev/mlx/mlx_disk.c optional mlx dev/mlx/mlx_pci.c optional mlx pci dev/mly/mly.c optional mly dev/mmc/mmc.c optional mmc dev/mmc/mmcbr_if.m standard dev/mmc/mmcbus_if.m standard dev/mmc/mmcsd.c optional mmcsd dev/mn/if_mn.c optional mn pci dev/mpr/mpr.c optional mpr dev/mpr/mpr_config.c optional mpr # XXX Work around clang warning, until maintainer approves fix. dev/mpr/mpr_mapping.c optional mpr \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/mpr/mpr_pci.c optional mpr pci dev/mpr/mpr_sas.c optional mpr \ compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}" dev/mpr/mpr_sas_lsi.c optional mpr dev/mpr/mpr_table.c optional mpr dev/mpr/mpr_user.c optional mpr dev/mps/mps.c optional mps dev/mps/mps_config.c optional mps # XXX Work around clang warning, until maintainer approves fix. dev/mps/mps_mapping.c optional mps \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/mps/mps_pci.c optional mps pci dev/mps/mps_sas.c optional mps \ compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}" dev/mps/mps_sas_lsi.c optional mps dev/mps/mps_table.c optional mps dev/mps/mps_user.c optional mps dev/mpt/mpt.c optional mpt dev/mpt/mpt_cam.c optional mpt dev/mpt/mpt_debug.c optional mpt dev/mpt/mpt_pci.c optional mpt pci dev/mpt/mpt_raid.c optional mpt dev/mpt/mpt_user.c optional mpt dev/mrsas/mrsas.c optional mrsas dev/mrsas/mrsas_cam.c optional mrsas dev/mrsas/mrsas_ioctl.c optional mrsas dev/mrsas/mrsas_fp.c optional mrsas dev/msk/if_msk.c optional msk dev/mvs/mvs.c optional mvs dev/mvs/mvs_if.m optional mvs dev/mvs/mvs_pci.c optional mvs pci dev/mwl/if_mwl.c optional mwl dev/mwl/if_mwl_pci.c optional mwl pci dev/mwl/mwlhal.c optional mwl mwlfw.c optional mwlfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk mw88W8363.fw:mw88W8363fw mwlboot.fw:mwlboot -mmwl -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "mwlfw.c" mw88W8363.fwo optional mwlfw \ dependency "mw88W8363.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "mw88W8363.fwo" mw88W8363.fw optional mwlfw \ dependency "$S/contrib/dev/mwl/mw88W8363.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "mw88W8363.fw" mwlboot.fwo optional mwlfw \ dependency "mwlboot.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "mwlboot.fwo" mwlboot.fw optional mwlfw \ dependency "$S/contrib/dev/mwl/mwlboot.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "mwlboot.fw" dev/mxge/if_mxge.c optional mxge pci dev/mxge/mxge_eth_z8e.c optional mxge pci dev/mxge/mxge_ethp_z8e.c optional mxge pci dev/mxge/mxge_rss_eth_z8e.c optional mxge pci dev/mxge/mxge_rss_ethp_z8e.c optional mxge pci dev/my/if_my.c optional my dev/nand/nand.c optional nand dev/nand/nand_bbt.c optional nand dev/nand/nand_cdev.c optional nand dev/nand/nand_generic.c optional nand dev/nand/nand_geom.c optional nand dev/nand/nand_id.c optional nand dev/nand/nandbus.c optional nand dev/nand/nandbus_if.m optional nand dev/nand/nand_if.m optional nand dev/nand/nandsim.c optional nandsim nand dev/nand/nandsim_chip.c optional nandsim nand dev/nand/nandsim_ctrl.c optional nandsim nand dev/nand/nandsim_log.c optional nandsim nand dev/nand/nandsim_swap.c optional nandsim nand dev/nand/nfc_if.m optional nand dev/ncr/ncr.c optional ncr pci dev/ncv/ncr53c500.c optional ncv dev/ncv/ncr53c500_pccard.c optional ncv pccard dev/netmap/netmap.c optional netmap dev/netmap/netmap_freebsd.c optional netmap dev/netmap/netmap_generic.c optional netmap dev/netmap/netmap_mbq.c optional netmap dev/netmap/netmap_mem2.c optional netmap dev/netmap/netmap_monitor.c optional netmap dev/netmap/netmap_offloadings.c optional netmap dev/netmap/netmap_pipe.c optional netmap dev/netmap/netmap_vale.c optional netmap # compile-with "${NORMAL_C} -Wconversion -Wextra" dev/nfsmb/nfsmb.c optional nfsmb pci dev/nge/if_nge.c optional nge dev/nxge/if_nxge.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-device.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-mm.c optional nxge dev/nxge/xgehal/xge-queue.c optional nxge dev/nxge/xgehal/xgehal-driver.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-ring.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-channel.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-fifo.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-stats.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-config.c optional nxge dev/nxge/xgehal/xgehal-mgmt.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nmdm/nmdm.c optional nmdm dev/nsp/nsp.c optional nsp dev/nsp/nsp_pccard.c optional nsp pccard dev/null/null.c standard dev/oce/oce_hw.c optional oce pci dev/oce/oce_if.c optional oce pci dev/oce/oce_mbox.c optional oce pci dev/oce/oce_queue.c optional oce pci dev/oce/oce_sysctl.c optional oce pci dev/oce/oce_util.c optional oce pci dev/ofw/ofw_bus_if.m optional fdt dev/ofw/ofw_bus_subr.c optional fdt dev/ofw/ofw_fdt.c optional fdt dev/ofw/ofw_if.m optional fdt dev/ofw/ofw_iicbus.c optional fdt iicbus dev/ofw/ofwbus.c optional fdt dev/ofw/openfirm.c optional fdt dev/ofw/openfirmio.c optional fdt dev/patm/if_patm.c optional patm pci dev/patm/if_patm_attach.c optional patm pci dev/patm/if_patm_intr.c optional patm pci dev/patm/if_patm_ioctl.c optional patm pci dev/patm/if_patm_rtables.c optional patm pci dev/patm/if_patm_rx.c optional patm pci dev/patm/if_patm_tx.c optional patm pci dev/pbio/pbio.c optional pbio isa dev/pccard/card_if.m standard dev/pccard/pccard.c optional pccard dev/pccard/pccard_cis.c optional pccard dev/pccard/pccard_cis_quirks.c optional pccard dev/pccard/pccard_device.c optional pccard dev/pccard/power_if.m standard dev/pccbb/pccbb.c optional cbb dev/pccbb/pccbb_isa.c optional cbb isa dev/pccbb/pccbb_pci.c optional cbb pci dev/pcf/pcf.c optional pcf dev/pci/eisa_pci.c optional pci eisa dev/pci/fixup_pci.c optional pci dev/pci/hostb_pci.c optional pci dev/pci/ignore_pci.c optional pci dev/pci/isa_pci.c optional pci isa dev/pci/pci.c optional pci dev/pci/pci_if.m standard dev/pci/pci_pci.c optional pci dev/pci/pci_subr.c optional pci dev/pci/pci_user.c optional pci dev/pci/pcib_if.m standard dev/pci/pcib_support.c standard dev/pci/vga_pci.c optional pci dev/pcn/if_pcn.c optional pcn pci dev/pdq/if_fea.c optional fea eisa dev/pdq/if_fpa.c optional fpa pci dev/pdq/pdq.c optional nowerror fea eisa | fpa pci dev/pdq/pdq_ifsubr.c optional nowerror fea eisa | fpa pci dev/ppbus/if_plip.c optional plip dev/ppbus/immio.c optional vpo dev/ppbus/lpbb.c optional lpbb dev/ppbus/lpt.c optional lpt dev/ppbus/pcfclock.c optional pcfclock dev/ppbus/ppb_1284.c optional ppbus dev/ppbus/ppb_base.c optional ppbus dev/ppbus/ppb_msq.c optional ppbus dev/ppbus/ppbconf.c optional ppbus dev/ppbus/ppbus_if.m optional ppbus dev/ppbus/ppi.c optional ppi dev/ppbus/pps.c optional pps dev/ppbus/vpo.c optional vpo dev/ppbus/vpoio.c optional vpo dev/ppc/ppc.c optional ppc dev/ppc/ppc_acpi.c optional ppc acpi dev/ppc/ppc_isa.c optional ppc isa dev/ppc/ppc_pci.c optional ppc pci dev/ppc/ppc_puc.c optional ppc puc dev/pst/pst-iop.c optional pst dev/pst/pst-pci.c optional pst pci dev/pst/pst-raid.c optional pst dev/pty/pty.c optional pty dev/puc/puc.c optional puc dev/puc/puc_cfg.c optional puc dev/puc/puc_pccard.c optional puc pccard dev/puc/puc_pci.c optional puc pci dev/puc/pucdata.c optional puc pci dev/quicc/quicc_core.c optional quicc dev/ral/rt2560.c optional ral dev/ral/rt2661.c optional ral dev/ral/rt2860.c optional ral dev/ral/if_ral_pci.c optional ral pci rt2561fw.c optional rt2561fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561.fw:rt2561fw -mrt2561 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2561fw.c" rt2561fw.fwo optional rt2561fw | ralfw \ dependency "rt2561.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2561fw.fwo" rt2561.fw optional rt2561fw | ralfw \ dependency "$S/contrib/dev/ral/rt2561.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2561.fw" rt2561sfw.c optional rt2561sfw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561s.fw:rt2561sfw -mrt2561s -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2561sfw.c" rt2561sfw.fwo optional rt2561sfw | ralfw \ dependency "rt2561s.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2561sfw.fwo" rt2561s.fw optional rt2561sfw | ralfw \ dependency "$S/contrib/dev/ral/rt2561s.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2561s.fw" rt2661fw.c optional rt2661fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2661.fw:rt2661fw -mrt2661 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2661fw.c" rt2661fw.fwo optional rt2661fw | ralfw \ dependency "rt2661.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2661fw.fwo" rt2661.fw optional rt2661fw | ralfw \ dependency "$S/contrib/dev/ral/rt2661.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2661.fw" rt2860fw.c optional rt2860fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2860.fw:rt2860fw -mrt2860 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2860fw.c" rt2860fw.fwo optional rt2860fw | ralfw \ dependency "rt2860.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2860fw.fwo" rt2860.fw optional rt2860fw | ralfw \ dependency "$S/contrib/dev/ral/rt2860.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2860.fw" dev/random/randomdev.c standard dev/random/random_adaptors.c standard dev/random/dummy_rng.c standard dev/random/live_entropy_sources.c standard dev/random/random_harvestq.c standard dev/random/randomdev_soft.c optional random dev/random/yarrow.c optional random dev/random/fortuna.c optional random dev/random/hash.c optional random dev/rc/rc.c optional rc dev/re/if_re.c optional re dev/rl/if_rl.c optional rl pci dev/rndtest/rndtest.c optional rndtest dev/rp/rp.c optional rp dev/rp/rp_isa.c optional rp isa dev/rp/rp_pci.c optional rp pci dev/safe/safe.c optional safe dev/scc/scc_if.m optional scc dev/scc/scc_bfe_ebus.c optional scc ebus dev/scc/scc_bfe_quicc.c optional scc quicc dev/scc/scc_bfe_sbus.c optional scc fhc | scc sbus dev/scc/scc_core.c optional scc dev/scc/scc_dev_quicc.c optional scc quicc dev/scc/scc_dev_sab82532.c optional scc dev/scc/scc_dev_z8530.c optional scc dev/scd/scd.c optional scd isa dev/scd/scd_isa.c optional scd isa dev/sdhci/sdhci.c optional sdhci dev/sdhci/sdhci_if.m optional sdhci dev/sdhci/sdhci_pci.c optional sdhci pci dev/sf/if_sf.c optional sf pci dev/sge/if_sge.c optional sge pci dev/si/si.c optional si dev/si/si2_z280.c optional si dev/si/si3_t225.c optional si dev/si/si_eisa.c optional si eisa dev/si/si_isa.c optional si isa dev/si/si_pci.c optional si pci dev/siba/siba.c optional siba dev/siba/siba_bwn.c optional siba_bwn pci dev/siba/siba_cc.c optional siba dev/siba/siba_core.c optional siba | siba_bwn pci dev/siba/siba_pcib.c optional siba pci dev/siis/siis.c optional siis pci dev/sis/if_sis.c optional sis pci dev/sk/if_sk.c optional sk pci dev/smbus/smb.c optional smb dev/smbus/smbconf.c optional smbus dev/smbus/smbus.c optional smbus dev/smbus/smbus_if.m optional smbus dev/smc/if_smc.c optional smc dev/smc/if_smc_fdt.c optional smc fdt dev/sn/if_sn.c optional sn dev/sn/if_sn_isa.c optional sn isa dev/sn/if_sn_pccard.c optional sn pccard dev/snp/snp.c optional snp dev/sound/clone.c optional sound dev/sound/unit.c optional sound dev/sound/isa/ad1816.c optional snd_ad1816 isa dev/sound/isa/ess.c optional snd_ess isa dev/sound/isa/gusc.c optional snd_gusc isa dev/sound/isa/mss.c optional snd_mss isa dev/sound/isa/sb16.c optional snd_sb16 isa dev/sound/isa/sb8.c optional snd_sb8 isa dev/sound/isa/sbc.c optional snd_sbc isa dev/sound/isa/sndbuf_dma.c optional sound isa dev/sound/pci/als4000.c optional snd_als4000 pci dev/sound/pci/atiixp.c optional snd_atiixp pci dev/sound/pci/cmi.c optional snd_cmi pci dev/sound/pci/cs4281.c optional snd_cs4281 pci dev/sound/pci/csa.c optional snd_csa pci dev/sound/pci/csapcm.c optional snd_csa pci dev/sound/pci/ds1.c optional snd_ds1 pci dev/sound/pci/emu10k1.c optional snd_emu10k1 pci dev/sound/pci/emu10kx.c optional snd_emu10kx pci dev/sound/pci/emu10kx-pcm.c optional snd_emu10kx pci dev/sound/pci/emu10kx-midi.c optional snd_emu10kx pci dev/sound/pci/envy24.c optional snd_envy24 pci dev/sound/pci/envy24ht.c optional snd_envy24ht pci dev/sound/pci/es137x.c optional snd_es137x pci dev/sound/pci/fm801.c optional snd_fm801 pci dev/sound/pci/ich.c optional snd_ich pci dev/sound/pci/maestro.c optional snd_maestro pci dev/sound/pci/maestro3.c optional snd_maestro3 pci dev/sound/pci/neomagic.c optional snd_neomagic pci dev/sound/pci/solo.c optional snd_solo pci dev/sound/pci/spicds.c optional snd_spicds pci dev/sound/pci/t4dwave.c optional snd_t4dwave pci dev/sound/pci/via8233.c optional snd_via8233 pci dev/sound/pci/via82c686.c optional snd_via82c686 pci dev/sound/pci/vibes.c optional snd_vibes pci dev/sound/pci/hda/hdaa.c optional snd_hda pci dev/sound/pci/hda/hdaa_patches.c optional snd_hda pci dev/sound/pci/hda/hdac.c optional snd_hda pci dev/sound/pci/hda/hdac_if.m optional snd_hda pci dev/sound/pci/hda/hdacc.c optional snd_hda pci dev/sound/pci/hdspe.c optional snd_hdspe pci dev/sound/pci/hdspe-pcm.c optional snd_hdspe pci dev/sound/pcm/ac97.c optional sound dev/sound/pcm/ac97_if.m optional sound dev/sound/pcm/ac97_patch.c optional sound dev/sound/pcm/buffer.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/channel.c optional sound dev/sound/pcm/channel_if.m optional sound dev/sound/pcm/dsp.c optional sound dev/sound/pcm/feeder.c optional sound dev/sound/pcm/feeder_chain.c optional sound dev/sound/pcm/feeder_eq.c optional sound \ dependency "feeder_eq_gen.h" \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_if.m optional sound dev/sound/pcm/feeder_format.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_matrix.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_mixer.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_rate.c optional sound \ dependency "feeder_rate_gen.h" \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_volume.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/mixer.c optional sound dev/sound/pcm/mixer_if.m optional sound dev/sound/pcm/sndstat.c optional sound dev/sound/pcm/sound.c optional sound dev/sound/pcm/vchan.c optional sound dev/sound/usb/uaudio.c optional snd_uaudio usb dev/sound/usb/uaudio_pcm.c optional snd_uaudio usb dev/sound/midi/midi.c optional sound dev/sound/midi/mpu401.c optional sound dev/sound/midi/mpu_if.m optional sound dev/sound/midi/mpufoi_if.m optional sound dev/sound/midi/sequencer.c optional sound dev/sound/midi/synth_if.m optional sound dev/spibus/ofw_spibus.c optional fdt spibus dev/spibus/spibus.c optional spibus \ dependency "spibus_if.h" dev/spibus/spibus_if.m optional spibus dev/ste/if_ste.c optional ste pci dev/stg/tmc18c30.c optional stg dev/stg/tmc18c30_isa.c optional stg isa dev/stg/tmc18c30_pccard.c optional stg pccard dev/stg/tmc18c30_pci.c optional stg pci dev/stg/tmc18c30_subr.c optional stg dev/stge/if_stge.c optional stge dev/streams/streams.c optional streams dev/sym/sym_hipd.c optional sym \ dependency "$S/dev/sym/sym_{conf,defs}.h" dev/syscons/blank/blank_saver.c optional blank_saver dev/syscons/daemon/daemon_saver.c optional daemon_saver dev/syscons/dragon/dragon_saver.c optional dragon_saver dev/syscons/fade/fade_saver.c optional fade_saver dev/syscons/fire/fire_saver.c optional fire_saver dev/syscons/green/green_saver.c optional green_saver dev/syscons/logo/logo.c optional logo_saver dev/syscons/logo/logo_saver.c optional logo_saver dev/syscons/rain/rain_saver.c optional rain_saver dev/syscons/schistory.c optional sc dev/syscons/scmouse.c optional sc dev/syscons/scterm.c optional sc dev/syscons/scvidctl.c optional sc dev/syscons/snake/snake_saver.c optional snake_saver dev/syscons/star/star_saver.c optional star_saver dev/syscons/syscons.c optional sc dev/syscons/sysmouse.c optional sc dev/syscons/warp/warp_saver.c optional warp_saver dev/tdfx/tdfx_linux.c optional tdfx_linux tdfx compat_linux dev/tdfx/tdfx_pci.c optional tdfx pci dev/ti/if_ti.c optional ti pci dev/tl/if_tl.c optional tl pci dev/trm/trm.c optional trm dev/twa/tw_cl_init.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_intr.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_io.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_misc.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_cam.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_freebsd.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twe/twe.c optional twe dev/twe/twe_freebsd.c optional twe dev/tws/tws.c optional tws dev/tws/tws_cam.c optional tws dev/tws/tws_hdm.c optional tws dev/tws/tws_services.c optional tws dev/tws/tws_user.c optional tws dev/tx/if_tx.c optional tx dev/txp/if_txp.c optional txp dev/uart/uart_bus_acpi.c optional uart acpi #dev/uart/uart_bus_cbus.c optional uart cbus dev/uart/uart_bus_ebus.c optional uart ebus dev/uart/uart_bus_fdt.c optional uart fdt dev/uart/uart_bus_isa.c optional uart isa dev/uart/uart_bus_pccard.c optional uart pccard dev/uart/uart_bus_pci.c optional uart pci dev/uart/uart_bus_puc.c optional uart puc dev/uart/uart_bus_scc.c optional uart scc dev/uart/uart_core.c optional uart dev/uart/uart_dbg.c optional uart gdb dev/uart/uart_dev_ns8250.c optional uart uart_ns8250 dev/uart/uart_dev_pl011.c optional uart pl011 dev/uart/uart_dev_quicc.c optional uart quicc dev/uart/uart_dev_sab82532.c optional uart uart_sab82532 dev/uart/uart_dev_sab82532.c optional uart scc dev/uart/uart_dev_z8530.c optional uart uart_z8530 dev/uart/uart_dev_z8530.c optional uart scc dev/uart/uart_if.m optional uart dev/uart/uart_subr.c optional uart dev/uart/uart_tty.c optional uart dev/ubsec/ubsec.c optional ubsec # # USB controller drivers # dev/usb/controller/at91dci.c optional at91dci dev/usb/controller/at91dci_atmelarm.c optional at91dci at91rm9200 dev/usb/controller/musb_otg.c optional musb dev/usb/controller/musb_otg_atmelarm.c optional musb at91rm9200 dev/usb/controller/dwc_otg.c optional dwcotg dev/usb/controller/dwc_otg_fdt.c optional dwcotg fdt dev/usb/controller/ehci.c optional ehci dev/usb/controller/ehci_pci.c optional ehci pci dev/usb/controller/ohci.c optional ohci dev/usb/controller/ohci_atmelarm.c optional ohci at91rm9200 dev/usb/controller/ohci_pci.c optional ohci pci dev/usb/controller/uhci.c optional uhci dev/usb/controller/uhci_pci.c optional uhci pci dev/usb/controller/xhci.c optional xhci dev/usb/controller/xhci_pci.c optional xhci pci dev/usb/controller/saf1761_otg.c optional saf1761otg dev/usb/controller/saf1761_otg_fdt.c optional saf1761otg fdt dev/usb/controller/uss820dci.c optional uss820dci dev/usb/controller/uss820dci_atmelarm.c optional uss820dci at91rm9200 dev/usb/controller/usb_controller.c optional usb # # USB storage drivers # dev/usb/storage/umass.c optional umass dev/usb/storage/urio.c optional urio dev/usb/storage/ustorage_fs.c optional usfs # # USB core # dev/usb/usb_busdma.c optional usb dev/usb/usb_compat_linux.c optional usb dev/usb/usb_core.c optional usb dev/usb/usb_debug.c optional usb dev/usb/usb_dev.c optional usb dev/usb/usb_device.c optional usb dev/usb/usb_dynamic.c optional usb dev/usb/usb_error.c optional usb dev/usb/usb_generic.c optional usb dev/usb/usb_handle_request.c optional usb dev/usb/usb_hid.c optional usb dev/usb/usb_hub.c optional usb dev/usb/usb_if.m optional usb dev/usb/usb_lookup.c optional usb dev/usb/usb_mbuf.c optional usb dev/usb/usb_msctest.c optional usb dev/usb/usb_parse.c optional usb dev/usb/usb_pf.c optional usb dev/usb/usb_process.c optional usb dev/usb/usb_request.c optional usb dev/usb/usb_transfer.c optional usb dev/usb/usb_util.c optional usb # # USB network drivers # dev/usb/net/if_aue.c optional aue dev/usb/net/if_axe.c optional axe dev/usb/net/if_axge.c optional axge dev/usb/net/if_cdce.c optional cdce dev/usb/net/if_cue.c optional cue dev/usb/net/if_ipheth.c optional ipheth dev/usb/net/if_kue.c optional kue dev/usb/net/if_mos.c optional mos dev/usb/net/if_rue.c optional rue dev/usb/net/if_smsc.c optional smsc dev/usb/net/if_udav.c optional udav dev/usb/net/if_usie.c optional usie dev/usb/net/if_urndis.c optional urndis dev/usb/net/ruephy.c optional rue dev/usb/net/usb_ethernet.c optional aue | axe | axge | cdce | cue | kue | \ mos | rue | smsc | udav | ipheth | \ urndis dev/usb/net/uhso.c optional uhso # # USB WLAN drivers # dev/usb/wlan/if_rsu.c optional rsu rsu-rtl8712fw.c optional rsu-rtl8712fw | rsufw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rsu-rtl8712fw.fw:rsu-rtl8712fw:120 -mrsu-rtl8712fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rsu-rtl8712fw.c" rsu-rtl8712fw.fwo optional rsu-rtl8712fw | rsufw \ dependency "rsu-rtl8712fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rsu-rtl8712fw.fwo" rsu-rtl8712fw.fw optional rsu-rtl8712.fw | rsufw \ dependency "$S/contrib/dev/rsu/rsu-rtl8712fw.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rsu-rtl8712fw.fw" dev/usb/wlan/if_rum.c optional rum dev/usb/wlan/if_run.c optional run runfw.c optional runfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk run.fw:runfw -mrunfw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "runfw.c" runfw.fwo optional runfw \ dependency "run.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "runfw.fwo" run.fw optional runfw \ dependency "$S/contrib/dev/run/rt2870.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "run.fw" dev/usb/wlan/if_uath.c optional uath dev/usb/wlan/if_upgt.c optional upgt dev/usb/wlan/if_ural.c optional ural dev/usb/wlan/if_urtw.c optional urtw dev/usb/wlan/if_urtwn.c optional urtwn urtwn-rtl8188eufw.c optional urtwn-rtl8188eufw | urtwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk urtwn-rtl8188eufw.fw:urtwn-rtl8188eufw:111 -murtwn-rtl8188eufw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "urtwn-rtl8188eufw.c" urtwn-rtl8188eufw.fwo optional urtwn-rtl8188eufw | urtwnfw \ dependency "urtwn-rtl8188eufw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "urtwn-rtl8188eufw.fwo" urtwn-rtl8188eufw.fw optional urtwn-rtl8188eufw | urtwnfw \ dependency "$S/contrib/dev/urtwn/urtwn-rtl8188eufw.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "urtwn-rtl8188eufw.fw" urtwn-rtl8192cfwT.c optional urtwn-rtl8192cfwT | urtwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk urtwn-rtl8192cfwT.fw:urtwn-rtl8192cfwT:111 -murtwn-rtl8192cfwT -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "urtwn-rtl8192cfwT.c" urtwn-rtl8192cfwT.fwo optional urtwn-rtl8192cfwT | urtwnfw \ dependency "urtwn-rtl8192cfwT.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "urtwn-rtl8192cfwT.fwo" urtwn-rtl8192cfwT.fw optional urtwn-rtl8192cfwT | urtwnfw \ dependency "$S/contrib/dev/urtwn/urtwn-rtl8192cfwT.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "urtwn-rtl8192cfwT.fw" urtwn-rtl8192cfwU.c optional urtwn-rtl8192cfwU | urtwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk urtwn-rtl8192cfwU.fw:urtwn-rtl8192cfwU:111 -murtwn-rtl8192cfwU -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "urtwn-rtl8192cfwU.c" urtwn-rtl8192cfwU.fwo optional urtwn-rtl8192cfwU | urtwnfw \ dependency "urtwn-rtl8192cfwU.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "urtwn-rtl8192cfwU.fwo" urtwn-rtl8192cfwU.fw optional urtwn-rtl8192cfwU | urtwnfw \ dependency "$S/contrib/dev/urtwn/urtwn-rtl8192cfwU.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "urtwn-rtl8192cfwU.fw" dev/usb/wlan/if_zyd.c optional zyd # # USB serial and parallel port drivers # dev/usb/serial/u3g.c optional u3g dev/usb/serial/uark.c optional uark dev/usb/serial/ubsa.c optional ubsa dev/usb/serial/ubser.c optional ubser dev/usb/serial/uchcom.c optional uchcom dev/usb/serial/ucycom.c optional ucycom dev/usb/serial/ufoma.c optional ufoma dev/usb/serial/uftdi.c optional uftdi dev/usb/serial/ugensa.c optional ugensa dev/usb/serial/uipaq.c optional uipaq dev/usb/serial/ulpt.c optional ulpt dev/usb/serial/umcs.c optional umcs dev/usb/serial/umct.c optional umct dev/usb/serial/umodem.c optional umodem dev/usb/serial/umoscom.c optional umoscom dev/usb/serial/uplcom.c optional uplcom dev/usb/serial/uslcom.c optional uslcom dev/usb/serial/uvisor.c optional uvisor dev/usb/serial/uvscom.c optional uvscom dev/usb/serial/usb_serial.c optional ucom | u3g | uark | ubsa | ubser | \ uchcom | ucycom | ufoma | uftdi | \ ugensa | uipaq | umcs | umct | \ umodem | umoscom | uplcom | usie | \ uslcom | uvisor | uvscom # # USB misc drivers # dev/usb/misc/ufm.c optional ufm dev/usb/misc/udbp.c optional udbp dev/usb/misc/uled.c optional uled # # USB input drivers # dev/usb/input/atp.c optional atp dev/usb/input/uep.c optional uep dev/usb/input/uhid.c optional uhid dev/usb/input/ukbd.c optional ukbd dev/usb/input/ums.c optional ums dev/usb/input/wsp.c optional wsp # # USB quirks # dev/usb/quirk/usb_quirk.c optional usb # # USB templates # dev/usb/template/usb_template.c optional usb_template dev/usb/template/usb_template_audio.c optional usb_template dev/usb/template/usb_template_cdce.c optional usb_template dev/usb/template/usb_template_kbd.c optional usb_template dev/usb/template/usb_template_modem.c optional usb_template dev/usb/template/usb_template_mouse.c optional usb_template dev/usb/template/usb_template_msc.c optional usb_template dev/usb/template/usb_template_mtp.c optional usb_template dev/usb/template/usb_template_phone.c optional usb_template dev/usb/template/usb_template_serialnet.c optional usb_template # # USB END # dev/utopia/idtphy.c optional utopia dev/utopia/suni.c optional utopia dev/utopia/utopia.c optional utopia dev/vge/if_vge.c optional vge dev/viapm/viapm.c optional viapm pci dev/virtio/virtio.c optional virtio dev/virtio/virtqueue.c optional virtio dev/virtio/virtio_bus_if.m optional virtio dev/virtio/virtio_if.m optional virtio dev/virtio/pci/virtio_pci.c optional virtio_pci dev/virtio/mmio/virtio_mmio.c optional virtio_mmio dev/virtio/mmio/virtio_mmio_if.m optional virtio_mmio dev/virtio/network/if_vtnet.c optional vtnet dev/virtio/block/virtio_blk.c optional virtio_blk dev/virtio/balloon/virtio_balloon.c optional virtio_balloon dev/virtio/scsi/virtio_scsi.c optional virtio_scsi dev/virtio/random/virtio_random.c optional virtio_random dev/virtio/console/virtio_console.c optional virtio_console dev/vkbd/vkbd.c optional vkbd dev/vr/if_vr.c optional vr pci dev/vt/colors/vt_termcolors.c optional vt dev/vt/font/vt_font_default.c optional vt dev/vt/font/vt_mouse_cursor.c optional vt dev/vt/hw/efifb/efifb.c optional vt_efifb dev/vt/hw/fb/vt_fb.c optional vt dev/vt/hw/vga/vt_vga.c optional vt vt_vga dev/vt/logo/logo_freebsd.c optional vt splash dev/vt/vt_buf.c optional vt dev/vt/vt_consolectl.c optional vt dev/vt/vt_core.c optional vt dev/vt/vt_font.c optional vt dev/vt/vt_sysmouse.c optional vt dev/vte/if_vte.c optional vte pci dev/vx/if_vx.c optional vx dev/vx/if_vx_eisa.c optional vx eisa dev/vx/if_vx_pci.c optional vx pci dev/vxge/vxge.c optional vxge dev/vxge/vxgehal/vxgehal-ifmsg.c optional vxge dev/vxge/vxgehal/vxgehal-mrpcim.c optional vxge dev/vxge/vxgehal/vxge-queue.c optional vxge dev/vxge/vxgehal/vxgehal-ring.c optional vxge dev/vxge/vxgehal/vxgehal-swapper.c optional vxge dev/vxge/vxgehal/vxgehal-mgmt.c optional vxge dev/vxge/vxgehal/vxgehal-srpcim.c optional vxge dev/vxge/vxgehal/vxgehal-config.c optional vxge dev/vxge/vxgehal/vxgehal-blockpool.c optional vxge dev/vxge/vxgehal/vxgehal-doorbells.c optional vxge dev/vxge/vxgehal/vxgehal-mgmtaux.c optional vxge dev/vxge/vxgehal/vxgehal-device.c optional vxge dev/vxge/vxgehal/vxgehal-mm.c optional vxge dev/vxge/vxgehal/vxgehal-driver.c optional vxge dev/vxge/vxgehal/vxgehal-virtualpath.c optional vxge dev/vxge/vxgehal/vxgehal-channel.c optional vxge dev/vxge/vxgehal/vxgehal-fifo.c optional vxge dev/watchdog/watchdog.c standard dev/wb/if_wb.c optional wb pci dev/wds/wd7000.c optional wds isa dev/wi/if_wi.c optional wi dev/wi/if_wi_pccard.c optional wi pccard dev/wi/if_wi_pci.c optional wi pci dev/wl/if_wl.c optional wl isa dev/wpi/if_wpi.c optional wpi pci wpifw.c optional wpifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk wpi.fw:wpifw:153229 -mwpi -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "wpifw.c" wpifw.fwo optional wpifw \ dependency "wpi.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "wpifw.fwo" wpi.fw optional wpifw \ dependency "$S/contrib/dev/wpi/iwlwifi-3945-15.32.2.9.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "wpi.fw" dev/xe/if_xe.c optional xe dev/xe/if_xe_pccard.c optional xe pccard dev/xen/balloon/balloon.c optional xen | xenhvm dev/xen/blkfront/blkfront.c optional xen | xenhvm dev/xen/blkback/blkback.c optional xen | xenhvm dev/xen/console/console.c optional xen | xenhvm dev/xen/console/xencons_ring.c optional xen | xenhvm dev/xen/control/control.c optional xen | xenhvm dev/xen/grant_table/grant_table.c optional xen | xenhvm dev/xen/netback/netback.c optional xen | xenhvm dev/xen/netfront/netfront.c optional xen | xenhvm dev/xen/xenpci/xenpci.c optional xenpci dev/xen/timer/timer.c optional xen | xenhvm dev/xen/pvcpu/pvcpu.c optional xen | xenhvm dev/xen/xenstore/xenstore.c optional xen | xenhvm dev/xen/xenstore/xenstore_dev.c optional xen | xenhvm dev/xen/xenstore/xenstored_dev.c optional xen | xenhvm dev/xen/evtchn/evtchn_dev.c optional xen | xenhvm dev/xen/privcmd/privcmd.c optional xen | xenhvm dev/xl/if_xl.c optional xl pci dev/xl/xlphy.c optional xl pci fs/autofs/autofs.c optional autofs fs/autofs/autofs_vfsops.c optional autofs fs/autofs/autofs_vnops.c optional autofs fs/deadfs/dead_vnops.c standard fs/devfs/devfs_devs.c standard fs/devfs/devfs_dir.c standard fs/devfs/devfs_rule.c standard fs/devfs/devfs_vfsops.c standard fs/devfs/devfs_vnops.c standard fs/fdescfs/fdesc_vfsops.c optional fdescfs fs/fdescfs/fdesc_vnops.c optional fdescfs fs/fifofs/fifo_vnops.c standard fs/cuse/cuse.c optional cuse fs/fuse/fuse_device.c optional fuse fs/fuse/fuse_file.c optional fuse fs/fuse/fuse_internal.c optional fuse fs/fuse/fuse_io.c optional fuse fs/fuse/fuse_ipc.c optional fuse fs/fuse/fuse_main.c optional fuse fs/fuse/fuse_node.c optional fuse fs/fuse/fuse_vfsops.c optional fuse fs/fuse/fuse_vnops.c optional fuse fs/msdosfs/msdosfs_conv.c optional msdosfs fs/msdosfs/msdosfs_denode.c optional msdosfs fs/msdosfs/msdosfs_fat.c optional msdosfs fs/msdosfs/msdosfs_fileno.c optional msdosfs fs/msdosfs/msdosfs_iconv.c optional msdosfs_iconv fs/msdosfs/msdosfs_lookup.c optional msdosfs fs/msdosfs/msdosfs_vfsops.c optional msdosfs fs/msdosfs/msdosfs_vnops.c optional msdosfs fs/nandfs/bmap.c optional nandfs fs/nandfs/nandfs_alloc.c optional nandfs fs/nandfs/nandfs_bmap.c optional nandfs fs/nandfs/nandfs_buffer.c optional nandfs fs/nandfs/nandfs_cleaner.c optional nandfs fs/nandfs/nandfs_cpfile.c optional nandfs fs/nandfs/nandfs_dat.c optional nandfs fs/nandfs/nandfs_dir.c optional nandfs fs/nandfs/nandfs_ifile.c optional nandfs fs/nandfs/nandfs_segment.c optional nandfs fs/nandfs/nandfs_subr.c optional nandfs fs/nandfs/nandfs_sufile.c optional nandfs fs/nandfs/nandfs_vfsops.c optional nandfs fs/nandfs/nandfs_vnops.c optional nandfs fs/nfs/nfs_commonkrpc.c optional nfscl | nfsd fs/nfs/nfs_commonsubs.c optional nfscl | nfsd fs/nfs/nfs_commonport.c optional nfscl | nfsd fs/nfs/nfs_commonacl.c optional nfscl | nfsd fs/nfsclient/nfs_clcomsubs.c optional nfscl fs/nfsclient/nfs_clsubs.c optional nfscl fs/nfsclient/nfs_clstate.c optional nfscl fs/nfsclient/nfs_clkrpc.c optional nfscl fs/nfsclient/nfs_clrpcops.c optional nfscl fs/nfsclient/nfs_clvnops.c optional nfscl fs/nfsclient/nfs_clnode.c optional nfscl fs/nfsclient/nfs_clvfsops.c optional nfscl fs/nfsclient/nfs_clport.c optional nfscl fs/nfsclient/nfs_clbio.c optional nfscl fs/nfsclient/nfs_clnfsiod.c optional nfscl fs/nfsserver/nfs_fha_new.c optional nfsd inet fs/nfsserver/nfs_nfsdsocket.c optional nfsd inet fs/nfsserver/nfs_nfsdsubs.c optional nfsd inet fs/nfsserver/nfs_nfsdstate.c optional nfsd inet fs/nfsserver/nfs_nfsdkrpc.c optional nfsd inet fs/nfsserver/nfs_nfsdserv.c optional nfsd inet fs/nfsserver/nfs_nfsdport.c optional nfsd inet fs/nfsserver/nfs_nfsdcache.c optional nfsd inet fs/nullfs/null_subr.c optional nullfs fs/nullfs/null_vfsops.c optional nullfs fs/nullfs/null_vnops.c optional nullfs fs/procfs/procfs.c optional procfs fs/procfs/procfs_ctl.c optional procfs fs/procfs/procfs_dbregs.c optional procfs fs/procfs/procfs_fpregs.c optional procfs fs/procfs/procfs_ioctl.c optional procfs fs/procfs/procfs_map.c optional procfs fs/procfs/procfs_mem.c optional procfs fs/procfs/procfs_note.c optional procfs fs/procfs/procfs_osrel.c optional procfs fs/procfs/procfs_regs.c optional procfs fs/procfs/procfs_rlimit.c optional procfs fs/procfs/procfs_status.c optional procfs fs/procfs/procfs_type.c optional procfs fs/pseudofs/pseudofs.c optional pseudofs fs/pseudofs/pseudofs_fileno.c optional pseudofs fs/pseudofs/pseudofs_vncache.c optional pseudofs fs/pseudofs/pseudofs_vnops.c optional pseudofs fs/smbfs/smbfs_io.c optional smbfs fs/smbfs/smbfs_node.c optional smbfs fs/smbfs/smbfs_smb.c optional smbfs fs/smbfs/smbfs_subr.c optional smbfs fs/smbfs/smbfs_vfsops.c optional smbfs fs/smbfs/smbfs_vnops.c optional smbfs fs/udf/osta.c optional udf fs/udf/udf_iconv.c optional udf_iconv fs/udf/udf_vfsops.c optional udf fs/udf/udf_vnops.c optional udf fs/unionfs/union_subr.c optional unionfs fs/unionfs/union_vfsops.c optional unionfs fs/unionfs/union_vnops.c optional unionfs fs/tmpfs/tmpfs_vnops.c optional tmpfs fs/tmpfs/tmpfs_fifoops.c optional tmpfs fs/tmpfs/tmpfs_vfsops.c optional tmpfs fs/tmpfs/tmpfs_subr.c optional tmpfs gdb/gdb_cons.c optional gdb gdb/gdb_main.c optional gdb gdb/gdb_packet.c optional gdb geom/bde/g_bde.c optional geom_bde geom/bde/g_bde_crypt.c optional geom_bde geom/bde/g_bde_lock.c optional geom_bde geom/bde/g_bde_work.c optional geom_bde geom/cache/g_cache.c optional geom_cache geom/concat/g_concat.c optional geom_concat geom/eli/g_eli.c optional geom_eli geom/eli/g_eli_crypto.c optional geom_eli geom/eli/g_eli_ctl.c optional geom_eli geom/eli/g_eli_integrity.c optional geom_eli geom/eli/g_eli_key.c optional geom_eli geom/eli/g_eli_key_cache.c optional geom_eli geom/eli/g_eli_privacy.c optional geom_eli geom/eli/pkcs5v2.c optional geom_eli geom/gate/g_gate.c optional geom_gate geom/geom_aes.c optional geom_aes geom/geom_bsd.c optional geom_bsd geom/geom_bsd_enc.c optional geom_bsd | geom_part_bsd geom/geom_ccd.c optional ccd | geom_ccd geom/geom_ctl.c standard geom/geom_dev.c standard geom/geom_disk.c standard geom/geom_dump.c standard geom/geom_event.c standard geom/geom_fox.c optional geom_fox geom/geom_flashmap.c optional fdt cfi | fdt nand geom/geom_io.c standard geom/geom_kern.c standard geom/geom_map.c optional geom_map geom/geom_mbr.c optional geom_mbr geom/geom_mbr_enc.c optional geom_mbr geom/geom_pc98.c optional geom_pc98 geom/geom_pc98_enc.c optional geom_pc98 geom/geom_redboot.c optional geom_redboot geom/geom_slice.c standard geom/geom_subr.c standard geom/geom_sunlabel.c optional geom_sunlabel geom/geom_sunlabel_enc.c optional geom_sunlabel geom/geom_vfs.c standard geom/geom_vol_ffs.c optional geom_vol geom/journal/g_journal.c optional geom_journal geom/journal/g_journal_ufs.c optional geom_journal geom/label/g_label.c optional geom_label | geom_label_gpt geom/label/g_label_ext2fs.c optional geom_label geom/label/g_label_iso9660.c optional geom_label geom/label/g_label_msdosfs.c optional geom_label geom/label/g_label_ntfs.c optional geom_label geom/label/g_label_reiserfs.c optional geom_label geom/label/g_label_ufs.c optional geom_label geom/label/g_label_gpt.c optional geom_label | geom_label_gpt geom/label/g_label_disk_ident.c optional geom_label geom/linux_lvm/g_linux_lvm.c optional geom_linux_lvm geom/mirror/g_mirror.c optional geom_mirror geom/mirror/g_mirror_ctl.c optional geom_mirror geom/mountver/g_mountver.c optional geom_mountver geom/multipath/g_multipath.c optional geom_multipath geom/nop/g_nop.c optional geom_nop geom/part/g_part.c standard geom/part/g_part_if.m standard geom/part/g_part_apm.c optional geom_part_apm geom/part/g_part_bsd.c optional geom_part_bsd geom/part/g_part_bsd64.c optional geom_part_bsd64 geom/part/g_part_ebr.c optional geom_part_ebr geom/part/g_part_gpt.c optional geom_part_gpt geom/part/g_part_ldm.c optional geom_part_ldm geom/part/g_part_mbr.c optional geom_part_mbr geom/part/g_part_pc98.c optional geom_part_pc98 geom/part/g_part_vtoc8.c optional geom_part_vtoc8 geom/raid/g_raid.c optional geom_raid geom/raid/g_raid_ctl.c optional geom_raid geom/raid/g_raid_md_if.m optional geom_raid geom/raid/g_raid_tr_if.m optional geom_raid geom/raid/md_ddf.c optional geom_raid geom/raid/md_intel.c optional geom_raid geom/raid/md_jmicron.c optional geom_raid geom/raid/md_nvidia.c optional geom_raid geom/raid/md_promise.c optional geom_raid geom/raid/md_sii.c optional geom_raid geom/raid/tr_concat.c optional geom_raid geom/raid/tr_raid0.c optional geom_raid geom/raid/tr_raid1.c optional geom_raid geom/raid/tr_raid1e.c optional geom_raid geom/raid/tr_raid5.c optional geom_raid geom/raid3/g_raid3.c optional geom_raid3 geom/raid3/g_raid3_ctl.c optional geom_raid3 geom/shsec/g_shsec.c optional geom_shsec geom/stripe/g_stripe.c optional geom_stripe geom/uncompress/g_uncompress.c optional geom_uncompress contrib/xz-embedded/freebsd/xz_malloc.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_crc32.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_bcj.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_stream.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" geom/uzip/g_uzip.c optional geom_uzip geom/vinum/geom_vinum.c optional geom_vinum geom/vinum/geom_vinum_create.c optional geom_vinum geom/vinum/geom_vinum_drive.c optional geom_vinum geom/vinum/geom_vinum_plex.c optional geom_vinum geom/vinum/geom_vinum_volume.c optional geom_vinum geom/vinum/geom_vinum_subr.c optional geom_vinum geom/vinum/geom_vinum_raid5.c optional geom_vinum geom/vinum/geom_vinum_share.c optional geom_vinum geom/vinum/geom_vinum_list.c optional geom_vinum geom/vinum/geom_vinum_rm.c optional geom_vinum geom/vinum/geom_vinum_init.c optional geom_vinum geom/vinum/geom_vinum_state.c optional geom_vinum geom/vinum/geom_vinum_rename.c optional geom_vinum geom/vinum/geom_vinum_move.c optional geom_vinum geom/vinum/geom_vinum_events.c optional geom_vinum geom/virstor/binstream.c optional geom_virstor geom/virstor/g_virstor.c optional geom_virstor geom/virstor/g_virstor_md.c optional geom_virstor geom/zero/g_zero.c optional geom_zero fs/ext2fs/ext2_alloc.c optional ext2fs fs/ext2fs/ext2_balloc.c optional ext2fs fs/ext2fs/ext2_bmap.c optional ext2fs fs/ext2fs/ext2_extents.c optional ext2fs fs/ext2fs/ext2_inode.c optional ext2fs fs/ext2fs/ext2_inode_cnv.c optional ext2fs fs/ext2fs/ext2_hash.c optional ext2fs fs/ext2fs/ext2_htree.c optional ext2fs fs/ext2fs/ext2_lookup.c optional ext2fs fs/ext2fs/ext2_subr.c optional ext2fs fs/ext2fs/ext2_vfsops.c optional ext2fs fs/ext2fs/ext2_vnops.c optional ext2fs gnu/fs/reiserfs/reiserfs_hashes.c optional reiserfs \ warning "kernel contains GPL contaminated ReiserFS filesystem" gnu/fs/reiserfs/reiserfs_inode.c optional reiserfs gnu/fs/reiserfs/reiserfs_item_ops.c optional reiserfs gnu/fs/reiserfs/reiserfs_namei.c optional reiserfs gnu/fs/reiserfs/reiserfs_prints.c optional reiserfs gnu/fs/reiserfs/reiserfs_stree.c optional reiserfs gnu/fs/reiserfs/reiserfs_vfsops.c optional reiserfs gnu/fs/reiserfs/reiserfs_vnops.c optional reiserfs # isa/isa_if.m standard isa/isa_common.c optional isa isa/isahint.c optional isa isa/pnp.c optional isa isapnp isa/pnpparse.c optional isa isapnp fs/cd9660/cd9660_bmap.c optional cd9660 fs/cd9660/cd9660_lookup.c optional cd9660 fs/cd9660/cd9660_node.c optional cd9660 fs/cd9660/cd9660_rrip.c optional cd9660 fs/cd9660/cd9660_util.c optional cd9660 fs/cd9660/cd9660_vfsops.c optional cd9660 fs/cd9660/cd9660_vnops.c optional cd9660 fs/cd9660/cd9660_iconv.c optional cd9660_iconv kern/bus_if.m standard kern/clock_if.m standard kern/cpufreq_if.m standard kern/device_if.m standard kern/imgact_binmisc.c optional imagact_binmisc kern/imgact_elf.c standard kern/imgact_elf32.c optional compat_freebsd32 kern/imgact_shell.c standard kern/inflate.c optional gzip kern/init_main.c standard kern/init_sysent.c standard kern/ksched.c optional _kposix_priority_scheduling kern/kern_acct.c standard kern/kern_alq.c optional alq kern/kern_clock.c standard kern/kern_condvar.c standard kern/kern_conf.c standard kern/kern_cons.c standard kern/kern_cpu.c standard kern/kern_cpuset.c standard kern/kern_context.c standard kern/kern_descrip.c standard kern/kern_dtrace.c optional kdtrace_hooks kern/kern_dump.c standard kern/kern_environment.c standard kern/kern_et.c standard kern/kern_event.c standard kern/kern_exec.c standard kern/kern_exit.c standard kern/kern_fail.c standard kern/kern_ffclock.c standard kern/kern_fork.c standard kern/kern_gzio.c optional gzio kern/kern_hhook.c standard kern/kern_idle.c standard kern/kern_intr.c standard kern/kern_jail.c standard kern/kern_khelp.c standard kern/kern_kthread.c standard kern/kern_ktr.c optional ktr kern/kern_ktrace.c standard kern/kern_linker.c standard kern/kern_lock.c standard kern/kern_lockf.c standard kern/kern_lockstat.c optional kdtrace_hooks kern/kern_loginclass.c standard kern/kern_malloc.c standard kern/kern_mbuf.c standard kern/kern_mib.c standard kern/kern_module.c standard kern/kern_mtxpool.c standard kern/kern_mutex.c standard kern/kern_ntptime.c standard kern/kern_osd.c standard kern/kern_physio.c standard kern/kern_pmc.c standard kern/kern_poll.c optional device_polling kern/kern_priv.c standard kern/kern_proc.c standard kern/kern_procctl.c standard kern/kern_prot.c standard kern/kern_racct.c standard kern/kern_rangelock.c standard kern/kern_rctl.c standard kern/kern_resource.c standard kern/kern_rmlock.c standard kern/kern_rwlock.c standard kern/kern_sdt.c optional kdtrace_hooks kern/kern_sema.c standard kern/kern_sharedpage.c standard kern/kern_shutdown.c standard kern/kern_sig.c standard kern/kern_switch.c standard kern/kern_sx.c standard kern/kern_synch.c standard kern/kern_syscalls.c standard kern/kern_sysctl.c standard kern/kern_tc.c standard kern/kern_thr.c standard kern/kern_thread.c standard kern/kern_time.c standard kern/kern_timeout.c standard kern/kern_umtx.c standard kern/kern_uuid.c standard kern/kern_xxx.c standard kern/link_elf.c standard kern/linker_if.m standard kern/md4c.c optional netsmb kern/md5c.c standard kern/p1003_1b.c standard kern/posix4_mib.c standard kern/sched_4bsd.c optional sched_4bsd kern/sched_ule.c optional sched_ule kern/serdev_if.m standard kern/stack_protector.c standard \ compile-with "${NORMAL_C:N-fstack-protector*}" kern/subr_acl_nfs4.c optional ufs_acl | zfs kern/subr_acl_posix1e.c optional ufs_acl kern/subr_autoconf.c standard kern/subr_blist.c standard kern/subr_bus.c standard kern/subr_bus_dma.c standard kern/subr_bufring.c standard kern/subr_capability.c standard kern/subr_clock.c standard kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_eventhandler.c standard kern/subr_fattime.c standard kern/subr_firmware.c optional firmware kern/subr_hash.c standard kern/subr_hints.c standard kern/subr_kdb.c standard kern/subr_kobj.c standard kern/subr_lock.c standard kern/subr_log.c standard kern/subr_mbpool.c optional libmbpool kern/subr_mchain.c optional libmchain kern/subr_module.c standard kern/subr_msgbuf.c standard kern/subr_param.c standard kern/subr_pcpu.c standard kern/subr_pctrie.c standard kern/subr_power.c standard kern/subr_prf.c standard kern/subr_prof.c standard kern/subr_rman.c standard kern/subr_rtc.c standard kern/subr_sbuf.c standard kern/subr_scanf.c standard kern/subr_sglist.c standard kern/subr_sleepqueue.c standard kern/subr_smp.c standard kern/subr_stack.c optional ddb | stack | ktr kern/subr_taskqueue.c standard kern/subr_terminal.c optional vt kern/subr_trap.c standard kern/subr_turnstile.c standard kern/subr_uio.c standard kern/subr_unit.c standard kern/subr_vmem.c standard kern/subr_witness.c optional witness kern/sys_capability.c standard kern/sys_generic.c standard kern/sys_pipe.c standard kern/sys_procdesc.c standard kern/sys_process.c standard kern/sys_socket.c standard kern/syscalls.c standard kern/sysv_ipc.c standard kern/sysv_msg.c optional sysvmsg kern/sysv_sem.c optional sysvsem kern/sysv_shm.c optional sysvshm kern/tty.c standard kern/tty_compat.c optional compat_43tty kern/tty_info.c standard kern/tty_inq.c standard kern/tty_outq.c standard kern/tty_pts.c standard kern/tty_tty.c standard kern/tty_ttydisc.c standard kern/uipc_accf.c standard kern/uipc_debug.c optional ddb kern/uipc_domain.c standard kern/uipc_mbuf.c standard kern/uipc_mbuf2.c standard kern/uipc_mqueue.c optional p1003_1b_mqueue kern/uipc_sem.c optional p1003_1b_semaphores kern/uipc_shm.c standard kern/uipc_sockbuf.c standard kern/uipc_socket.c standard kern/uipc_syscalls.c standard kern/uipc_usrreq.c standard kern/vfs_acl.c standard kern/vfs_aio.c optional vfs_aio kern/vfs_bio.c standard kern/vfs_cache.c standard kern/vfs_cluster.c standard kern/vfs_default.c standard kern/vfs_export.c standard kern/vfs_extattr.c standard kern/vfs_hash.c standard kern/vfs_init.c standard kern/vfs_lookup.c standard kern/vfs_mount.c standard kern/vfs_mountroot.c standard kern/vfs_subr.c standard kern/vfs_syscalls.c standard kern/vfs_vnops.c standard # # Kernel GSS-API # gssd.h optional kgssapi \ dependency "$S/kgssapi/gssd.x" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -hM $S/kgssapi/gssd.x | grep -v pthread.h > gssd.h" \ no-obj no-implicit-rule before-depend local \ clean "gssd.h" gssd_xdr.c optional kgssapi \ dependency "$S/kgssapi/gssd.x gssd.h" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -c $S/kgssapi/gssd.x -o gssd_xdr.c" \ no-implicit-rule before-depend local \ clean "gssd_xdr.c" gssd_clnt.c optional kgssapi \ dependency "$S/kgssapi/gssd.x gssd.h" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -lM $S/kgssapi/gssd.x | grep -v string.h > gssd_clnt.c" \ no-implicit-rule before-depend local \ clean "gssd_clnt.c" kgssapi/gss_accept_sec_context.c optional kgssapi kgssapi/gss_add_oid_set_member.c optional kgssapi kgssapi/gss_acquire_cred.c optional kgssapi kgssapi/gss_canonicalize_name.c optional kgssapi kgssapi/gss_create_empty_oid_set.c optional kgssapi kgssapi/gss_delete_sec_context.c optional kgssapi kgssapi/gss_display_status.c optional kgssapi kgssapi/gss_export_name.c optional kgssapi kgssapi/gss_get_mic.c optional kgssapi kgssapi/gss_init_sec_context.c optional kgssapi kgssapi/gss_impl.c optional kgssapi kgssapi/gss_import_name.c optional kgssapi kgssapi/gss_names.c optional kgssapi kgssapi/gss_pname_to_uid.c optional kgssapi kgssapi/gss_release_buffer.c optional kgssapi kgssapi/gss_release_cred.c optional kgssapi kgssapi/gss_release_name.c optional kgssapi kgssapi/gss_release_oid_set.c optional kgssapi kgssapi/gss_set_cred_option.c optional kgssapi kgssapi/gss_test_oid_set_member.c optional kgssapi kgssapi/gss_unwrap.c optional kgssapi kgssapi/gss_verify_mic.c optional kgssapi kgssapi/gss_wrap.c optional kgssapi kgssapi/gss_wrap_size_limit.c optional kgssapi kgssapi/gssd_prot.c optional kgssapi kgssapi/krb5/krb5_mech.c optional kgssapi kgssapi/krb5/kcrypto.c optional kgssapi kgssapi/krb5/kcrypto_aes.c optional kgssapi kgssapi/krb5/kcrypto_arcfour.c optional kgssapi kgssapi/krb5/kcrypto_des.c optional kgssapi kgssapi/krb5/kcrypto_des3.c optional kgssapi kgssapi/kgss_if.m optional kgssapi kgssapi/gsstest.c optional kgssapi_debug # These files in libkern/ are those needed by all architectures. Some # of the files in libkern/ are only needed on some architectures, e.g., # libkern/divdi3.c is needed by i386 but not alpha. Also, some of these # routines may be optimized for a particular platform. In either case, # the file should be moved to conf/files. from here. # libkern/arc4random.c standard libkern/bcd.c standard libkern/bsearch.c standard libkern/crc32.c standard libkern/explicit_bzero.c standard libkern/fnmatch.c standard libkern/iconv.c optional libiconv libkern/iconv_converter_if.m optional libiconv libkern/iconv_ucs.c optional libiconv libkern/iconv_xlat.c optional libiconv libkern/iconv_xlat16.c optional libiconv libkern/inet_aton.c standard libkern/inet_ntoa.c standard libkern/inet_ntop.c standard libkern/inet_pton.c standard libkern/jenkins_hash.c standard libkern/murmur3_32.c standard libkern/mcount.c optional profiling-routine libkern/memcchr.c standard libkern/memchr.c optional fdt | gdb libkern/memcmp.c standard libkern/memmem.c optional gdb libkern/qsort.c standard libkern/qsort_r.c standard libkern/random.c standard libkern/scanc.c standard libkern/strcasecmp.c standard libkern/strcat.c standard libkern/strchr.c standard libkern/strcmp.c standard libkern/strcpy.c standard libkern/strcspn.c standard libkern/strdup.c standard libkern/strndup.c standard libkern/strlcat.c standard libkern/strlcpy.c standard libkern/strlen.c standard libkern/strncmp.c standard libkern/strncpy.c standard libkern/strnlen.c standard libkern/strrchr.c standard libkern/strsep.c standard libkern/strspn.c standard libkern/strstr.c standard libkern/strtol.c standard libkern/strtoq.c standard libkern/strtoul.c standard libkern/strtouq.c standard libkern/strvalid.c standard libkern/timingsafe_bcmp.c standard net/bpf.c standard net/bpf_buffer.c optional bpf net/bpf_jitter.c optional bpf_jitter net/bpf_filter.c optional bpf | netgraph_bpf net/bpf_zerocopy.c optional bpf net/bridgestp.c optional bridge | if_bridge net/flowtable.c optional flowtable inet | flowtable inet6 net/ieee8023ad_lacp.c optional lagg net/if.c standard net/if_arcsubr.c optional arcnet net/if_atmsubr.c optional atm net/if_bridge.c optional bridge inet | if_bridge inet net/if_clone.c standard net/if_dead.c standard net/if_debug.c optional ddb net/if_disc.c optional disc net/if_edsc.c optional edsc net/if_enc.c optional enc ipsec inet | enc ipsec inet6 net/if_epair.c optional epair net/if_ethersubr.c optional ether net/if_fddisubr.c optional fddi net/if_fwsubr.c optional fwip net/if_gif.c optional gif inet | gif inet6 | \ netgraph_gif inet | netgraph_gif inet6 net/if_gre.c optional gre inet | gre inet6 net/if_iso88025subr.c optional token net/if_lagg.c optional lagg net/if_loop.c optional loop net/if_llatbl.c standard net/if_me.c optional me inet net/if_media.c standard net/if_mib.c standard net/if_spppfr.c optional sppp | netgraph_sppp net/if_spppsubr.c optional sppp | netgraph_sppp net/if_stf.c optional stf inet inet6 net/if_tun.c optional tun net/if_tap.c optional tap net/if_vlan.c optional vlan net/if_vxlan.c optional vxlan inet | vxlan inet6 net/mppcc.c optional netgraph_mppc_compression net/mppcd.c optional netgraph_mppc_compression net/netisr.c standard net/pfil.c optional ether | inet net/radix.c standard net/radix_mpath.c standard net/raw_cb.c standard net/raw_usrreq.c standard net/route.c standard net/rss_config.c optional inet rss | inet6 rss net/rtsock.c standard net/slcompress.c optional netgraph_vjc | sppp | \ netgraph_sppp net/toeplitz.c optional inet rss | inet6 rss net/vnet.c optional vimage net/zlib.c optional crypto | geom_uzip | ipsec | \ mxge | netgraph_deflate | \ ddb_ctf | gzio | geom_uncompress net80211/ieee80211.c optional wlan net80211/ieee80211_acl.c optional wlan wlan_acl net80211/ieee80211_action.c optional wlan net80211/ieee80211_ageq.c optional wlan net80211/ieee80211_adhoc.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_ageq.c optional wlan net80211/ieee80211_amrr.c optional wlan | wlan_amrr net80211/ieee80211_crypto.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_crypto_ccmp.c optional wlan wlan_ccmp net80211/ieee80211_crypto_none.c optional wlan net80211/ieee80211_crypto_tkip.c optional wlan wlan_tkip net80211/ieee80211_crypto_wep.c optional wlan wlan_wep net80211/ieee80211_ddb.c optional wlan ddb net80211/ieee80211_dfs.c optional wlan net80211/ieee80211_freebsd.c optional wlan net80211/ieee80211_hostap.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_ht.c optional wlan net80211/ieee80211_hwmp.c optional wlan ieee80211_support_mesh net80211/ieee80211_input.c optional wlan net80211/ieee80211_ioctl.c optional wlan net80211/ieee80211_mesh.c optional wlan ieee80211_support_mesh \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_monitor.c optional wlan net80211/ieee80211_node.c optional wlan net80211/ieee80211_output.c optional wlan net80211/ieee80211_phy.c optional wlan net80211/ieee80211_power.c optional wlan net80211/ieee80211_proto.c optional wlan net80211/ieee80211_radiotap.c optional wlan net80211/ieee80211_ratectl.c optional wlan net80211/ieee80211_ratectl_none.c optional wlan net80211/ieee80211_regdomain.c optional wlan net80211/ieee80211_rssadapt.c optional wlan wlan_rssadapt net80211/ieee80211_scan.c optional wlan net80211/ieee80211_scan_sta.c optional wlan net80211/ieee80211_sta.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_superg.c optional wlan ieee80211_support_superg net80211/ieee80211_scan_sw.c optional wlan net80211/ieee80211_tdma.c optional wlan ieee80211_support_tdma net80211/ieee80211_wds.c optional wlan net80211/ieee80211_xauth.c optional wlan wlan_xauth net80211/ieee80211_alq.c optional wlan ieee80211_alq netgraph/atm/ccatm/ng_ccatm.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/ng_atm.c optional ngatm_atm netgraph/atm/ngatmbase.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscfu/ng_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscop/ng_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/uni/ng_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/bluetooth/common/ng_bluetooth.c optional netgraph_bluetooth netgraph/bluetooth/drivers/bt3c/ng_bt3c_pccard.c optional netgraph_bluetooth_bt3c netgraph/bluetooth/drivers/h4/ng_h4.c optional netgraph_bluetooth_h4 netgraph/bluetooth/drivers/ubt/ng_ubt.c optional netgraph_bluetooth_ubt usb netgraph/bluetooth/drivers/ubtbcmfw/ubtbcmfw.c optional netgraph_bluetooth_ubtbcmfw usb netgraph/bluetooth/hci/ng_hci_cmds.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_evnt.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_main.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_misc.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_ulpi.c optional netgraph_bluetooth_hci netgraph/bluetooth/l2cap/ng_l2cap_cmds.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_evnt.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_llpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_main.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_misc.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_ulpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/socket/ng_btsocket.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_hci_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_rfcomm.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_sco.c optional netgraph_bluetooth_socket netgraph/netflow/netflow.c optional netgraph_netflow netgraph/netflow/netflow_v9.c optional netgraph_netflow netgraph/netflow/ng_netflow.c optional netgraph_netflow netgraph/ng_UI.c optional netgraph_UI netgraph/ng_async.c optional netgraph_async netgraph/ng_atmllc.c optional netgraph_atmllc netgraph/ng_base.c optional netgraph netgraph/ng_bpf.c optional netgraph_bpf netgraph/ng_bridge.c optional netgraph_bridge netgraph/ng_car.c optional netgraph_car netgraph/ng_cisco.c optional netgraph_cisco netgraph/ng_deflate.c optional netgraph_deflate netgraph/ng_device.c optional netgraph_device netgraph/ng_echo.c optional netgraph_echo netgraph/ng_eiface.c optional netgraph_eiface netgraph/ng_ether.c optional netgraph_ether netgraph/ng_ether_echo.c optional netgraph_ether_echo netgraph/ng_frame_relay.c optional netgraph_frame_relay netgraph/ng_gif.c optional netgraph_gif inet6 | netgraph_gif inet netgraph/ng_gif_demux.c optional netgraph_gif_demux netgraph/ng_hole.c optional netgraph_hole netgraph/ng_iface.c optional netgraph_iface netgraph/ng_ip_input.c optional netgraph_ip_input netgraph/ng_ipfw.c optional netgraph_ipfw inet ipfirewall netgraph/ng_ksocket.c optional netgraph_ksocket netgraph/ng_l2tp.c optional netgraph_l2tp netgraph/ng_lmi.c optional netgraph_lmi netgraph/ng_mppc.c optional netgraph_mppc_compression | \ netgraph_mppc_encryption netgraph/ng_nat.c optional netgraph_nat inet libalias netgraph/ng_one2many.c optional netgraph_one2many netgraph/ng_parse.c optional netgraph netgraph/ng_patch.c optional netgraph_patch netgraph/ng_pipe.c optional netgraph_pipe netgraph/ng_ppp.c optional netgraph_ppp netgraph/ng_pppoe.c optional netgraph_pppoe netgraph/ng_pptpgre.c optional netgraph_pptpgre netgraph/ng_pred1.c optional netgraph_pred1 netgraph/ng_rfc1490.c optional netgraph_rfc1490 netgraph/ng_socket.c optional netgraph_socket netgraph/ng_split.c optional netgraph_split netgraph/ng_sppp.c optional netgraph_sppp netgraph/ng_tag.c optional netgraph_tag netgraph/ng_tcpmss.c optional netgraph_tcpmss netgraph/ng_tee.c optional netgraph_tee netgraph/ng_tty.c optional netgraph_tty netgraph/ng_vjc.c optional netgraph_vjc netgraph/ng_vlan.c optional netgraph_vlan netinet/accf_data.c optional accept_filter_data inet netinet/accf_dns.c optional accept_filter_dns inet netinet/accf_http.c optional accept_filter_http inet netinet/if_atm.c optional atm netinet/if_ether.c optional inet ether netinet/igmp.c optional inet netinet/in.c optional inet netinet/in_debug.c optional inet ddb netinet/in_kdtrace.c optional inet | inet6 netinet/ip_carp.c optional inet carp | inet6 carp netinet/in_gif.c optional gif inet | netgraph_gif inet netinet/ip_gre.c optional gre inet netinet/ip_id.c optional inet netinet/in_mcast.c optional inet netinet/in_pcb.c optional inet | inet6 netinet/in_pcbgroup.c optional inet pcbgroup | inet6 pcbgroup netinet/in_proto.c optional inet | inet6 netinet/in_rmx.c optional inet netinet/in_rss.c optional inet rss netinet/ip_divert.c optional inet ipdivert ipfirewall netinet/ip_ecn.c optional inet | inet6 netinet/ip_encap.c optional inet | inet6 netinet/ip_fastfwd.c optional inet netinet/ip_icmp.c optional inet | inet6 netinet/ip_input.c optional inet netinet/ip_ipsec.c optional inet ipsec netinet/ip_mroute.c optional mrouting inet netinet/ip_options.c optional inet netinet/ip_output.c optional inet netinet/raw_ip.c optional inet | inet6 netinet/cc/cc.c optional inet | inet6 netinet/cc/cc_newreno.c optional inet | inet6 netinet/sctp_asconf.c optional inet sctp | inet6 sctp netinet/sctp_auth.c optional inet sctp | inet6 sctp netinet/sctp_bsd_addr.c optional inet sctp | inet6 sctp netinet/sctp_cc_functions.c optional inet sctp | inet6 sctp netinet/sctp_crc32.c optional inet sctp | inet6 sctp netinet/sctp_indata.c optional inet sctp | inet6 sctp netinet/sctp_input.c optional inet sctp | inet6 sctp netinet/sctp_output.c optional inet sctp | inet6 sctp netinet/sctp_pcb.c optional inet sctp | inet6 sctp netinet/sctp_peeloff.c optional inet sctp | inet6 sctp netinet/sctp_ss_functions.c optional inet sctp | inet6 sctp netinet/sctp_syscalls.c optional inet sctp | inet6 sctp netinet/sctp_sysctl.c optional inet sctp | inet6 sctp netinet/sctp_timer.c optional inet sctp | inet6 sctp netinet/sctp_usrreq.c optional inet sctp | inet6 sctp netinet/sctputil.c optional inet sctp | inet6 sctp netinet/tcp_debug.c optional tcpdebug netinet/tcp_hostcache.c optional inet | inet6 netinet/tcp_input.c optional inet | inet6 netinet/tcp_lro.c optional inet | inet6 netinet/tcp_output.c optional inet | inet6 netinet/tcp_offload.c optional tcp_offload inet | tcp_offload inet6 netinet/tcp_reass.c optional inet | inet6 netinet/tcp_sack.c optional inet | inet6 netinet/tcp_subr.c optional inet | inet6 netinet/tcp_syncache.c optional inet | inet6 netinet/tcp_timer.c optional inet | inet6 netinet/tcp_timewait.c optional inet | inet6 netinet/tcp_usrreq.c optional inet | inet6 netinet/udp_usrreq.c optional inet | inet6 netinet/libalias/alias.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_db.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_mod.c optional libalias | netgraph_nat netinet/libalias/alias_proxy.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_util.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_sctp.c optional libalias inet | netgraph_nat inet netinet6/dest6.c optional inet6 netinet6/frag6.c optional inet6 netinet6/icmp6.c optional inet6 netinet6/in6.c optional inet6 netinet6/in6_cksum.c optional inet6 netinet6/in6_gif.c optional gif inet6 | netgraph_gif inet6 netinet6/in6_ifattach.c optional inet6 netinet6/in6_mcast.c optional inet6 netinet6/in6_pcb.c optional inet6 netinet6/in6_pcbgroup.c optional inet6 pcbgroup netinet6/in6_proto.c optional inet6 netinet6/in6_rmx.c optional inet6 netinet6/in6_rss.c optional inet6 rss netinet6/in6_src.c optional inet6 netinet6/ip6_forward.c optional inet6 netinet6/ip6_gre.c optional gre inet6 netinet6/ip6_id.c optional inet6 netinet6/ip6_input.c optional inet6 netinet6/ip6_mroute.c optional mrouting inet6 netinet6/ip6_output.c optional inet6 netinet6/ip6_ipsec.c optional inet6 ipsec netinet6/mld6.c optional inet6 netinet6/nd6.c optional inet6 netinet6/nd6_nbr.c optional inet6 netinet6/nd6_rtr.c optional inet6 netinet6/raw_ip6.c optional inet6 netinet6/route6.c optional inet6 netinet6/scope6.c optional inet6 netinet6/sctp6_usrreq.c optional inet6 sctp netinet6/udp6_usrreq.c optional inet6 netipsec/ipsec.c optional ipsec inet | ipsec inet6 netipsec/ipsec_input.c optional ipsec inet | ipsec inet6 netipsec/ipsec_mbuf.c optional ipsec inet | ipsec inet6 netipsec/ipsec_output.c optional ipsec inet | ipsec inet6 netipsec/key.c optional ipsec inet | ipsec inet6 netipsec/key_debug.c optional ipsec inet | ipsec inet6 netipsec/keysock.c optional ipsec inet | ipsec inet6 netipsec/xform_ah.c optional ipsec inet | ipsec inet6 netipsec/xform_esp.c optional ipsec inet | ipsec inet6 netipsec/xform_ipcomp.c optional ipsec inet | ipsec inet6 netipsec/xform_ipip.c optional ipsec inet | ipsec inet6 netipsec/xform_tcp.c optional ipsec inet tcp_signature | \ ipsec inet6 tcp_signature netnatm/natm.c optional natm netnatm/natm_pcb.c optional natm netnatm/natm_proto.c optional natm netpfil/ipfw/dn_heap.c optional inet dummynet netpfil/ipfw/dn_sched_fifo.c optional inet dummynet netpfil/ipfw/dn_sched_prio.c optional inet dummynet netpfil/ipfw/dn_sched_qfq.c optional inet dummynet netpfil/ipfw/dn_sched_rr.c optional inet dummynet netpfil/ipfw/dn_sched_wf2q.c optional inet dummynet netpfil/ipfw/ip_dummynet.c optional inet dummynet netpfil/ipfw/ip_dn_io.c optional inet dummynet netpfil/ipfw/ip_dn_glue.c optional inet dummynet netpfil/ipfw/ip_fw2.c optional inet ipfirewall netpfil/ipfw/ip_fw_dynamic.c optional inet ipfirewall netpfil/ipfw/ip_fw_log.c optional inet ipfirewall netpfil/ipfw/ip_fw_pfil.c optional inet ipfirewall netpfil/ipfw/ip_fw_sockopt.c optional inet ipfirewall netpfil/ipfw/ip_fw_table.c optional inet ipfirewall netpfil/ipfw/ip_fw_table_algo.c optional inet ipfirewall netpfil/ipfw/ip_fw_table_value.c optional inet ipfirewall netpfil/ipfw/ip_fw_iface.c optional inet ipfirewall netpfil/ipfw/ip_fw_nat.c optional inet ipfirewall_nat netpfil/pf/if_pflog.c optional pflog pf inet netpfil/pf/if_pfsync.c optional pfsync pf inet netpfil/pf/pf.c optional pf inet netpfil/pf/pf_if.c optional pf inet netpfil/pf/pf_ioctl.c optional pf inet netpfil/pf/pf_lb.c optional pf inet netpfil/pf/pf_norm.c optional pf inet netpfil/pf/pf_osfp.c optional pf inet netpfil/pf/pf_ruleset.c optional pf inet netpfil/pf/pf_table.c optional pf inet netpfil/pf/in4_cksum.c optional pf inet netsmb/smb_conn.c optional netsmb netsmb/smb_crypt.c optional netsmb netsmb/smb_dev.c optional netsmb netsmb/smb_iod.c optional netsmb netsmb/smb_rq.c optional netsmb netsmb/smb_smb.c optional netsmb netsmb/smb_subr.c optional netsmb netsmb/smb_trantcp.c optional netsmb netsmb/smb_usr.c optional netsmb nfs/bootp_subr.c optional bootp nfscl nfs/krpc_subr.c optional bootp nfscl nfs/nfs_diskless.c optional nfscl nfs_root nfs/nfs_fha.c optional nfsd nfs/nfs_lock.c optional nfscl | nfslockd | nfsd nfs/nfs_nfssvc.c optional nfscl | nfsd nlm/nlm_advlock.c optional nfslockd | nfsd nlm/nlm_prot_clnt.c optional nfslockd | nfsd nlm/nlm_prot_impl.c optional nfslockd | nfsd nlm/nlm_prot_server.c optional nfslockd | nfsd nlm/nlm_prot_svc.c optional nfslockd | nfsd nlm/nlm_prot_xdr.c optional nfslockd | nfsd nlm/sm_inter_xdr.c optional nfslockd | nfsd # Linux Kernel Compatibility API ofed/include/linux/linux_kmod.c optional ofed | compat_linuxapi \ no-depend compile-with "${OFED_C}" ofed/include/linux/linux_compat.c optional ofed | compat_linuxapi \ no-depend compile-with "${OFED_C}" ofed/include/linux/linux_pci.c optional ofed | compat_linuxapi \ no-depend compile-with "${OFED_C}" ofed/include/linux/linux_idr.c optional ofed | compat_linuxapi \ no-depend compile-with "${OFED_C}" ofed/include/linux/linux_radix.c optional ofed | compat_linuxapi \ no-depend compile-with "${OFED_C}" # OpenFabrics Enterprise Distribution (Infiniband) ofed/drivers/infiniband/core/addr.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/agent.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/cache.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" # XXX Mad.c must be ordered before cm.c for sysinit sets to occur in # the correct order. ofed/drivers/infiniband/core/mad.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/cm.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/ -Wno-unused-function" ofed/drivers/infiniband/core/cma.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/device.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/fmr_pool.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/iwcm.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/local_sa.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/mad_rmpp.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/multicast.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/notice.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/packer.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/sa_query.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/smi.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/sysfs.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ucm.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ucma.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ud_header.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/umem.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/user_mad.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_cmd.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_main.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_marshall.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/verbs.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/ulp/ipoib/ipoib_cm.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" #ofed/drivers/infiniband/ulp/ipoib/ipoib_fs.c optional ipoib \ # no-depend \ # compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_multicast.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_verbs.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" #ofed/drivers/infiniband/ulp/ipoib/ipoib_vlan.c optional ipoib \ # no-depend \ # compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_main.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_rx.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_cma.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_tx.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/hw/mlx4/alias_GUID.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/mcg.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/sysfs.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/cm.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/ah.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/cq.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/doorbell.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/mad.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/main.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/mr.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/qp.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/srq.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/wc.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/net/mlx4/alloc.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/catas.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/cmd.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/cq.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/eq.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/fw.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/icm.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/intf.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/main.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/mcg.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/ -Wno-unused" ofed/drivers/net/mlx4/mr.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/pd.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/port.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/profile.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/qp.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/reset.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/sense.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/srq.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/resource_tracker.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/sys_tune.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_cq.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/utils.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_main.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_netdev.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_port.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_resources.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_rx.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_tx.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/infiniband/hw/mthca/mthca_allocator.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_av.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_catas.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_cmd.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_cq.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_eq.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mad.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_main.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mcg.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_memfree.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mr.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_pd.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_profile.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_provider.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_qp.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_reset.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_srq.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_uar.c optional mthca \ no-depend compile-with "${OFED_C}" # crypto support opencrypto/cast.c optional crypto | ipsec opencrypto/criov.c optional crypto opencrypto/crypto.c optional crypto opencrypto/cryptodev.c optional cryptodev opencrypto/cryptodev_if.m optional crypto opencrypto/cryptosoft.c optional crypto opencrypto/cryptodeflate.c optional crypto opencrypto/gmac.c optional crypto opencrypto/gfmult.c optional crypto opencrypto/rmd160.c optional crypto | ipsec opencrypto/skipjack.c optional crypto opencrypto/xform.c optional crypto rpc/auth_none.c optional krpc | nfslockd | nfscl | nfsd rpc/auth_unix.c optional krpc | nfslockd | nfscl | nfsd rpc/authunix_prot.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_bck.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_dg.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_rc.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_vc.c optional krpc | nfslockd | nfscl | nfsd rpc/getnetconfig.c optional krpc | nfslockd | nfscl | nfsd rpc/replay.c optional krpc | nfslockd | nfscl | nfsd rpc/rpc_callmsg.c optional krpc | nfslockd | nfscl | nfsd rpc/rpc_generic.c optional krpc | nfslockd | nfscl | nfsd rpc/rpc_prot.c optional krpc | nfslockd | nfscl | nfsd rpc/rpcb_clnt.c optional krpc | nfslockd | nfscl | nfsd rpc/rpcb_prot.c optional krpc | nfslockd | nfscl | nfsd rpc/svc.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_auth.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_auth_unix.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_dg.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_generic.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_vc.c optional krpc | nfslockd | nfscl | nfsd rpc/rpcsec_gss/rpcsec_gss.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_conf.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_misc.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_prot.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/svc_rpcsec_gss.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi security/audit/audit.c optional audit security/audit/audit_arg.c optional audit security/audit/audit_bsm.c optional audit security/audit/audit_bsm_klib.c optional audit security/audit/audit_pipe.c optional audit security/audit/audit_syscalls.c standard security/audit/audit_trigger.c optional audit security/audit/audit_worker.c optional audit security/audit/bsm_domain.c optional audit security/audit/bsm_errno.c optional audit security/audit/bsm_fcntl.c optional audit security/audit/bsm_socket_type.c optional audit security/audit/bsm_token.c optional audit security/mac/mac_audit.c optional mac audit security/mac/mac_cred.c optional mac security/mac/mac_framework.c optional mac security/mac/mac_inet.c optional mac inet | mac inet6 security/mac/mac_inet6.c optional mac inet6 security/mac/mac_label.c optional mac security/mac/mac_net.c optional mac security/mac/mac_pipe.c optional mac security/mac/mac_posix_sem.c optional mac security/mac/mac_posix_shm.c optional mac security/mac/mac_priv.c optional mac security/mac/mac_process.c optional mac security/mac/mac_socket.c optional mac security/mac/mac_syscalls.c standard security/mac/mac_system.c optional mac security/mac/mac_sysv_msg.c optional mac security/mac/mac_sysv_sem.c optional mac security/mac/mac_sysv_shm.c optional mac security/mac/mac_vfs.c optional mac security/mac_biba/mac_biba.c optional mac_biba security/mac_bsdextended/mac_bsdextended.c optional mac_bsdextended security/mac_bsdextended/ugidfw_system.c optional mac_bsdextended security/mac_bsdextended/ugidfw_vnode.c optional mac_bsdextended security/mac_ifoff/mac_ifoff.c optional mac_ifoff security/mac_lomac/mac_lomac.c optional mac_lomac security/mac_mls/mac_mls.c optional mac_mls security/mac_none/mac_none.c optional mac_none security/mac_partition/mac_partition.c optional mac_partition security/mac_portacl/mac_portacl.c optional mac_portacl security/mac_seeotheruids/mac_seeotheruids.c optional mac_seeotheruids security/mac_stub/mac_stub.c optional mac_stub security/mac_test/mac_test.c optional mac_test teken/teken.c optional sc | vt ufs/ffs/ffs_alloc.c optional ffs ufs/ffs/ffs_balloc.c optional ffs ufs/ffs/ffs_inode.c optional ffs ufs/ffs/ffs_snapshot.c optional ffs ufs/ffs/ffs_softdep.c optional ffs ufs/ffs/ffs_subr.c optional ffs ufs/ffs/ffs_tables.c optional ffs ufs/ffs/ffs_vfsops.c optional ffs ufs/ffs/ffs_vnops.c optional ffs ufs/ffs/ffs_rawread.c optional ffs directio ufs/ffs/ffs_suspend.c optional ffs ufs/ufs/ufs_acl.c optional ffs ufs/ufs/ufs_bmap.c optional ffs ufs/ufs/ufs_dirhash.c optional ffs ufs/ufs/ufs_extattr.c optional ffs ufs/ufs/ufs_gjournal.c optional ffs UFS_GJOURNAL ufs/ufs/ufs_inode.c optional ffs ufs/ufs/ufs_lookup.c optional ffs ufs/ufs/ufs_quota.c optional ffs ufs/ufs/ufs_vfsops.c optional ffs ufs/ufs/ufs_vnops.c optional ffs vm/default_pager.c standard vm/device_pager.c standard vm/phys_pager.c standard vm/redzone.c optional DEBUG_REDZONE vm/sg_pager.c standard vm/swap_pager.c standard vm/uma_core.c standard vm/uma_dbg.c standard vm/memguard.c optional DEBUG_MEMGUARD vm/vm_fault.c standard vm/vm_glue.c standard vm/vm_init.c standard vm/vm_kern.c standard vm/vm_map.c standard vm/vm_meter.c standard vm/vm_mmap.c standard vm/vm_object.c standard vm/vm_page.c standard vm/vm_pageout.c standard vm/vm_pager.c standard vm/vm_phys.c standard vm/vm_radix.c standard vm/vm_reserv.c standard vm/vm_unix.c standard vm/vm_zeroidle.c standard vm/vnode_pager.c standard xen/features.c optional xen | xenhvm xen/xenbus/xenbus_if.m optional xen | xenhvm xen/xenbus/xenbus.c optional xen | xenhvm xen/xenbus/xenbusb_if.m optional xen | xenhvm xen/xenbus/xenbusb.c optional xen | xenhvm xen/xenbus/xenbusb_front.c optional xen | xenhvm xen/xenbus/xenbusb_back.c optional xen | xenhvm xdr/xdr.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_array.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_mbuf.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_mem.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_reference.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_sizeof.c optional krpc | nfslockd | nfscl | nfsd diff --git a/sys/dev/drm2/i915/intel_display.c b/sys/dev/drm2/i915/intel_display.c index 86c3a4de54cf..a844c7f55f49 100644 --- a/sys/dev/drm2/i915/intel_display.c +++ b/sys/dev/drm2/i915/intel_display.c @@ -1,7281 +1,7281 @@ /* * Copyright © 2006-2007 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. * * Authors: * Eric Anholt */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #define HAS_eDP (intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP)) bool intel_pipe_has_type(struct drm_crtc *crtc, int type); static void intel_increase_pllclock(struct drm_crtc *crtc); static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on); typedef struct { /* given values */ int n; int m1, m2; int p1, p2; /* derived values */ int dot; int vco; int m; int p; } intel_clock_t; typedef struct { int min, max; } intel_range_t; typedef struct { int dot_limit; int p2_slow, p2_fast; } intel_p2_t; #define INTEL_P2_NUM 2 typedef struct intel_limit intel_limit_t; struct intel_limit { intel_range_t dot, vco, n, m, m1, m2, p, p1; intel_p2_t p2; bool (* find_pll)(const intel_limit_t *, struct drm_crtc *, int, int, intel_clock_t *, intel_clock_t *); }; /* FDI */ #define IRONLAKE_FDI_FREQ 2700000 /* in kHz for mode->clock */ static bool intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock); static bool intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock); static bool intel_find_pll_g4x_dp(const intel_limit_t *, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock); static bool intel_find_pll_ironlake_dp(const intel_limit_t *, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock); static inline u32 /* units of 100MHz */ intel_fdi_link_freq(struct drm_device *dev) { if (IS_GEN5(dev)) { struct drm_i915_private *dev_priv = dev->dev_private; return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2; } else return 27; } static const intel_limit_t intel_limits_i8xx_dvo = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 930000, .max = 1400000 }, .n = { .min = 3, .max = 16 }, .m = { .min = 96, .max = 140 }, .m1 = { .min = 18, .max = 26 }, .m2 = { .min = 6, .max = 16 }, .p = { .min = 4, .max = 128 }, .p1 = { .min = 2, .max = 33 }, .p2 = { .dot_limit = 165000, .p2_slow = 4, .p2_fast = 2 }, .find_pll = intel_find_best_PLL, }; static const intel_limit_t intel_limits_i8xx_lvds = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 930000, .max = 1400000 }, .n = { .min = 3, .max = 16 }, .m = { .min = 96, .max = 140 }, .m1 = { .min = 18, .max = 26 }, .m2 = { .min = 6, .max = 16 }, .p = { .min = 4, .max = 128 }, .p1 = { .min = 1, .max = 6 }, .p2 = { .dot_limit = 165000, .p2_slow = 14, .p2_fast = 7 }, .find_pll = intel_find_best_PLL, }; static const intel_limit_t intel_limits_i9xx_sdvo = { .dot = { .min = 20000, .max = 400000 }, .vco = { .min = 1400000, .max = 2800000 }, .n = { .min = 1, .max = 6 }, .m = { .min = 70, .max = 120 }, .m1 = { .min = 10, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 5, .max = 80 }, .p1 = { .min = 1, .max = 8 }, .p2 = { .dot_limit = 200000, .p2_slow = 10, .p2_fast = 5 }, .find_pll = intel_find_best_PLL, }; static const intel_limit_t intel_limits_i9xx_lvds = { .dot = { .min = 20000, .max = 400000 }, .vco = { .min = 1400000, .max = 2800000 }, .n = { .min = 1, .max = 6 }, .m = { .min = 70, .max = 120 }, .m1 = { .min = 10, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 7, .max = 98 }, .p1 = { .min = 1, .max = 8 }, .p2 = { .dot_limit = 112000, .p2_slow = 14, .p2_fast = 7 }, .find_pll = intel_find_best_PLL, }; static const intel_limit_t intel_limits_g4x_sdvo = { .dot = { .min = 25000, .max = 270000 }, .vco = { .min = 1750000, .max = 3500000}, .n = { .min = 1, .max = 4 }, .m = { .min = 104, .max = 138 }, .m1 = { .min = 17, .max = 23 }, .m2 = { .min = 5, .max = 11 }, .p = { .min = 10, .max = 30 }, .p1 = { .min = 1, .max = 3}, .p2 = { .dot_limit = 270000, .p2_slow = 10, .p2_fast = 10 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_g4x_hdmi = { .dot = { .min = 22000, .max = 400000 }, .vco = { .min = 1750000, .max = 3500000}, .n = { .min = 1, .max = 4 }, .m = { .min = 104, .max = 138 }, .m1 = { .min = 16, .max = 23 }, .m2 = { .min = 5, .max = 11 }, .p = { .min = 5, .max = 80 }, .p1 = { .min = 1, .max = 8}, .p2 = { .dot_limit = 165000, .p2_slow = 10, .p2_fast = 5 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_g4x_single_channel_lvds = { .dot = { .min = 20000, .max = 115000 }, .vco = { .min = 1750000, .max = 3500000 }, .n = { .min = 1, .max = 3 }, .m = { .min = 104, .max = 138 }, .m1 = { .min = 17, .max = 23 }, .m2 = { .min = 5, .max = 11 }, .p = { .min = 28, .max = 112 }, .p1 = { .min = 2, .max = 8 }, .p2 = { .dot_limit = 0, .p2_slow = 14, .p2_fast = 14 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_g4x_dual_channel_lvds = { .dot = { .min = 80000, .max = 224000 }, .vco = { .min = 1750000, .max = 3500000 }, .n = { .min = 1, .max = 3 }, .m = { .min = 104, .max = 138 }, .m1 = { .min = 17, .max = 23 }, .m2 = { .min = 5, .max = 11 }, .p = { .min = 14, .max = 42 }, .p1 = { .min = 2, .max = 6 }, .p2 = { .dot_limit = 0, .p2_slow = 7, .p2_fast = 7 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_g4x_display_port = { .dot = { .min = 161670, .max = 227000 }, .vco = { .min = 1750000, .max = 3500000}, .n = { .min = 1, .max = 2 }, .m = { .min = 97, .max = 108 }, .m1 = { .min = 0x10, .max = 0x12 }, .m2 = { .min = 0x05, .max = 0x06 }, .p = { .min = 10, .max = 20 }, .p1 = { .min = 1, .max = 2}, .p2 = { .dot_limit = 0, .p2_slow = 10, .p2_fast = 10 }, .find_pll = intel_find_pll_g4x_dp, }; static const intel_limit_t intel_limits_pineview_sdvo = { .dot = { .min = 20000, .max = 400000}, .vco = { .min = 1700000, .max = 3500000 }, /* Pineview's Ncounter is a ring counter */ .n = { .min = 3, .max = 6 }, .m = { .min = 2, .max = 256 }, /* Pineview only has one combined m divider, which we treat as m2. */ .m1 = { .min = 0, .max = 0 }, .m2 = { .min = 0, .max = 254 }, .p = { .min = 5, .max = 80 }, .p1 = { .min = 1, .max = 8 }, .p2 = { .dot_limit = 200000, .p2_slow = 10, .p2_fast = 5 }, .find_pll = intel_find_best_PLL, }; static const intel_limit_t intel_limits_pineview_lvds = { .dot = { .min = 20000, .max = 400000 }, .vco = { .min = 1700000, .max = 3500000 }, .n = { .min = 3, .max = 6 }, .m = { .min = 2, .max = 256 }, .m1 = { .min = 0, .max = 0 }, .m2 = { .min = 0, .max = 254 }, .p = { .min = 7, .max = 112 }, .p1 = { .min = 1, .max = 8 }, .p2 = { .dot_limit = 112000, .p2_slow = 14, .p2_fast = 14 }, .find_pll = intel_find_best_PLL, }; /* Ironlake / Sandybridge * * We calculate clock using (register_value + 2) for N/M1/M2, so here * the range value for them is (actual_value - 2). */ static const intel_limit_t intel_limits_ironlake_dac = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 5 }, .m = { .min = 79, .max = 127 }, .m1 = { .min = 12, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 5, .max = 80 }, .p1 = { .min = 1, .max = 8 }, .p2 = { .dot_limit = 225000, .p2_slow = 10, .p2_fast = 5 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_ironlake_single_lvds = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 3 }, .m = { .min = 79, .max = 118 }, .m1 = { .min = 12, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 28, .max = 112 }, .p1 = { .min = 2, .max = 8 }, .p2 = { .dot_limit = 225000, .p2_slow = 14, .p2_fast = 14 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_ironlake_dual_lvds = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 3 }, .m = { .min = 79, .max = 127 }, .m1 = { .min = 12, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 14, .max = 56 }, .p1 = { .min = 2, .max = 8 }, .p2 = { .dot_limit = 225000, .p2_slow = 7, .p2_fast = 7 }, .find_pll = intel_g4x_find_best_PLL, }; /* LVDS 100mhz refclk limits. */ static const intel_limit_t intel_limits_ironlake_single_lvds_100m = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 2 }, .m = { .min = 79, .max = 126 }, .m1 = { .min = 12, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 28, .max = 112 }, .p1 = { .min = 2, .max = 8 }, .p2 = { .dot_limit = 225000, .p2_slow = 14, .p2_fast = 14 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 3 }, .m = { .min = 79, .max = 126 }, .m1 = { .min = 12, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 14, .max = 42 }, .p1 = { .min = 2, .max = 6 }, .p2 = { .dot_limit = 225000, .p2_slow = 7, .p2_fast = 7 }, .find_pll = intel_g4x_find_best_PLL, }; static const intel_limit_t intel_limits_ironlake_display_port = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000}, .n = { .min = 1, .max = 2 }, .m = { .min = 81, .max = 90 }, .m1 = { .min = 12, .max = 22 }, .m2 = { .min = 5, .max = 9 }, .p = { .min = 10, .max = 20 }, .p1 = { .min = 1, .max = 2}, .p2 = { .dot_limit = 0, .p2_slow = 10, .p2_fast = 10 }, .find_pll = intel_find_pll_ironlake_dp, }; u32 intel_dpio_read(struct drm_i915_private *dev_priv, int reg) { u32 val = 0; mtx_lock(&dev_priv->dpio_lock); if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) { DRM_ERROR("DPIO idle wait timed out\n"); goto out_unlock; } I915_WRITE(DPIO_REG, reg); I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_READ | DPIO_PORTID | DPIO_BYTE); if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) { DRM_ERROR("DPIO read wait timed out\n"); goto out_unlock; } val = I915_READ(DPIO_DATA); out_unlock: mtx_unlock(&dev_priv->dpio_lock); return val; } #if 0 static void intel_dpio_write(struct drm_i915_private *dev_priv, int reg, u32 val) { mtx_lock(&dev_priv->dpio_lock); if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) { DRM_ERROR("DPIO idle wait timed out\n"); goto out_unlock; } I915_WRITE(DPIO_DATA, val); I915_WRITE(DPIO_REG, reg); I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_WRITE | DPIO_PORTID | DPIO_BYTE); if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) DRM_ERROR("DPIO write wait timed out\n"); out_unlock: mtx_unlock(&dev_priv->dpio_lock); } #endif static void vlv_init_dpio(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; /* Reset the DPIO config */ I915_WRITE(DPIO_CTL, 0); POSTING_READ(DPIO_CTL); I915_WRITE(DPIO_CTL, 1); POSTING_READ(DPIO_CTL); } static int intel_dual_link_lvds_callback(const struct dmi_system_id *id) { DRM_INFO("Forcing lvds to dual link mode on %s\n", id->ident); return 1; } static const struct dmi_system_id intel_dual_link_lvds[] = { { .callback = intel_dual_link_lvds_callback, .ident = "Apple MacBook Pro (Core i5/i7 Series)", .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc."), DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro8,2"), }, }, { } /* terminating entry */ }; static bool is_dual_link_lvds(struct drm_i915_private *dev_priv, unsigned int reg) { unsigned int val; /* use the module option value if specified */ if (i915_lvds_channel_mode > 0) return i915_lvds_channel_mode == 2; if (dmi_check_system(intel_dual_link_lvds)) return true; if (dev_priv->lvds_val) val = dev_priv->lvds_val; else { /* BIOS should set the proper LVDS register value at boot, but * in reality, it doesn't set the value when the lid is closed; * we need to check "the value to be set" in VBT when LVDS * register is uninitialized. */ val = I915_READ(reg); if (!(val & ~LVDS_DETECTED)) val = dev_priv->bios_lvds_val; dev_priv->lvds_val = val; } return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP; } static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc, int refclk) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; const intel_limit_t *limit; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) { if (is_dual_link_lvds(dev_priv, PCH_LVDS)) { /* LVDS dual channel */ if (refclk == 100000) limit = &intel_limits_ironlake_dual_lvds_100m; else limit = &intel_limits_ironlake_dual_lvds; } else { if (refclk == 100000) limit = &intel_limits_ironlake_single_lvds_100m; else limit = &intel_limits_ironlake_single_lvds; } } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) || HAS_eDP) limit = &intel_limits_ironlake_display_port; else limit = &intel_limits_ironlake_dac; return limit; } static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; const intel_limit_t *limit; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) { if (is_dual_link_lvds(dev_priv, LVDS)) /* LVDS with dual channel */ limit = &intel_limits_g4x_dual_channel_lvds; else /* LVDS with dual channel */ limit = &intel_limits_g4x_single_channel_lvds; } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI) || intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG)) { limit = &intel_limits_g4x_hdmi; } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO)) { limit = &intel_limits_g4x_sdvo; } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) { limit = &intel_limits_g4x_display_port; } else /* The option is for other outputs */ limit = &intel_limits_i9xx_sdvo; return limit; } static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk) { struct drm_device *dev = crtc->dev; const intel_limit_t *limit; if (HAS_PCH_SPLIT(dev)) limit = intel_ironlake_limit(crtc, refclk); else if (IS_G4X(dev)) { limit = intel_g4x_limit(crtc); } else if (IS_PINEVIEW(dev)) { if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) limit = &intel_limits_pineview_lvds; else limit = &intel_limits_pineview_sdvo; } else if (!IS_GEN2(dev)) { if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) limit = &intel_limits_i9xx_lvds; else limit = &intel_limits_i9xx_sdvo; } else { if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) limit = &intel_limits_i8xx_lvds; else limit = &intel_limits_i8xx_dvo; } return limit; } /* m1 is reserved as 0 in Pineview, n is a ring counter */ static void pineview_clock(int refclk, intel_clock_t *clock) { clock->m = clock->m2 + 2; clock->p = clock->p1 * clock->p2; clock->vco = refclk * clock->m / clock->n; clock->dot = clock->vco / clock->p; } static void intel_clock(struct drm_device *dev, int refclk, intel_clock_t *clock) { if (IS_PINEVIEW(dev)) { pineview_clock(refclk, clock); return; } clock->m = 5 * (clock->m1 + 2) + (clock->m2 + 2); clock->p = clock->p1 * clock->p2; clock->vco = refclk * clock->m / (clock->n + 2); clock->dot = clock->vco / clock->p; } /** * Returns whether any output on the specified pipe is of the specified type */ bool intel_pipe_has_type(struct drm_crtc *crtc, int type) { struct drm_device *dev = crtc->dev; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder; list_for_each_entry(encoder, &mode_config->encoder_list, base.head) if (encoder->base.crtc == crtc && encoder->type == type) return true; return false; } #define INTELPllInvalid(s) do { /* DRM_DEBUG(s); */ return false; } while (0) /** * Returns whether the given set of divisors are valid for a given refclk with * the given connectors. */ static bool intel_PLL_is_valid(struct drm_device *dev, const intel_limit_t *limit, const intel_clock_t *clock) { if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1) INTELPllInvalid("p1 out of range\n"); if (clock->p < limit->p.min || limit->p.max < clock->p) INTELPllInvalid("p out of range\n"); if (clock->m2 < limit->m2.min || limit->m2.max < clock->m2) INTELPllInvalid("m2 out of range\n"); if (clock->m1 < limit->m1.min || limit->m1.max < clock->m1) INTELPllInvalid("m1 out of range\n"); if (clock->m1 <= clock->m2 && !IS_PINEVIEW(dev)) INTELPllInvalid("m1 <= m2\n"); if (clock->m < limit->m.min || limit->m.max < clock->m) INTELPllInvalid("m out of range\n"); if (clock->n < limit->n.min || limit->n.max < clock->n) INTELPllInvalid("n out of range\n"); if (clock->vco < limit->vco.min || limit->vco.max < clock->vco) INTELPllInvalid("vco out of range\n"); /* XXX: We may need to be checking "Dot clock" depending on the multiplier, * connector, etc., rather than just a single range. */ if (clock->dot < limit->dot.min || limit->dot.max < clock->dot) INTELPllInvalid("dot out of range\n"); return true; } static bool intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; intel_clock_t clock; int err = target; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) && (I915_READ(LVDS)) != 0) { /* * For LVDS, if the panel is on, just rely on its current * settings for dual-channel. We haven't figured out how to * reliably set up different single/dual channel state, if we * even can. */ if (is_dual_link_lvds(dev_priv, LVDS)) clock.p2 = limit->p2.p2_fast; else clock.p2 = limit->p2.p2_slow; } else { if (target < limit->p2.dot_limit) clock.p2 = limit->p2.p2_slow; else clock.p2 = limit->p2.p2_fast; } memset(best_clock, 0, sizeof(*best_clock)); for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) { for (clock.m2 = limit->m2.min; clock.m2 <= limit->m2.max; clock.m2++) { /* m1 is always 0 in Pineview */ if (clock.m2 >= clock.m1 && !IS_PINEVIEW(dev)) break; for (clock.n = limit->n.min; clock.n <= limit->n.max; clock.n++) { for (clock.p1 = limit->p1.min; clock.p1 <= limit->p1.max; clock.p1++) { int this_err; intel_clock(dev, refclk, &clock); if (!intel_PLL_is_valid(dev, limit, &clock)) continue; if (match_clock && clock.p != match_clock->p) continue; this_err = abs(clock.dot - target); if (this_err < err) { *best_clock = clock; err = this_err; } } } } } return (err != target); } static bool intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; intel_clock_t clock; int max_n; bool found; /* approximately equals target * 0.00585 */ int err_most = (target >> 8) + (target >> 9); found = false; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) { int lvds_reg; if (HAS_PCH_SPLIT(dev)) lvds_reg = PCH_LVDS; else lvds_reg = LVDS; if ((I915_READ(lvds_reg) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP) clock.p2 = limit->p2.p2_fast; else clock.p2 = limit->p2.p2_slow; } else { if (target < limit->p2.dot_limit) clock.p2 = limit->p2.p2_slow; else clock.p2 = limit->p2.p2_fast; } memset(best_clock, 0, sizeof(*best_clock)); max_n = limit->n.max; /* based on hardware requirement, prefer smaller n to precision */ for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) { /* based on hardware requirement, prefere larger m1,m2 */ for (clock.m1 = limit->m1.max; clock.m1 >= limit->m1.min; clock.m1--) { for (clock.m2 = limit->m2.max; clock.m2 >= limit->m2.min; clock.m2--) { for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) { int this_err; intel_clock(dev, refclk, &clock); if (!intel_PLL_is_valid(dev, limit, &clock)) continue; if (match_clock && clock.p != match_clock->p) continue; this_err = abs(clock.dot - target); if (this_err < err_most) { *best_clock = clock; err_most = this_err; max_n = clock.n; found = true; } } } } } return found; } static bool intel_find_pll_ironlake_dp(const intel_limit_t *limit, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock) { struct drm_device *dev = crtc->dev; intel_clock_t clock; if (target < 200000) { clock.n = 1; clock.p1 = 2; clock.p2 = 10; clock.m1 = 12; clock.m2 = 9; } else { clock.n = 2; clock.p1 = 1; clock.p2 = 10; clock.m1 = 14; clock.m2 = 8; } intel_clock(dev, refclk, &clock); memcpy(best_clock, &clock, sizeof(intel_clock_t)); return true; } /* DisplayPort has only two frequencies, 162MHz and 270MHz */ static bool intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock) { intel_clock_t clock; if (target < 200000) { clock.p1 = 2; clock.p2 = 10; clock.n = 2; clock.m1 = 23; clock.m2 = 8; } else { clock.p1 = 1; clock.p2 = 10; clock.n = 1; clock.m1 = 14; clock.m2 = 2; } clock.m = 5 * (clock.m1 + 2) + (clock.m2 + 2); clock.p = (clock.p1 * clock.p2); clock.dot = 96000 * clock.m / (clock.n + 2) / clock.p; clock.vco = 0; memcpy(best_clock, &clock, sizeof(intel_clock_t)); return true; } static void ironlake_wait_for_vblank(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; u32 frame, frame_reg = PIPEFRAME(pipe); frame = I915_READ(frame_reg); if (wait_for(I915_READ_NOTRACE(frame_reg) != frame, 50)) DRM_DEBUG_KMS("vblank wait timed out\n"); } /** * intel_wait_for_vblank - wait for vblank on a given pipe * @dev: drm device * @pipe: pipe to wait for * * Wait for vblank to occur on a given pipe. Needed for various bits of * mode setting code. */ void intel_wait_for_vblank(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; int pipestat_reg = PIPESTAT(pipe); if (INTEL_INFO(dev)->gen >= 5) { ironlake_wait_for_vblank(dev, pipe); return; } /* Clear existing vblank status. Note this will clear any other * sticky status fields as well. * * This races with i915_driver_irq_handler() with the result * that either function could miss a vblank event. Here it is not * fatal, as we will either wait upon the next vblank interrupt or * timeout. Generally speaking intel_wait_for_vblank() is only * called during modeset at which time the GPU should be idle and * should *not* be performing page flips and thus not waiting on * vblanks... * Currently, the result of us stealing a vblank from the irq * handler is that a single frame will be skipped during swapbuffers. */ I915_WRITE(pipestat_reg, I915_READ(pipestat_reg) | PIPE_VBLANK_INTERRUPT_STATUS); /* Wait for vblank interrupt bit to set */ if (_intel_wait_for(dev, I915_READ(pipestat_reg) & PIPE_VBLANK_INTERRUPT_STATUS, 50, 1, "915vbl")) DRM_DEBUG_KMS("vblank wait timed out\n"); } /* * intel_wait_for_pipe_off - wait for pipe to turn off * @dev: drm device * @pipe: pipe to wait for * * After disabling a pipe, we can't wait for vblank in the usual way, * spinning on the vblank interrupt status bit, since we won't actually * see an interrupt when the pipe is disabled. * * On Gen4 and above: * wait for the pipe register state bit to turn off * * Otherwise: * wait for the display line value to settle (it usually * ends up stopping at the start of the next frame). * */ void intel_wait_for_pipe_off(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; if (INTEL_INFO(dev)->gen >= 4) { int reg = PIPECONF(pipe); /* Wait for the Pipe State to go off */ if (_intel_wait_for(dev, (I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0, 100, 1, "915pip")) DRM_DEBUG_KMS("pipe_off wait timed out\n"); } else { u32 last_line, line_mask; int reg = PIPEDSL(pipe); unsigned long timeout = jiffies + msecs_to_jiffies(100); if (IS_GEN2(dev)) line_mask = DSL_LINEMASK_GEN2; else line_mask = DSL_LINEMASK_GEN3; /* Wait for the display line to settle */ do { last_line = I915_READ(reg) & line_mask; DELAY(5000); } while (((I915_READ(reg) & line_mask) != last_line) && time_after(timeout, jiffies)); if (time_after(jiffies, timeout)) DRM_DEBUG_KMS("pipe_off wait timed out\n"); } } static const char *state_string(bool enabled) { return enabled ? "on" : "off"; } /* Only for pre-ILK configs */ static void assert_pll(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { int reg; u32 val; bool cur_state; reg = DPLL(pipe); val = I915_READ(reg); cur_state = !!(val & DPLL_VCO_ENABLE); if (cur_state != state) printf("PLL state assertion failure (expected %s, current %s)\n", state_string(state), state_string(cur_state)); } #define assert_pll_enabled(d, p) assert_pll(d, p, true) #define assert_pll_disabled(d, p) assert_pll(d, p, false) /* For ILK+ */ static void assert_pch_pll(struct drm_i915_private *dev_priv, struct intel_crtc *intel_crtc, bool state) { int reg; u32 val; bool cur_state; if (HAS_PCH_LPT(dev_priv->dev)) { DRM_DEBUG_DRIVER("LPT detected: skipping PCH PLL test\n"); return; } if (!intel_crtc->pch_pll) { printf("asserting PCH PLL enabled with no PLL\n"); return; } if (HAS_PCH_CPT(dev_priv->dev)) { u32 pch_dpll; pch_dpll = I915_READ(PCH_DPLL_SEL); /* Make sure the selected PLL is enabled to the transcoder */ KASSERT(((pch_dpll >> (4 * intel_crtc->pipe)) & 8) != 0, ("transcoder %d PLL not enabled\n", intel_crtc->pipe)); } reg = intel_crtc->pch_pll->pll_reg; val = I915_READ(reg); cur_state = !!(val & DPLL_VCO_ENABLE); if (cur_state != state) printf("PCH PLL state assertion failure (expected %s, current %s)\n", state_string(state), state_string(cur_state)); } #define assert_pch_pll_enabled(d, p) assert_pch_pll(d, p, true) #define assert_pch_pll_disabled(d, p) assert_pch_pll(d, p, false) static void assert_fdi_tx(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { int reg; u32 val; bool cur_state; if (IS_HASWELL(dev_priv->dev)) { /* On Haswell, DDI is used instead of FDI_TX_CTL */ reg = DDI_FUNC_CTL(pipe); val = I915_READ(reg); cur_state = !!(val & PIPE_DDI_FUNC_ENABLE); } else { reg = FDI_TX_CTL(pipe); val = I915_READ(reg); cur_state = !!(val & FDI_TX_ENABLE); } if (cur_state != state) printf("FDI TX state assertion failure (expected %s, current %s)\n", state_string(state), state_string(cur_state)); } #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true) #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false) static void assert_fdi_rx(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { int reg; u32 val; bool cur_state; if (IS_HASWELL(dev_priv->dev) && pipe > 0) { DRM_ERROR("Attempting to enable FDI_RX on Haswell pipe > 0\n"); return; } else { reg = FDI_RX_CTL(pipe); val = I915_READ(reg); cur_state = !!(val & FDI_RX_ENABLE); } if (cur_state != state) printf("FDI RX state assertion failure (expected %s, current %s)\n", state_string(state), state_string(cur_state)); } #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true) #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false) static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; /* ILK FDI PLL is always enabled */ if (dev_priv->info->gen == 5) return; /* On Haswell, DDI ports are responsible for the FDI PLL setup */ if (IS_HASWELL(dev_priv->dev)) return; reg = FDI_TX_CTL(pipe); val = I915_READ(reg); if (!(val & FDI_TX_PLL_ENABLE)) printf("FDI TX PLL assertion failure, should be active but is disabled\n"); } static void assert_fdi_rx_pll_enabled(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; if (IS_HASWELL(dev_priv->dev) && pipe > 0) { DRM_ERROR("Attempting to enable FDI on Haswell with pipe > 0\n"); return; } reg = FDI_RX_CTL(pipe); val = I915_READ(reg); if (!(val & FDI_RX_PLL_ENABLE)) printf("FDI RX PLL assertion failure, should be active but is disabled\n"); } static void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) { int pp_reg, lvds_reg; u32 val; enum pipe panel_pipe = PIPE_A; bool locked = true; if (HAS_PCH_SPLIT(dev_priv->dev)) { pp_reg = PCH_PP_CONTROL; lvds_reg = PCH_LVDS; } else { pp_reg = PP_CONTROL; lvds_reg = LVDS; } val = I915_READ(pp_reg); if (!(val & PANEL_POWER_ON) || ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS)) locked = false; if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT) panel_pipe = PIPE_B; if (panel_pipe == pipe && locked) printf("panel assertion failure, pipe %c regs locked\n", pipe_name(pipe)); } void assert_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { int reg; u32 val; bool cur_state; /* if we need the pipe A quirk it must be always on */ if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) state = true; reg = PIPECONF(pipe); val = I915_READ(reg); cur_state = !!(val & PIPECONF_ENABLE); if (cur_state != state) printf("pipe %c assertion failure (expected %s, current %s)\n", pipe_name(pipe), state_string(state), state_string(cur_state)); } static void assert_plane(struct drm_i915_private *dev_priv, enum plane plane, bool state) { int reg; u32 val; bool cur_state; reg = DSPCNTR(plane); val = I915_READ(reg); cur_state = !!(val & DISPLAY_PLANE_ENABLE); if (cur_state != state) printf("plane %c assertion failure, (expected %s, current %s)\n", plane_name(plane), state_string(state), state_string(cur_state)); } #define assert_plane_enabled(d, p) assert_plane(d, p, true) #define assert_plane_disabled(d, p) assert_plane(d, p, false) static void assert_planes_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg, i; u32 val; int cur_pipe; /* Planes are fixed to pipes on ILK+ */ if (HAS_PCH_SPLIT(dev_priv->dev)) { reg = DSPCNTR(pipe); val = I915_READ(reg); if ((val & DISPLAY_PLANE_ENABLE) != 0) printf("plane %c assertion failure, should be disabled but not\n", plane_name(pipe)); return; } /* Need to check both planes against the pipe */ for (i = 0; i < 2; i++) { reg = DSPCNTR(i); val = I915_READ(reg); cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >> DISPPLANE_SEL_PIPE_SHIFT; if ((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe) printf("plane %c assertion failure, should be off on pipe %c but is still active\n", plane_name(i), pipe_name(pipe)); } } static void assert_pch_refclk_enabled(struct drm_i915_private *dev_priv) { u32 val; bool enabled; if (HAS_PCH_LPT(dev_priv->dev)) { DRM_DEBUG_DRIVER("LPT does not has PCH refclk, skipping check\n"); return; } val = I915_READ(PCH_DREF_CONTROL); enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK | DREF_SUPERSPREAD_SOURCE_MASK)); if (!enabled) printf("PCH refclk assertion failure, should be active but is disabled\n"); } static void assert_transcoder_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; bool enabled; reg = TRANSCONF(pipe); val = I915_READ(reg); enabled = !!(val & TRANS_ENABLE); if (enabled) printf("transcoder assertion failed, should be off on pipe %c but is still active\n", pipe_name(pipe)); } static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv, enum pipe pipe, u32 val) { if ((val & PORT_ENABLE) == 0) return false; if (HAS_PCH_CPT(dev_priv->dev)) { if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe)) return false; } else { if ((val & TRANSCODER_MASK) != TRANSCODER(pipe)) return false; } return true; } static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv, enum pipe pipe, u32 val) { if ((val & LVDS_PORT_EN) == 0) return false; if (HAS_PCH_CPT(dev_priv->dev)) { if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe)) return false; } else { if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe)) return false; } return true; } static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv, enum pipe pipe, u32 val) { if ((val & ADPA_DAC_ENABLE) == 0) return false; if (HAS_PCH_CPT(dev_priv->dev)) { if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe)) return false; } else { if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe)) return false; } return true; } static bool dp_pipe_enabled(struct drm_i915_private *dev_priv, enum pipe pipe, u32 port_sel, u32 val) { if ((val & DP_PORT_EN) == 0) return false; if (HAS_PCH_CPT(dev_priv->dev)) { u32 trans_dp_ctl_reg = TRANS_DP_CTL(pipe); u32 trans_dp_ctl = I915_READ(trans_dp_ctl_reg); if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel) return false; } else { if ((val & DP_PIPE_MASK) != (pipe << 30)) return false; } return true; } static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv, enum pipe pipe, int reg, u32 port_sel) { u32 val = I915_READ(reg); if (dp_pipe_enabled(dev_priv, pipe, port_sel, val)) printf("PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n", reg, pipe_name(pipe)); } static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv, enum pipe pipe, int reg) { u32 val = I915_READ(reg); if (hdmi_pipe_enabled(dev_priv, val, pipe)) printf("PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n", reg, pipe_name(pipe)); } static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B); assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C); assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D); reg = PCH_ADPA; val = I915_READ(reg); if (adpa_pipe_enabled(dev_priv, val, pipe)) printf("PCH VGA enabled on transcoder %c, should be disabled\n", pipe_name(pipe)); reg = PCH_LVDS; val = I915_READ(reg); if (lvds_pipe_enabled(dev_priv, val, pipe)) printf("PCH LVDS enabled on transcoder %c, should be disabled\n", pipe_name(pipe)); assert_pch_hdmi_disabled(dev_priv, pipe, HDMIB); assert_pch_hdmi_disabled(dev_priv, pipe, HDMIC); assert_pch_hdmi_disabled(dev_priv, pipe, HDMID); } /** * intel_enable_pll - enable a PLL * @dev_priv: i915 private structure * @pipe: pipe PLL to enable * * Enable @pipe's PLL so we can start pumping pixels from a plane. Check to * make sure the PLL reg is writable first though, since the panel write * protect mechanism may be enabled. * * Note! This is for pre-ILK only. */ static void intel_enable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; /* No really, not for ILK+ */ KASSERT(dev_priv->info->gen < 5, ("Wrong device gen")); /* PLL is protected by panel, make sure we can write it */ if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev)) assert_panel_unlocked(dev_priv, pipe); reg = DPLL(pipe); val = I915_READ(reg); val |= DPLL_VCO_ENABLE; /* We do this three times for luck */ I915_WRITE(reg, val); POSTING_READ(reg); DELAY(150); /* wait for warmup */ I915_WRITE(reg, val); POSTING_READ(reg); DELAY(150); /* wait for warmup */ I915_WRITE(reg, val); POSTING_READ(reg); DELAY(150); /* wait for warmup */ } /** * intel_disable_pll - disable a PLL * @dev_priv: i915 private structure * @pipe: pipe PLL to disable * * Disable the PLL for @pipe, making sure the pipe is off first. * * Note! This is for pre-ILK only. */ static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; /* Don't disable pipe A or pipe A PLLs if needed */ if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE)) return; /* Make sure the pipe isn't still relying on us */ assert_pipe_disabled(dev_priv, pipe); reg = DPLL(pipe); val = I915_READ(reg); val &= ~DPLL_VCO_ENABLE; I915_WRITE(reg, val); POSTING_READ(reg); } /* SBI access */ static void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value) { mtx_lock(&dev_priv->dpio_lock); if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_READY) == 0, 100)) { DRM_ERROR("timeout waiting for SBI to become ready\n"); goto out_unlock; } I915_WRITE(SBI_ADDR, (reg << 16)); I915_WRITE(SBI_DATA, value); I915_WRITE(SBI_CTL_STAT, SBI_BUSY | SBI_CTL_OP_CRWR); if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_READY | SBI_RESPONSE_SUCCESS)) == 0, 100)) { DRM_ERROR("timeout waiting for SBI to complete write transaction\n"); goto out_unlock; } out_unlock: mtx_unlock(&dev_priv->dpio_lock); } static u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg) { u32 value; value = 0; mtx_lock(&dev_priv->dpio_lock); if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_READY) == 0, 100)) { DRM_ERROR("timeout waiting for SBI to become ready\n"); goto out_unlock; } I915_WRITE(SBI_ADDR, (reg << 16)); I915_WRITE(SBI_CTL_STAT, SBI_BUSY | SBI_CTL_OP_CRRD); if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_READY | SBI_RESPONSE_SUCCESS)) == 0, 100)) { DRM_ERROR("timeout waiting for SBI to complete read transaction\n"); goto out_unlock; } value = I915_READ(SBI_DATA); out_unlock: mtx_unlock(&dev_priv->dpio_lock); return value; } /** * intel_enable_pch_pll - enable PCH PLL * @dev_priv: i915 private structure * @pipe: pipe PLL to enable * * The PCH PLL needs to be enabled before the PCH transcoder, since it * drives the transcoder clock. */ static void intel_enable_pch_pll(struct intel_crtc *intel_crtc) { struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private; struct intel_pch_pll *pll; int reg; u32 val; /* PCH PLLs only available on ILK, SNB and IVB */ KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen")); pll = intel_crtc->pch_pll; if (pll == NULL) return; if (pll->refcount == 0) { DRM_DEBUG_KMS("pll->refcount == 0\n"); return; } DRM_DEBUG_KMS("enable PCH PLL %x (active %d, on? %d)for crtc %d\n", pll->pll_reg, pll->active, pll->on, intel_crtc->base.base.id); /* PCH refclock must be enabled first */ assert_pch_refclk_enabled(dev_priv); if (pll->active++ && pll->on) { assert_pch_pll_enabled(dev_priv, intel_crtc); return; } DRM_DEBUG_KMS("enabling PCH PLL %x\n", pll->pll_reg); reg = pll->pll_reg; val = I915_READ(reg); val |= DPLL_VCO_ENABLE; I915_WRITE(reg, val); POSTING_READ(reg); DELAY(200); pll->on = true; } static void intel_disable_pch_pll(struct intel_crtc *intel_crtc) { struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private; struct intel_pch_pll *pll = intel_crtc->pch_pll; int reg; u32 val; /* PCH only available on ILK+ */ KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen")); if (pll == NULL) return; if (pll->refcount == 0) { DRM_DEBUG_KMS("pll->refcount == 0\n"); return; } DRM_DEBUG_KMS("disable PCH PLL %x (active %d, on? %d) for crtc %d\n", pll->pll_reg, pll->active, pll->on, intel_crtc->base.base.id); if (pll->active == 0) { DRM_DEBUG_KMS("pll->active == 0\n"); assert_pch_pll_disabled(dev_priv, intel_crtc); return; } if (--pll->active) { assert_pch_pll_enabled(dev_priv, intel_crtc); return; } DRM_DEBUG_KMS("disabling PCH PLL %x\n", pll->pll_reg); /* Make sure transcoder isn't still depending on us */ assert_transcoder_disabled(dev_priv, intel_crtc->pipe); reg = pll->pll_reg; val = I915_READ(reg); val &= ~DPLL_VCO_ENABLE; I915_WRITE(reg, val); POSTING_READ(reg); DELAY(200); pll->on = false; } static void intel_enable_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val, pipeconf_val; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; /* PCH only available on ILK+ */ KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen")); /* Make sure PCH DPLL is enabled */ assert_pch_pll_enabled(dev_priv, to_intel_crtc(crtc)); /* FDI must be feeding us bits for PCH ports */ assert_fdi_tx_enabled(dev_priv, pipe); assert_fdi_rx_enabled(dev_priv, pipe); if (IS_HASWELL(dev_priv->dev) && pipe > 0) { DRM_ERROR("Attempting to enable transcoder on Haswell with pipe > 0\n"); return; } reg = TRANSCONF(pipe); val = I915_READ(reg); pipeconf_val = I915_READ(PIPECONF(pipe)); if (HAS_PCH_IBX(dev_priv->dev)) { /* * make the BPC in transcoder be consistent with * that in pipeconf reg. */ val &= ~PIPE_BPC_MASK; val |= pipeconf_val & PIPE_BPC_MASK; } val &= ~TRANS_INTERLACE_MASK; if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) if (HAS_PCH_IBX(dev_priv->dev) && intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO)) val |= TRANS_LEGACY_INTERLACED_ILK; else val |= TRANS_INTERLACED; else val |= TRANS_PROGRESSIVE; I915_WRITE(reg, val | TRANS_ENABLE); if (_intel_wait_for(dev_priv->dev, I915_READ(reg) & TRANS_STATE_ENABLE, 100, 1, "915trc")) DRM_ERROR("failed to enable transcoder %d\n", pipe); } static void intel_disable_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; /* FDI relies on the transcoder */ assert_fdi_tx_disabled(dev_priv, pipe); assert_fdi_rx_disabled(dev_priv, pipe); /* Ports must be off as well */ assert_pch_ports_disabled(dev_priv, pipe); reg = TRANSCONF(pipe); val = I915_READ(reg); val &= ~TRANS_ENABLE; I915_WRITE(reg, val); /* wait for PCH transcoder off, transcoder state */ if (_intel_wait_for(dev_priv->dev, (I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50, 1, "915trd")) DRM_ERROR("failed to disable transcoder %d\n", pipe); } /** * intel_enable_pipe - enable a pipe, asserting requirements * @dev_priv: i915 private structure * @pipe: pipe to enable * @pch_port: on ILK+, is this pipe driving a PCH port or not * * Enable @pipe, making sure that various hardware specific requirements * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc. * * @pipe should be %PIPE_A or %PIPE_B. * * Will wait until the pipe is actually running (i.e. first vblank) before * returning. */ static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, bool pch_port) { int reg; u32 val; /* * A pipe without a PLL won't actually be able to drive bits from * a plane. On ILK+ the pipe PLLs are integrated, so we don't * need the check. */ if (!HAS_PCH_SPLIT(dev_priv->dev)) assert_pll_enabled(dev_priv, pipe); else { if (pch_port) { /* if driving the PCH, we need FDI enabled */ assert_fdi_rx_pll_enabled(dev_priv, pipe); assert_fdi_tx_pll_enabled(dev_priv, pipe); } /* FIXME: assert CPU port conditions for SNB+ */ } reg = PIPECONF(pipe); val = I915_READ(reg); if (val & PIPECONF_ENABLE) return; I915_WRITE(reg, val | PIPECONF_ENABLE); intel_wait_for_vblank(dev_priv->dev, pipe); } /** * intel_disable_pipe - disable a pipe, asserting requirements * @dev_priv: i915 private structure * @pipe: pipe to disable * * Disable @pipe, making sure that various hardware specific requirements * are met, if applicable, e.g. plane disabled, panel fitter off, etc. * * @pipe should be %PIPE_A or %PIPE_B. * * Will wait until the pipe has shut down before returning. */ static void intel_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) { int reg; u32 val; /* * Make sure planes won't keep trying to pump pixels to us, * or we might hang the display. */ assert_planes_disabled(dev_priv, pipe); /* Don't disable pipe A or pipe A PLLs if needed */ if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE)) return; reg = PIPECONF(pipe); val = I915_READ(reg); if ((val & PIPECONF_ENABLE) == 0) return; I915_WRITE(reg, val & ~PIPECONF_ENABLE); intel_wait_for_pipe_off(dev_priv->dev, pipe); } /* * Plane regs are double buffered, going from enabled->disabled needs a * trigger in order to latch. The display address reg provides this. */ void intel_flush_display_plane(struct drm_i915_private *dev_priv, enum plane plane) { I915_WRITE(DSPADDR(plane), I915_READ(DSPADDR(plane))); I915_WRITE(DSPSURF(plane), I915_READ(DSPSURF(plane))); } /** * intel_enable_plane - enable a display plane on a given pipe * @dev_priv: i915 private structure * @plane: plane to enable * @pipe: pipe being fed * * Enable @plane on @pipe, making sure that @pipe is running first. */ static void intel_enable_plane(struct drm_i915_private *dev_priv, enum plane plane, enum pipe pipe) { int reg; u32 val; /* If the pipe isn't enabled, we can't pump pixels and may hang */ assert_pipe_enabled(dev_priv, pipe); reg = DSPCNTR(plane); val = I915_READ(reg); if (val & DISPLAY_PLANE_ENABLE) return; I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE); intel_flush_display_plane(dev_priv, plane); intel_wait_for_vblank(dev_priv->dev, pipe); } /** * intel_disable_plane - disable a display plane * @dev_priv: i915 private structure * @plane: plane to disable * @pipe: pipe consuming the data * * Disable @plane; should be an independent operation. */ static void intel_disable_plane(struct drm_i915_private *dev_priv, enum plane plane, enum pipe pipe) { int reg; u32 val; reg = DSPCNTR(plane); val = I915_READ(reg); if ((val & DISPLAY_PLANE_ENABLE) == 0) return; I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE); intel_flush_display_plane(dev_priv, plane); intel_wait_for_vblank(dev_priv->dev, pipe); } static void disable_pch_dp(struct drm_i915_private *dev_priv, enum pipe pipe, int reg, u32 port_sel) { u32 val = I915_READ(reg); if (dp_pipe_enabled(dev_priv, pipe, port_sel, val)) { DRM_DEBUG_KMS("Disabling pch dp %x on pipe %d\n", reg, pipe); I915_WRITE(reg, val & ~DP_PORT_EN); } } static void disable_pch_hdmi(struct drm_i915_private *dev_priv, enum pipe pipe, int reg) { u32 val = I915_READ(reg); if (hdmi_pipe_enabled(dev_priv, val, pipe)) { DRM_DEBUG_KMS("Disabling pch HDMI %x on pipe %d\n", reg, pipe); I915_WRITE(reg, val & ~PORT_ENABLE); } } /* Disable any ports connected to this transcoder */ static void intel_disable_pch_ports(struct drm_i915_private *dev_priv, enum pipe pipe) { u32 reg, val; val = I915_READ(PCH_PP_CONTROL); I915_WRITE(PCH_PP_CONTROL, val | PANEL_UNLOCK_REGS); disable_pch_dp(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B); disable_pch_dp(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C); disable_pch_dp(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D); reg = PCH_ADPA; val = I915_READ(reg); if (adpa_pipe_enabled(dev_priv, val, pipe)) I915_WRITE(reg, val & ~ADPA_DAC_ENABLE); reg = PCH_LVDS; val = I915_READ(reg); if (lvds_pipe_enabled(dev_priv, val, pipe)) { DRM_DEBUG_KMS("disable lvds on pipe %d val 0x%08x\n", pipe, val); I915_WRITE(reg, val & ~LVDS_PORT_EN); POSTING_READ(reg); DELAY(100); } disable_pch_hdmi(dev_priv, pipe, HDMIB); disable_pch_hdmi(dev_priv, pipe, HDMIC); disable_pch_hdmi(dev_priv, pipe, HDMID); } int intel_pin_and_fence_fb_obj(struct drm_device *dev, struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined) { struct drm_i915_private *dev_priv = dev->dev_private; u32 alignment; int ret; alignment = 0; /* shut gcc */ switch (obj->tiling_mode) { case I915_TILING_NONE: if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) alignment = 128 * 1024; else if (INTEL_INFO(dev)->gen >= 4) alignment = 4 * 1024; else alignment = 64 * 1024; break; case I915_TILING_X: /* pin() will align the object as required by fence */ alignment = 0; break; case I915_TILING_Y: /* FIXME: Is this true? */ DRM_ERROR("Y tiled not allowed for scan out buffers\n"); return -EINVAL; default: KASSERT(0, ("Wrong tiling for fb obj")); } dev_priv->mm.interruptible = false; ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined); if (ret) goto err_interruptible; /* Install a fence for tiled scan-out. Pre-i965 always needs a * fence, whereas 965+ only requires a fence if using * framebuffer compression. For simplicity, we always install * a fence as the cost is not that onerous. */ ret = i915_gem_object_get_fence(obj); if (ret) goto err_unpin; i915_gem_object_pin_fence(obj); dev_priv->mm.interruptible = true; return 0; err_unpin: i915_gem_object_unpin_from_display_plane(obj); err_interruptible: dev_priv->mm.interruptible = true; return ret; } void intel_unpin_fb_obj(struct drm_i915_gem_object *obj) { i915_gem_object_unpin_fence(obj); i915_gem_object_unpin_from_display_plane(obj); } static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_framebuffer *intel_fb; struct drm_i915_gem_object *obj; int plane = intel_crtc->plane; unsigned long Start, Offset; u32 dspcntr; u32 reg; switch (plane) { case 0: case 1: break; default: DRM_ERROR("Can't update plane %d in SAREA\n", plane); return -EINVAL; } intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; reg = DSPCNTR(plane); dspcntr = I915_READ(reg); /* Mask out pixel format bits in case we change it */ dspcntr &= ~DISPPLANE_PIXFORMAT_MASK; switch (fb->bits_per_pixel) { case 8: dspcntr |= DISPPLANE_8BPP; break; case 16: if (fb->depth == 15) dspcntr |= DISPPLANE_15_16BPP; else dspcntr |= DISPPLANE_16BPP; break; case 24: case 32: dspcntr |= DISPPLANE_32BPP_NO_ALPHA; break; default: DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel); return -EINVAL; } if (INTEL_INFO(dev)->gen >= 4) { if (obj->tiling_mode != I915_TILING_NONE) dspcntr |= DISPPLANE_TILED; else dspcntr &= ~DISPPLANE_TILED; } I915_WRITE(reg, dspcntr); Start = obj->gtt_offset; Offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8); DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n", Start, Offset, x, y, fb->pitches[0]); I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]); if (INTEL_INFO(dev)->gen >= 4) { I915_MODIFY_DISPBASE(DSPSURF(plane), Start); I915_WRITE(DSPTILEOFF(plane), (y << 16) | x); I915_WRITE(DSPADDR(plane), Offset); } else I915_WRITE(DSPADDR(plane), Start + Offset); POSTING_READ(reg); return (0); } static int ironlake_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_framebuffer *intel_fb; struct drm_i915_gem_object *obj; int plane = intel_crtc->plane; unsigned long Start, Offset; u32 dspcntr; u32 reg; switch (plane) { case 0: case 1: case 2: break; default: DRM_ERROR("Can't update plane %d in SAREA\n", plane); return -EINVAL; } intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; reg = DSPCNTR(plane); dspcntr = I915_READ(reg); /* Mask out pixel format bits in case we change it */ dspcntr &= ~DISPPLANE_PIXFORMAT_MASK; switch (fb->bits_per_pixel) { case 8: dspcntr |= DISPPLANE_8BPP; break; case 16: if (fb->depth != 16) { DRM_ERROR("bpp 16, depth %d\n", fb->depth); return -EINVAL; } dspcntr |= DISPPLANE_16BPP; break; case 24: case 32: if (fb->depth == 24) dspcntr |= DISPPLANE_32BPP_NO_ALPHA; else if (fb->depth == 30) dspcntr |= DISPPLANE_32BPP_30BIT_NO_ALPHA; else { DRM_ERROR("bpp %d depth %d\n", fb->bits_per_pixel, fb->depth); return -EINVAL; } break; default: DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel); return -EINVAL; } if (obj->tiling_mode != I915_TILING_NONE) dspcntr |= DISPPLANE_TILED; else dspcntr &= ~DISPPLANE_TILED; /* must disable */ dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; I915_WRITE(reg, dspcntr); Start = obj->gtt_offset; Offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8); DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n", Start, Offset, x, y, fb->pitches[0]); I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]); I915_MODIFY_DISPBASE(DSPSURF(plane), Start); I915_WRITE(DSPTILEOFF(plane), (y << 16) | x); I915_WRITE(DSPADDR(plane), Offset); POSTING_READ(reg); return 0; } /* Assume fb object is pinned & idle & fenced and just update base pointers */ static int intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y, enum mode_set_atomic state) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; if (dev_priv->display.disable_fbc) dev_priv->display.disable_fbc(dev); intel_increase_pllclock(crtc); return dev_priv->display.update_plane(crtc, fb, x, y); } static int intel_finish_fb(struct drm_framebuffer *old_fb) { struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj; struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; bool was_interruptible = dev_priv->mm.interruptible; int ret; mtx_lock(&dev->event_lock); while (!atomic_load_acq_int(&dev_priv->mm.wedged) && atomic_load_acq_int(&obj->pending_flip) != 0) { msleep(&obj->pending_flip, &dev->event_lock, 0, "915flp", 0); } mtx_unlock(&dev->event_lock); /* Big Hammer, we also need to ensure that any pending * MI_WAIT_FOR_EVENT inside a user batch buffer on the * current scanout is retired before unpinning the old * framebuffer. * * This should only fail upon a hung GPU, in which case we * can safely continue. */ dev_priv->mm.interruptible = false; ret = i915_gem_object_finish_gpu(obj); dev_priv->mm.interruptible = was_interruptible; return ret; } static int intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, struct drm_framebuffer *old_fb) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; #if 0 struct drm_i915_master_private *master_priv; #endif struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int ret; /* no fb bound */ if (!crtc->fb) { DRM_ERROR("No FB bound\n"); return 0; } if(intel_crtc->plane > dev_priv->num_pipe) { DRM_ERROR("no plane for crtc: plane %d, num_pipes %d\n", intel_crtc->plane, dev_priv->num_pipe); return -EINVAL; } DRM_LOCK(dev); ret = intel_pin_and_fence_fb_obj(dev, to_intel_framebuffer(crtc->fb)->obj, NULL); if (ret != 0) { DRM_UNLOCK(dev); DRM_ERROR("pin & fence failed\n"); return ret; } if (old_fb) intel_finish_fb(old_fb); ret = dev_priv->display.update_plane(crtc, crtc->fb, x, y); if (ret) { intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj); DRM_UNLOCK(dev); DRM_ERROR("failed to update base address\n"); return ret; } if (old_fb) { intel_wait_for_vblank(dev, intel_crtc->pipe); intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj); } intel_update_fbc(dev); DRM_UNLOCK(dev); #if 0 if (!dev->primary->master) return 0; master_priv = dev->primary->master->driver_priv; if (!master_priv->sarea_priv) return 0; if (intel_crtc->pipe) { master_priv->sarea_priv->pipeB_x = x; master_priv->sarea_priv->pipeB_y = y; } else { master_priv->sarea_priv->pipeA_x = x; master_priv->sarea_priv->pipeA_y = y; } #else if (!dev_priv->sarea_priv) return 0; if (intel_crtc->pipe) { dev_priv->sarea_priv->planeB_x = x; dev_priv->sarea_priv->planeB_y = y; } else { dev_priv->sarea_priv->planeA_x = x; dev_priv->sarea_priv->planeA_y = y; } #endif return 0; } static void ironlake_set_pll_edp(struct drm_crtc *crtc, int clock) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 dpa_ctl; DRM_DEBUG_KMS("eDP PLL enable for clock %d\n", clock); dpa_ctl = I915_READ(DP_A); dpa_ctl &= ~DP_PLL_FREQ_MASK; if (clock < 200000) { u32 temp; dpa_ctl |= DP_PLL_FREQ_160MHZ; /* workaround for 160Mhz: 1) program 0x4600c bits 15:0 = 0x8124 2) program 0x46010 bit 0 = 1 3) program 0x46034 bit 24 = 1 4) program 0x64000 bit 14 = 1 */ temp = I915_READ(0x4600c); temp &= 0xffff0000; I915_WRITE(0x4600c, temp | 0x8124); temp = I915_READ(0x46010); I915_WRITE(0x46010, temp | 1); temp = I915_READ(0x46034); I915_WRITE(0x46034, temp | (1 << 24)); } else { dpa_ctl |= DP_PLL_FREQ_270MHZ; } I915_WRITE(DP_A, dpa_ctl); POSTING_READ(DP_A); DELAY(500); } static void intel_fdi_normal_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 reg, temp; /* enable normal train */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); if (IS_IVYBRIDGE(dev)) { temp &= ~FDI_LINK_TRAIN_NONE_IVB; temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE; } else { temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE; } I915_WRITE(reg, temp); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); if (HAS_PCH_CPT(dev)) { temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; temp |= FDI_LINK_TRAIN_NORMAL_CPT; } else { temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_NONE; } I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE); /* wait one idle pattern time */ POSTING_READ(reg); DELAY(1000); /* IVB wants error correction enabled */ if (IS_IVYBRIDGE(dev)) I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE | FDI_FE_ERRC_ENABLE); } static void cpt_phase_pointer_enable(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; u32 flags = I915_READ(SOUTH_CHICKEN1); flags |= FDI_PHASE_SYNC_OVR(pipe); I915_WRITE(SOUTH_CHICKEN1, flags); /* once to unlock... */ flags |= FDI_PHASE_SYNC_EN(pipe); I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to enable */ POSTING_READ(SOUTH_CHICKEN1); } /* The FDI link training functions for ILK/Ibexpeak. */ static void ironlake_fdi_link_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; u32 reg, temp, tries; /* FDI needs bits from pipe & plane first */ assert_pipe_enabled(dev_priv, pipe); assert_plane_enabled(dev_priv, plane); /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ reg = FDI_RX_IMR(pipe); temp = I915_READ(reg); temp &= ~FDI_RX_SYMBOL_LOCK; temp &= ~FDI_RX_BIT_LOCK; I915_WRITE(reg, temp); I915_READ(reg); DELAY(150); /* enable CPU FDI TX and PCH FDI RX */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~(7 << 19); temp |= (intel_crtc->fdi_lanes - 1) << 19; temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_1; I915_WRITE(reg, temp | FDI_TX_ENABLE); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_1; I915_WRITE(reg, temp | FDI_RX_ENABLE); POSTING_READ(reg); DELAY(150); /* Ironlake workaround, enable clock pointer after FDI enable*/ if (HAS_PCH_IBX(dev)) { I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR); I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR | FDI_RX_PHASE_SYNC_POINTER_EN); } reg = FDI_RX_IIR(pipe); for (tries = 0; tries < 5; tries++) { temp = I915_READ(reg); DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); if ((temp & FDI_RX_BIT_LOCK)) { DRM_DEBUG_KMS("FDI train 1 done.\n"); I915_WRITE(reg, temp | FDI_RX_BIT_LOCK); break; } } if (tries == 5) DRM_ERROR("FDI train 1 fail!\n"); /* Train 2 */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_2; I915_WRITE(reg, temp); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_2; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(150); reg = FDI_RX_IIR(pipe); for (tries = 0; tries < 5; tries++) { temp = I915_READ(reg); DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); if (temp & FDI_RX_SYMBOL_LOCK) { I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK); DRM_DEBUG_KMS("FDI train 2 done.\n"); break; } } if (tries == 5) DRM_ERROR("FDI train 2 fail!\n"); DRM_DEBUG_KMS("FDI train done\n"); } static const int snb_b_fdi_train_param[] = { FDI_LINK_TRAIN_400MV_0DB_SNB_B, FDI_LINK_TRAIN_400MV_6DB_SNB_B, FDI_LINK_TRAIN_600MV_3_5DB_SNB_B, FDI_LINK_TRAIN_800MV_0DB_SNB_B, }; /* The FDI link training functions for SNB/Cougarpoint. */ static void gen6_fdi_link_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 reg, temp, i, retry; /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ reg = FDI_RX_IMR(pipe); temp = I915_READ(reg); temp &= ~FDI_RX_SYMBOL_LOCK; temp &= ~FDI_RX_BIT_LOCK; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(150); /* enable CPU FDI TX and PCH FDI RX */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~(7 << 19); temp |= (intel_crtc->fdi_lanes - 1) << 19; temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_1; temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; /* SNB-B */ temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B; I915_WRITE(reg, temp | FDI_TX_ENABLE); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); if (HAS_PCH_CPT(dev)) { temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; temp |= FDI_LINK_TRAIN_PATTERN_1_CPT; } else { temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_1; } I915_WRITE(reg, temp | FDI_RX_ENABLE); POSTING_READ(reg); DELAY(150); if (HAS_PCH_CPT(dev)) cpt_phase_pointer_enable(dev, pipe); for (i = 0; i < 4; i++) { reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; temp |= snb_b_fdi_train_param[i]; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(500); for (retry = 0; retry < 5; retry++) { reg = FDI_RX_IIR(pipe); temp = I915_READ(reg); DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); if (temp & FDI_RX_BIT_LOCK) { I915_WRITE(reg, temp | FDI_RX_BIT_LOCK); DRM_DEBUG_KMS("FDI train 1 done.\n"); break; } DELAY(50); } if (retry < 5) break; } if (i == 4) DRM_ERROR("FDI train 1 fail!\n"); /* Train 2 */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_2; if (IS_GEN6(dev)) { temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; /* SNB-B */ temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B; } I915_WRITE(reg, temp); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); if (HAS_PCH_CPT(dev)) { temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; temp |= FDI_LINK_TRAIN_PATTERN_2_CPT; } else { temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_2; } I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(150); for (i = 0; i < 4; i++) { reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; temp |= snb_b_fdi_train_param[i]; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(500); for (retry = 0; retry < 5; retry++) { reg = FDI_RX_IIR(pipe); temp = I915_READ(reg); DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); if (temp & FDI_RX_SYMBOL_LOCK) { I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK); DRM_DEBUG_KMS("FDI train 2 done.\n"); break; } DELAY(50); } if (retry < 5) break; } if (i == 4) DRM_ERROR("FDI train 2 fail!\n"); DRM_DEBUG_KMS("FDI train done.\n"); } /* Manual link training for Ivy Bridge A0 parts */ static void ivb_manual_fdi_link_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 reg, temp, i; /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ reg = FDI_RX_IMR(pipe); temp = I915_READ(reg); temp &= ~FDI_RX_SYMBOL_LOCK; temp &= ~FDI_RX_BIT_LOCK; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(150); /* enable CPU FDI TX and PCH FDI RX */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~(7 << 19); temp |= (intel_crtc->fdi_lanes - 1) << 19; temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB); temp |= FDI_LINK_TRAIN_PATTERN_1_IVB; temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B; temp |= FDI_COMPOSITE_SYNC; I915_WRITE(reg, temp | FDI_TX_ENABLE); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_AUTO; temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; temp |= FDI_LINK_TRAIN_PATTERN_1_CPT; temp |= FDI_COMPOSITE_SYNC; I915_WRITE(reg, temp | FDI_RX_ENABLE); POSTING_READ(reg); DELAY(150); for (i = 0; i < 4; i++) { reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; temp |= snb_b_fdi_train_param[i]; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(500); reg = FDI_RX_IIR(pipe); temp = I915_READ(reg); DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); if (temp & FDI_RX_BIT_LOCK || (I915_READ(reg) & FDI_RX_BIT_LOCK)) { I915_WRITE(reg, temp | FDI_RX_BIT_LOCK); DRM_DEBUG_KMS("FDI train 1 done.\n"); break; } } if (i == 4) DRM_ERROR("FDI train 1 fail!\n"); /* Train 2 */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_NONE_IVB; temp |= FDI_LINK_TRAIN_PATTERN_2_IVB; temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B; I915_WRITE(reg, temp); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; temp |= FDI_LINK_TRAIN_PATTERN_2_CPT; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(150); for (i = 0; i < 4; i++ ) { reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; temp |= snb_b_fdi_train_param[i]; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(500); reg = FDI_RX_IIR(pipe); temp = I915_READ(reg); DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); if (temp & FDI_RX_SYMBOL_LOCK) { I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK); DRM_DEBUG_KMS("FDI train 2 done.\n"); break; } } if (i == 4) DRM_ERROR("FDI train 2 fail!\n"); DRM_DEBUG_KMS("FDI train done.\n"); } static void ironlake_fdi_pll_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 reg, temp; /* Write the TU size bits so error detection works */ I915_WRITE(FDI_RX_TUSIZE1(pipe), I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK); /* enable PCH FDI RX PLL, wait warmup plus DMI latency */ reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); temp &= ~((0x7 << 19) | (0x7 << 16)); temp |= (intel_crtc->fdi_lanes - 1) << 19; temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11; I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE); POSTING_READ(reg); DELAY(200); /* Switch from Rawclk to PCDclk */ temp = I915_READ(reg); I915_WRITE(reg, temp | FDI_PCDCLK); POSTING_READ(reg); DELAY(200); /* On Haswell, the PLL configuration for ports and pipes is handled * separately, as part of DDI setup */ if (!IS_HASWELL(dev)) { /* Enable CPU FDI TX PLL, always on for Ironlake */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); if ((temp & FDI_TX_PLL_ENABLE) == 0) { I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE); POSTING_READ(reg); DELAY(100); } } } static void cpt_phase_pointer_disable(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; u32 flags = I915_READ(SOUTH_CHICKEN1); flags &= ~(FDI_PHASE_SYNC_EN(pipe)); I915_WRITE(SOUTH_CHICKEN1, flags); /* once to disable... */ flags &= ~(FDI_PHASE_SYNC_OVR(pipe)); I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to lock */ POSTING_READ(SOUTH_CHICKEN1); } static void ironlake_fdi_disable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 reg, temp; /* disable CPU FDI tx and PCH FDI rx */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); I915_WRITE(reg, temp & ~FDI_TX_ENABLE); POSTING_READ(reg); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); temp &= ~(0x7 << 16); temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11; I915_WRITE(reg, temp & ~FDI_RX_ENABLE); POSTING_READ(reg); DELAY(100); /* Ironlake workaround, disable clock pointer after downing FDI */ if (HAS_PCH_IBX(dev)) { I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR); I915_WRITE(FDI_RX_CHICKEN(pipe), I915_READ(FDI_RX_CHICKEN(pipe) & ~FDI_RX_PHASE_SYNC_POINTER_EN)); } else if (HAS_PCH_CPT(dev)) { cpt_phase_pointer_disable(dev, pipe); } /* still set train pattern 1 */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_1; I915_WRITE(reg, temp); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); if (HAS_PCH_CPT(dev)) { temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; temp |= FDI_LINK_TRAIN_PATTERN_1_CPT; } else { temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_1; } /* BPC in FDI rx is consistent with that in PIPECONF */ temp &= ~(0x07 << 16); temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11; I915_WRITE(reg, temp); POSTING_READ(reg); DELAY(100); } static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; if (crtc->fb == NULL) return; DRM_LOCK(dev); intel_finish_fb(crtc->fb); DRM_UNLOCK(dev); } static bool intel_crtc_driving_pch(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder; /* * If there's a non-PCH eDP on this crtc, it must be DP_A, and that * must be driven by its own crtc; no sharing is possible. */ list_for_each_entry(encoder, &mode_config->encoder_list, base.head) { if (encoder->base.crtc != crtc) continue; /* On Haswell, LPT PCH handles the VGA connection via FDI, and Haswell * CPU handles all others */ if (IS_HASWELL(dev)) { /* It is still unclear how this will work on PPT, so throw up a warning */ if (!HAS_PCH_LPT(dev)) DRM_DEBUG_KMS("Haswell: PPT\n"); if (encoder->type == DRM_MODE_ENCODER_DAC) { DRM_DEBUG_KMS("Haswell detected DAC encoder, assuming is PCH\n"); return true; } else { DRM_DEBUG_KMS("Haswell detected encoder %d, assuming is CPU\n", encoder->type); return false; } } switch (encoder->type) { case INTEL_OUTPUT_EDP: if (!intel_encoder_is_pch_edp(&encoder->base)) return false; continue; } } return true; } /* Program iCLKIP clock to the desired frequency */ static void lpt_program_iclkip(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 divsel, phaseinc, auxdiv, phasedir = 0; u32 temp; /* It is necessary to ungate the pixclk gate prior to programming * the divisors, and gate it back when it is done. */ I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE); /* Disable SSCCTL */ intel_sbi_write(dev_priv, SBI_SSCCTL6, intel_sbi_read(dev_priv, SBI_SSCCTL6) | SBI_SSCCTL_DISABLE); /* 20MHz is a corner case which is out of range for the 7-bit divisor */ if (crtc->mode.clock == 20000) { auxdiv = 1; divsel = 0x41; phaseinc = 0x20; } else { /* The iCLK virtual clock root frequency is in MHz, * but the crtc->mode.clock in in KHz. To get the divisors, * it is necessary to divide one by another, so we * convert the virtual clock precision to KHz here for higher * precision. */ u32 iclk_virtual_root_freq = 172800 * 1000; u32 iclk_pi_range = 64; u32 desired_divisor, msb_divisor_value, pi_value; desired_divisor = (iclk_virtual_root_freq / crtc->mode.clock); msb_divisor_value = desired_divisor / iclk_pi_range; pi_value = desired_divisor % iclk_pi_range; auxdiv = 0; divsel = msb_divisor_value - 2; phaseinc = pi_value; } /* This should not happen with any sane values */ if ((SBI_SSCDIVINTPHASE_DIVSEL(divsel) & ~SBI_SSCDIVINTPHASE_DIVSEL_MASK)) DRM_DEBUG_KMS("DIVSEL_MASK"); if ((SBI_SSCDIVINTPHASE_DIR(phasedir) & ~SBI_SSCDIVINTPHASE_INCVAL_MASK)) DRM_DEBUG_KMS("INCVAL_MASK"); DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n", crtc->mode.clock, auxdiv, divsel, phasedir, phaseinc); /* Program SSCDIVINTPHASE6 */ temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6); temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK; temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel); temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK; temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc); temp |= SBI_SSCDIVINTPHASE_DIR(phasedir); temp |= SBI_SSCDIVINTPHASE_PROPAGATE; intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp); /* Program SSCAUXDIV */ temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6); temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1); temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv); intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp); /* Enable modulator and associated divider */ temp = intel_sbi_read(dev_priv, SBI_SSCCTL6); temp &= ~SBI_SSCCTL_DISABLE; intel_sbi_write(dev_priv, SBI_SSCCTL6, temp); /* Wait for initialization time */ DELAY(24); I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE); } /* * Enable PCH resources required for PCH ports: * - PCH PLLs * - FDI training & RX/TX * - update transcoder timings * - DP transcoding bits * - transcoder */ static void ironlake_pch_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 reg, temp; assert_transcoder_disabled(dev_priv, pipe); /* For PCH output, training FDI link */ dev_priv->display.fdi_link_train(crtc); intel_enable_pch_pll(intel_crtc); if (HAS_PCH_LPT(dev)) { DRM_DEBUG_KMS("LPT detected: programming iCLKIP\n"); lpt_program_iclkip(crtc); } else if (HAS_PCH_CPT(dev)) { u32 sel; temp = I915_READ(PCH_DPLL_SEL); switch (pipe) { default: case 0: temp |= TRANSA_DPLL_ENABLE; sel = TRANSA_DPLLB_SEL; break; case 1: temp |= TRANSB_DPLL_ENABLE; sel = TRANSB_DPLLB_SEL; break; case 2: temp |= TRANSC_DPLL_ENABLE; sel = TRANSC_DPLLB_SEL; break; } if (intel_crtc->pch_pll->pll_reg == _PCH_DPLL_B) temp |= sel; else temp &= ~sel; I915_WRITE(PCH_DPLL_SEL, temp); } /* set transcoder timing, panel must allow it */ assert_panel_unlocked(dev_priv, pipe); I915_WRITE(TRANS_HTOTAL(pipe), I915_READ(HTOTAL(pipe))); I915_WRITE(TRANS_HBLANK(pipe), I915_READ(HBLANK(pipe))); I915_WRITE(TRANS_HSYNC(pipe), I915_READ(HSYNC(pipe))); I915_WRITE(TRANS_VTOTAL(pipe), I915_READ(VTOTAL(pipe))); I915_WRITE(TRANS_VBLANK(pipe), I915_READ(VBLANK(pipe))); I915_WRITE(TRANS_VSYNC(pipe), I915_READ(VSYNC(pipe))); I915_WRITE(TRANS_VSYNCSHIFT(pipe), I915_READ(VSYNCSHIFT(pipe))); if (!IS_HASWELL(dev)) intel_fdi_normal_train(crtc); /* For PCH DP, enable TRANS_DP_CTL */ if (HAS_PCH_CPT(dev) && (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) || intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) { u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) >> 5; reg = TRANS_DP_CTL(pipe); temp = I915_READ(reg); temp &= ~(TRANS_DP_PORT_SEL_MASK | TRANS_DP_SYNC_MASK | TRANS_DP_BPC_MASK); temp |= (TRANS_DP_OUTPUT_ENABLE | TRANS_DP_ENH_FRAMING); temp |= bpc << 9; /* same format but at 11:9 */ if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC) temp |= TRANS_DP_HSYNC_ACTIVE_HIGH; if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC) temp |= TRANS_DP_VSYNC_ACTIVE_HIGH; switch (intel_trans_dp_port_sel(crtc)) { case PCH_DP_B: temp |= TRANS_DP_PORT_SEL_B; break; case PCH_DP_C: temp |= TRANS_DP_PORT_SEL_C; break; case PCH_DP_D: temp |= TRANS_DP_PORT_SEL_D; break; default: DRM_DEBUG_KMS("Wrong PCH DP port return. Guess port B\n"); temp |= TRANS_DP_PORT_SEL_B; break; } I915_WRITE(reg, temp); } intel_enable_transcoder(dev_priv, pipe); } static void intel_put_pch_pll(struct intel_crtc *intel_crtc) { struct intel_pch_pll *pll = intel_crtc->pch_pll; if (pll == NULL) return; if (pll->refcount == 0) { printf("bad PCH PLL refcount\n"); return; } --pll->refcount; intel_crtc->pch_pll = NULL; } static struct intel_pch_pll *intel_get_pch_pll(struct intel_crtc *intel_crtc, u32 dpll, u32 fp) { struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private; struct intel_pch_pll *pll; int i; pll = intel_crtc->pch_pll; if (pll) { DRM_DEBUG_KMS("CRTC:%d reusing existing PCH PLL %x\n", intel_crtc->base.base.id, pll->pll_reg); goto prepare; } if (HAS_PCH_IBX(dev_priv->dev)) { /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */ i = intel_crtc->pipe; pll = &dev_priv->pch_plls[i]; DRM_DEBUG_KMS("CRTC:%d using pre-allocated PCH PLL %x\n", intel_crtc->base.base.id, pll->pll_reg); goto found; } for (i = 0; i < dev_priv->num_pch_pll; i++) { pll = &dev_priv->pch_plls[i]; /* Only want to check enabled timings first */ if (pll->refcount == 0) continue; if (dpll == (I915_READ(pll->pll_reg) & 0x7fffffff) && fp == I915_READ(pll->fp0_reg)) { DRM_DEBUG_KMS("CRTC:%d sharing existing PCH PLL %x (refcount %d, ative %d)\n", intel_crtc->base.base.id, pll->pll_reg, pll->refcount, pll->active); goto found; } } /* Ok no matching timings, maybe there's a free one? */ for (i = 0; i < dev_priv->num_pch_pll; i++) { /* XXXKIB: HACK */ pll = &dev_priv->pch_plls[i]; if (pll->refcount == 0) { DRM_DEBUG_KMS("CRTC:%d allocated PCH PLL %x\n", intel_crtc->base.base.id, pll->pll_reg); goto found; } } return NULL; found: intel_crtc->pch_pll = pll; pll->refcount++; DRM_DEBUG_DRIVER("using pll %d for pipe %d\n", i, intel_crtc->pipe); prepare: /* separate function? */ DRM_DEBUG_DRIVER("switching PLL %x off\n", pll->pll_reg); /* Wait for the clocks to stabilize before rewriting the regs */ I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE); POSTING_READ(pll->pll_reg); DELAY(150); I915_WRITE(pll->fp0_reg, fp); I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE); pll->on = false; return pll; } void intel_cpt_verify_modeset(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; int dslreg = PIPEDSL(pipe), tc2reg = TRANS_CHICKEN2(pipe); u32 temp; temp = I915_READ(dslreg); DELAY(500); if (_intel_wait_for(dev, I915_READ(dslreg) != temp, 5, 1, "915cp1")) { /* Without this, mode sets may fail silently on FDI */ I915_WRITE(tc2reg, TRANS_AUTOTRAIN_GEN_STALL_DIS); DELAY(250); I915_WRITE(tc2reg, 0); if (_intel_wait_for(dev, I915_READ(dslreg) != temp, 5, 1, "915cp2")) DRM_ERROR("mode set failed: pipe %d stuck\n", pipe); } } static void ironlake_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; u32 temp; bool is_pch_port; if (intel_crtc->active) return; intel_crtc->active = true; intel_update_watermarks(dev); if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) { temp = I915_READ(PCH_LVDS); if ((temp & LVDS_PORT_EN) == 0) I915_WRITE(PCH_LVDS, temp | LVDS_PORT_EN); } is_pch_port = intel_crtc_driving_pch(crtc); if (is_pch_port) ironlake_fdi_pll_enable(crtc); else ironlake_fdi_disable(crtc); /* Enable panel fitting for LVDS */ if (dev_priv->pch_pf_size && (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) || HAS_eDP)) { /* Force use of hard-coded filter coefficients * as some pre-programmed values are broken, * e.g. x201. */ I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3); I915_WRITE(PF_WIN_POS(pipe), dev_priv->pch_pf_pos); I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size); } intel_enable_pipe(dev_priv, pipe, is_pch_port); intel_enable_plane(dev_priv, plane, pipe); if (is_pch_port) ironlake_pch_enable(crtc); intel_crtc_load_lut(crtc); DRM_LOCK(dev); intel_update_fbc(dev); DRM_UNLOCK(dev); intel_crtc_update_cursor(crtc, true); } static void ironlake_crtc_disable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; u32 reg, temp; if (!intel_crtc->active) return; intel_crtc_wait_for_pending_flips(crtc); drm_vblank_off(dev, pipe); intel_crtc_update_cursor(crtc, false); intel_disable_plane(dev_priv, plane, pipe); if (dev_priv->cfb_plane == plane) intel_disable_fbc(dev); intel_disable_pipe(dev_priv, pipe); /* Disable PF */ I915_WRITE(PF_CTL(pipe), 0); I915_WRITE(PF_WIN_SZ(pipe), 0); ironlake_fdi_disable(crtc); /* This is a horrible layering violation; we should be doing this in * the connector/encoder ->prepare instead, but we don't always have * enough information there about the config to know whether it will * actually be necessary or just cause undesired flicker. */ intel_disable_pch_ports(dev_priv, pipe); intel_disable_transcoder(dev_priv, pipe); if (HAS_PCH_CPT(dev)) { /* disable TRANS_DP_CTL */ reg = TRANS_DP_CTL(pipe); temp = I915_READ(reg); temp &= ~(TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK); temp |= TRANS_DP_PORT_SEL_NONE; I915_WRITE(reg, temp); /* disable DPLL_SEL */ temp = I915_READ(PCH_DPLL_SEL); switch (pipe) { case 0: temp &= ~(TRANSA_DPLL_ENABLE | TRANSA_DPLLB_SEL); break; case 1: temp &= ~(TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL); break; case 2: /* C shares PLL A or B */ temp &= ~(TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL); break; default: KASSERT(1, ("Wrong pipe %d", pipe)); /* wtf */ } I915_WRITE(PCH_DPLL_SEL, temp); } /* disable PCH DPLL */ intel_disable_pch_pll(intel_crtc); /* Switch from PCDclk to Rawclk */ reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); I915_WRITE(reg, temp & ~FDI_PCDCLK); /* Disable CPU FDI TX PLL */ reg = FDI_TX_CTL(pipe); temp = I915_READ(reg); I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE); POSTING_READ(reg); DELAY(100); reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE); /* Wait for the clocks to turn off. */ POSTING_READ(reg); DELAY(100); intel_crtc->active = false; intel_update_watermarks(dev); DRM_LOCK(dev); intel_update_fbc(dev); DRM_UNLOCK(dev); } static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; /* XXX: When our outputs are all unaware of DPMS modes other than off * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC. */ switch (mode) { case DRM_MODE_DPMS_ON: case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: DRM_DEBUG_KMS("crtc %d/%d dpms on\n", pipe, plane); ironlake_crtc_enable(crtc); break; case DRM_MODE_DPMS_OFF: DRM_DEBUG_KMS("crtc %d/%d dpms off\n", pipe, plane); ironlake_crtc_disable(crtc); break; } } static void ironlake_crtc_off(struct drm_crtc *crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); intel_put_pch_pll(intel_crtc); } static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable) { if (!enable && intel_crtc->overlay) { struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; DRM_LOCK(dev); dev_priv->mm.interruptible = false; (void) intel_overlay_switch_off(intel_crtc->overlay); dev_priv->mm.interruptible = true; DRM_UNLOCK(dev); } /* Let userspace switch the overlay on again. In most cases userspace * has to recompute where to put it anyway. */ } static void i9xx_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; if (intel_crtc->active) return; intel_crtc->active = true; intel_update_watermarks(dev); intel_enable_pll(dev_priv, pipe); intel_enable_pipe(dev_priv, pipe, false); intel_enable_plane(dev_priv, plane, pipe); intel_crtc_load_lut(crtc); intel_update_fbc(dev); /* Give the overlay scaler a chance to enable if it's on this pipe */ intel_crtc_dpms_overlay(intel_crtc, true); intel_crtc_update_cursor(crtc, true); } static void i9xx_crtc_disable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; if (!intel_crtc->active) return; /* Give the overlay scaler a chance to disable if it's on this pipe */ intel_crtc_wait_for_pending_flips(crtc); drm_vblank_off(dev, pipe); intel_crtc_dpms_overlay(intel_crtc, false); intel_crtc_update_cursor(crtc, false); if (dev_priv->cfb_plane == plane) intel_disable_fbc(dev); intel_disable_plane(dev_priv, plane, pipe); intel_disable_pipe(dev_priv, pipe); intel_disable_pll(dev_priv, pipe); intel_crtc->active = false; intel_update_fbc(dev); intel_update_watermarks(dev); } static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode) { /* XXX: When our outputs are all unaware of DPMS modes other than off * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC. */ switch (mode) { case DRM_MODE_DPMS_ON: case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: i9xx_crtc_enable(crtc); break; case DRM_MODE_DPMS_OFF: i9xx_crtc_disable(crtc); break; } } static void i9xx_crtc_off(struct drm_crtc *crtc) { } /** * Sets the power management mode of the pipe and plane. */ static void intel_crtc_dpms(struct drm_crtc *crtc, int mode) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; #if 0 struct drm_i915_master_private *master_priv; #endif struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; bool enabled; if (intel_crtc->dpms_mode == mode) return; intel_crtc->dpms_mode = mode; dev_priv->display.dpms(crtc, mode); #if 0 if (!dev->primary->master) return; master_priv = dev->primary->master->driver_priv; if (!master_priv->sarea_priv) return; #else if (!dev_priv->sarea_priv) return; #endif enabled = crtc->enabled && mode != DRM_MODE_DPMS_OFF; switch (pipe) { case 0: #if 0 master_priv->sarea_priv->pipeA_w = enabled ? crtc->mode.hdisplay : 0; master_priv->sarea_priv->pipeA_h = enabled ? crtc->mode.vdisplay : 0; #else dev_priv->sarea_priv->planeA_w = enabled ? crtc->mode.hdisplay : 0; dev_priv->sarea_priv->planeA_h = enabled ? crtc->mode.vdisplay : 0; #endif break; case 1: #if 0 master_priv->sarea_priv->pipeB_w = enabled ? crtc->mode.hdisplay : 0; master_priv->sarea_priv->pipeB_h = enabled ? crtc->mode.vdisplay : 0; #else dev_priv->sarea_priv->planeB_w = enabled ? crtc->mode.hdisplay : 0; dev_priv->sarea_priv->planeB_h = enabled ? crtc->mode.vdisplay : 0; #endif break; default: DRM_ERROR("Can't update pipe %c in SAREA\n", pipe_name(pipe)); break; } } static void intel_crtc_disable(struct drm_crtc *crtc) { struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private; struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF); dev_priv->display.off(crtc); assert_plane_disabled(dev->dev_private, to_intel_crtc(crtc)->plane); assert_pipe_disabled(dev->dev_private, to_intel_crtc(crtc)->pipe); if (crtc->fb) { DRM_LOCK(dev); intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj); DRM_UNLOCK(dev); } } /* Prepare for a mode set. * * Note we could be a lot smarter here. We need to figure out which outputs * will be enabled, which disabled (in short, how the config will changes) * and perform the minimum necessary steps to accomplish that, e.g. updating * watermarks, FBC configuration, making sure PLLs are programmed correctly, * panel fitting is in the proper state, etc. */ static void i9xx_crtc_prepare(struct drm_crtc *crtc) { i9xx_crtc_disable(crtc); } static void i9xx_crtc_commit(struct drm_crtc *crtc) { i9xx_crtc_enable(crtc); } static void ironlake_crtc_prepare(struct drm_crtc *crtc) { ironlake_crtc_disable(crtc); } static void ironlake_crtc_commit(struct drm_crtc *crtc) { ironlake_crtc_enable(crtc); } void intel_encoder_prepare(struct drm_encoder *encoder) { struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private; /* lvds has its own version of prepare see intel_lvds_prepare */ encoder_funcs->dpms(encoder, DRM_MODE_DPMS_OFF); } void intel_encoder_commit(struct drm_encoder *encoder) { struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private; struct drm_device *dev = encoder->dev; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); /* lvds has its own version of commit see intel_lvds_commit */ encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON); if (HAS_PCH_CPT(dev)) intel_cpt_verify_modeset(dev, intel_crtc->pipe); } void intel_encoder_destroy(struct drm_encoder *encoder) { struct intel_encoder *intel_encoder = to_intel_encoder(encoder); drm_encoder_cleanup(encoder); free(intel_encoder, DRM_MEM_KMS); } static bool intel_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode) { struct drm_device *dev = crtc->dev; if (HAS_PCH_SPLIT(dev)) { /* FDI link clock is fixed at 2.7G */ if (mode->clock * 3 > IRONLAKE_FDI_FREQ * 4) return false; } /* All interlaced capable intel hw wants timings in frames. Note though * that intel_lvds_mode_fixup does some funny tricks with the crtc * timings, so we need to be careful not to clobber these.*/ if (!(adjusted_mode->private_flags & INTEL_MODE_CRTC_TIMINGS_SET)) drm_mode_set_crtcinfo(adjusted_mode, 0); return true; } static int valleyview_get_display_clock_speed(struct drm_device *dev) { return 400000; /* FIXME */ } static int i945_get_display_clock_speed(struct drm_device *dev) { return 400000; } static int i915_get_display_clock_speed(struct drm_device *dev) { return 333000; } static int i9xx_misc_get_display_clock_speed(struct drm_device *dev) { return 200000; } static int i915gm_get_display_clock_speed(struct drm_device *dev) { u16 gcfgc = 0; gcfgc = pci_read_config(dev->device, GCFGC, 2); if (gcfgc & GC_LOW_FREQUENCY_ENABLE) return 133000; else { switch (gcfgc & GC_DISPLAY_CLOCK_MASK) { case GC_DISPLAY_CLOCK_333_MHZ: return 333000; default: case GC_DISPLAY_CLOCK_190_200_MHZ: return 190000; } } } static int i865_get_display_clock_speed(struct drm_device *dev) { return 266000; } static int i855_get_display_clock_speed(struct drm_device *dev) { u16 hpllcc = 0; /* Assume that the hardware is in the high speed state. This * should be the default. */ switch (hpllcc & GC_CLOCK_CONTROL_MASK) { case GC_CLOCK_133_200: case GC_CLOCK_100_200: return 200000; case GC_CLOCK_166_250: return 250000; case GC_CLOCK_100_133: return 133000; } /* Shouldn't happen */ return 0; } static int i830_get_display_clock_speed(struct drm_device *dev) { return 133000; } struct fdi_m_n { u32 tu; u32 gmch_m; u32 gmch_n; u32 link_m; u32 link_n; }; static void fdi_reduce_ratio(u32 *num, u32 *den) { while (*num > 0xffffff || *den > 0xffffff) { *num >>= 1; *den >>= 1; } } static void ironlake_compute_m_n(int bits_per_pixel, int nlanes, int pixel_clock, int link_clock, struct fdi_m_n *m_n) { m_n->tu = 64; /* default size */ /* BUG_ON(pixel_clock > INT_MAX / 36); */ m_n->gmch_m = bits_per_pixel * pixel_clock; m_n->gmch_n = link_clock * nlanes * 8; fdi_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n); m_n->link_m = pixel_clock; m_n->link_n = link_clock; fdi_reduce_ratio(&m_n->link_m, &m_n->link_n); } static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) { if (i915_panel_use_ssc >= 0) return i915_panel_use_ssc != 0; return dev_priv->lvds_use_ssc && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE); } /** * intel_choose_pipe_bpp_dither - figure out what color depth the pipe should send * @crtc: CRTC structure * @mode: requested mode * * A pipe may be connected to one or more outputs. Based on the depth of the * attached framebuffer, choose a good color depth to use on the pipe. * * If possible, match the pipe depth to the fb depth. In some cases, this * isn't ideal, because the connected output supports a lesser or restricted * set of depths. Resolve that here: * LVDS typically supports only 6bpc, so clamp down in that case * HDMI supports only 8bpc or 12bpc, so clamp to 8bpc with dither for 10bpc * Displays may support a restricted set as well, check EDID and clamp as * appropriate. * DP may want to dither down to 6bpc to fit larger modes * * RETURNS: * Dithering requirement (i.e. false if display bpc and pipe bpc match, * true if they don't match). */ static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc, unsigned int *pipe_bpp, struct drm_display_mode *mode) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_encoder *encoder; struct drm_connector *connector; unsigned int display_bpc = UINT_MAX, bpc; /* Walk the encoders & connectors on this crtc, get min bpc */ list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { struct intel_encoder *intel_encoder = to_intel_encoder(encoder); if (encoder->crtc != crtc) continue; if (intel_encoder->type == INTEL_OUTPUT_LVDS) { unsigned int lvds_bpc; if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) == LVDS_A3_POWER_UP) lvds_bpc = 8; else lvds_bpc = 6; if (lvds_bpc < display_bpc) { DRM_DEBUG_KMS("clamping display bpc (was %d) to LVDS (%d)\n", display_bpc, lvds_bpc); display_bpc = lvds_bpc; } continue; } if (intel_encoder->type == INTEL_OUTPUT_EDP) { /* Use VBT settings if we have an eDP panel */ unsigned int edp_bpc = dev_priv->edp.bpp / 3; if (edp_bpc < display_bpc) { DRM_DEBUG_KMS("clamping display bpc (was %d) to eDP (%d)\n", display_bpc, edp_bpc); display_bpc = edp_bpc; } continue; } /* Not one of the known troublemakers, check the EDID */ list_for_each_entry(connector, &dev->mode_config.connector_list, head) { if (connector->encoder != encoder) continue; /* Don't use an invalid EDID bpc value */ if (connector->display_info.bpc && connector->display_info.bpc < display_bpc) { DRM_DEBUG_KMS("clamping display bpc (was %d) to EDID reported max of %d\n", display_bpc, connector->display_info.bpc); display_bpc = connector->display_info.bpc; } } /* * HDMI is either 12 or 8, so if the display lets 10bpc sneak * through, clamp it down. (Note: >12bpc will be caught below.) */ if (intel_encoder->type == INTEL_OUTPUT_HDMI) { if (display_bpc > 8 && display_bpc < 12) { DRM_DEBUG_KMS("forcing bpc to 12 for HDMI\n"); display_bpc = 12; } else { DRM_DEBUG_KMS("forcing bpc to 8 for HDMI\n"); display_bpc = 8; } } } if (mode->private_flags & INTEL_MODE_DP_FORCE_6BPC) { DRM_DEBUG_KMS("Dithering DP to 6bpc\n"); display_bpc = 6; } /* * We could just drive the pipe at the highest bpc all the time and * enable dithering as needed, but that costs bandwidth. So choose * the minimum value that expresses the full color range of the fb but * also stays within the max display bpc discovered above. */ switch (crtc->fb->depth) { case 8: bpc = 8; /* since we go through a colormap */ break; case 15: case 16: bpc = 6; /* min is 18bpp */ break; case 24: bpc = 8; break; case 30: bpc = 10; break; case 48: bpc = 12; break; default: DRM_DEBUG("unsupported depth, assuming 24 bits\n"); bpc = min((unsigned int)8, display_bpc); break; } display_bpc = min(display_bpc, bpc); DRM_DEBUG_KMS("setting pipe bpc to %d (max display bpc %d)\n", bpc, display_bpc); *pipe_bpp = display_bpc * 3; return display_bpc != bpc; } static int i9xx_get_refclk(struct drm_crtc *crtc, int num_connectors) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; int refclk; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) && intel_panel_use_ssc(dev_priv) && num_connectors < 2) { refclk = dev_priv->lvds_ssc_freq * 1000; DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n", refclk / 1000); } else if (!IS_GEN2(dev)) { refclk = 96000; } else { refclk = 48000; } return refclk; } static void i9xx_adjust_sdvo_tv_clock(struct drm_display_mode *adjusted_mode, intel_clock_t *clock) { /* SDVO TV has fixed PLL values depend on its clock range, this mirrors vbios setting. */ if (adjusted_mode->clock >= 100000 && adjusted_mode->clock < 140500) { clock->p1 = 2; clock->p2 = 10; clock->n = 3; clock->m1 = 16; clock->m2 = 8; } else if (adjusted_mode->clock >= 140500 && adjusted_mode->clock <= 200000) { clock->p1 = 1; clock->p2 = 10; clock->n = 6; clock->m1 = 12; clock->m2 = 8; } } static void i9xx_update_pll_dividers(struct drm_crtc *crtc, intel_clock_t *clock, intel_clock_t *reduced_clock) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 fp, fp2 = 0; if (IS_PINEVIEW(dev)) { fp = (1 << clock->n) << 16 | clock->m1 << 8 | clock->m2; if (reduced_clock) fp2 = (1 << reduced_clock->n) << 16 | reduced_clock->m1 << 8 | reduced_clock->m2; } else { fp = clock->n << 16 | clock->m1 << 8 | clock->m2; if (reduced_clock) fp2 = reduced_clock->n << 16 | reduced_clock->m1 << 8 | reduced_clock->m2; } I915_WRITE(FP0(pipe), fp); intel_crtc->lowfreq_avail = false; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) && reduced_clock && i915_powersave) { I915_WRITE(FP1(pipe), fp2); intel_crtc->lowfreq_avail = true; } else { I915_WRITE(FP1(pipe), fp); } } static void intel_update_lvds(struct drm_crtc *crtc, intel_clock_t *clock, struct drm_display_mode *adjusted_mode) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 temp; temp = I915_READ(LVDS); temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP; if (pipe == 1) { temp |= LVDS_PIPEB_SELECT; } else { temp &= ~LVDS_PIPEB_SELECT; } /* set the corresponsding LVDS_BORDER bit */ temp |= dev_priv->lvds_border_bits; /* Set the B0-B3 data pairs corresponding to whether we're going to * set the DPLLs for dual-channel mode or not. */ if (clock->p2 == 7) temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP; else temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP); /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP) * appropriately here, but we need to look more thoroughly into how * panels behave in the two modes. */ /* set the dithering flag on LVDS as needed */ if (INTEL_INFO(dev)->gen >= 4) { if (dev_priv->lvds_dither) temp |= LVDS_ENABLE_DITHER; else temp &= ~LVDS_ENABLE_DITHER; } temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY); if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC) temp |= LVDS_HSYNC_POLARITY; if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC) temp |= LVDS_VSYNC_POLARITY; I915_WRITE(LVDS, temp); } static void i9xx_update_pll(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode, intel_clock_t *clock, intel_clock_t *reduced_clock, int num_connectors) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 dpll; bool is_sdvo; is_sdvo = intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO) || intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI); dpll = DPLL_VGA_MODE_DIS; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) dpll |= DPLLB_MODE_LVDS; else dpll |= DPLLB_MODE_DAC_SERIAL; if (is_sdvo) { int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode); if (pixel_multiplier > 1) { if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) dpll |= (pixel_multiplier - 1) << SDVO_MULTIPLIER_SHIFT_HIRES; } dpll |= DPLL_DVO_HIGH_SPEED; } if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) dpll |= DPLL_DVO_HIGH_SPEED; /* compute bitmask from p1 value */ if (IS_PINEVIEW(dev)) dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW; else { dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; if (IS_G4X(dev) && reduced_clock) dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; } switch (clock->p2) { case 5: dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5; break; case 7: dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7; break; case 10: dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10; break; case 14: dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; break; } if (INTEL_INFO(dev)->gen >= 4) dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT); if (is_sdvo && intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT)) dpll |= PLL_REF_INPUT_TVCLKINBC; else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT)) /* XXX: just matching BIOS for now */ /* dpll |= PLL_REF_INPUT_TVCLKINBC; */ dpll |= 3; else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) && intel_panel_use_ssc(dev_priv) && num_connectors < 2) dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; else dpll |= PLL_REF_INPUT_DREFCLK; dpll |= DPLL_VCO_ENABLE; I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE); POSTING_READ(DPLL(pipe)); DELAY(150); /* The LVDS pin pair needs to be on before the DPLLs are enabled. * This is an exception to the general rule that mode_set doesn't turn * things on. */ if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) intel_update_lvds(crtc, clock, adjusted_mode); if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) intel_dp_set_m_n(crtc, mode, adjusted_mode); I915_WRITE(DPLL(pipe), dpll); /* Wait for the clocks to stabilize. */ POSTING_READ(DPLL(pipe)); DELAY(150); if (INTEL_INFO(dev)->gen >= 4) { u32 temp = 0; if (is_sdvo) { temp = intel_mode_get_pixel_multiplier(adjusted_mode); if (temp > 1) temp = (temp - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; else temp = 0; } I915_WRITE(DPLL_MD(pipe), temp); } else { /* The pixel multiplier can only be updated once the * DPLL is enabled and the clocks are stable. * * So write it again. */ I915_WRITE(DPLL(pipe), dpll); } } static void i8xx_update_pll(struct drm_crtc *crtc, struct drm_display_mode *adjusted_mode, intel_clock_t *clock, int num_connectors) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 dpll; dpll = DPLL_VGA_MODE_DIS; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) { dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; } else { if (clock->p1 == 2) dpll |= PLL_P1_DIVIDE_BY_TWO; else dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT; if (clock->p2 == 4) dpll |= PLL_P2_DIVIDE_BY_4; } if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT)) /* XXX: just matching BIOS for now */ /* dpll |= PLL_REF_INPUT_TVCLKINBC; */ dpll |= 3; else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) && intel_panel_use_ssc(dev_priv) && num_connectors < 2) dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; else dpll |= PLL_REF_INPUT_DREFCLK; dpll |= DPLL_VCO_ENABLE; I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE); POSTING_READ(DPLL(pipe)); DELAY(150); I915_WRITE(DPLL(pipe), dpll); /* Wait for the clocks to stabilize. */ POSTING_READ(DPLL(pipe)); DELAY(150); /* The LVDS pin pair needs to be on before the DPLLs are enabled. * This is an exception to the general rule that mode_set doesn't turn * things on. */ if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) intel_update_lvds(crtc, clock, adjusted_mode); /* The pixel multiplier can only be updated once the * DPLL is enabled and the clocks are stable. * * So write it again. */ I915_WRITE(DPLL(pipe), dpll); } static int i9xx_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode, int x, int y, struct drm_framebuffer *old_fb) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; int refclk, num_connectors = 0; intel_clock_t clock, reduced_clock; u32 dspcntr, pipeconf, vsyncshift; bool ok, has_reduced_clock = false, is_sdvo = false; bool is_lvds = false, is_tv = false, is_dp = false; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder; const intel_limit_t *limit; int ret; list_for_each_entry(encoder, &mode_config->encoder_list, base.head) { if (encoder->base.crtc != crtc) continue; switch (encoder->type) { case INTEL_OUTPUT_LVDS: is_lvds = true; break; case INTEL_OUTPUT_SDVO: case INTEL_OUTPUT_HDMI: is_sdvo = true; if (encoder->needs_tv_clock) is_tv = true; break; case INTEL_OUTPUT_TVOUT: is_tv = true; break; case INTEL_OUTPUT_DISPLAYPORT: is_dp = true; break; } num_connectors++; } refclk = i9xx_get_refclk(crtc, num_connectors); /* * Returns a set of divisors for the desired target clock with the given * refclk, or false. The returned values represent the clock equation: * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. */ limit = intel_limit(crtc, refclk); ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL, &clock); if (!ok) { DRM_ERROR("Couldn't find PLL settings for mode!\n"); return -EINVAL; } /* Ensure that the cursor is valid for the new mode before changing... */ intel_crtc_update_cursor(crtc, true); if (is_lvds && dev_priv->lvds_downclock_avail) { /* * Ensure we match the reduced clock's P to the target clock. * If the clocks don't match, we can't switch the display clock * by using the FP0/FP1. In such case we will disable the LVDS * downclock feature. */ has_reduced_clock = limit->find_pll(limit, crtc, dev_priv->lvds_downclock, refclk, &clock, &reduced_clock); } if (is_sdvo && is_tv) i9xx_adjust_sdvo_tv_clock(adjusted_mode, &clock); i9xx_update_pll_dividers(crtc, &clock, has_reduced_clock ? &reduced_clock : NULL); if (IS_GEN2(dev)) i8xx_update_pll(crtc, adjusted_mode, &clock, num_connectors); else i9xx_update_pll(crtc, mode, adjusted_mode, &clock, has_reduced_clock ? &reduced_clock : NULL, num_connectors); /* setup pipeconf */ pipeconf = I915_READ(PIPECONF(pipe)); /* Set up the display plane register */ dspcntr = DISPPLANE_GAMMA_ENABLE; if (pipe == 0) dspcntr &= ~DISPPLANE_SEL_PIPE_MASK; else dspcntr |= DISPPLANE_SEL_PIPE_B; if (pipe == 0 && INTEL_INFO(dev)->gen < 4) { /* Enable pixel doubling when the dot clock is > 90% of the (display) * core speed. * * XXX: No double-wide on 915GM pipe B. Is that the only reason for the * pipe == 0 check? */ if (mode->clock > dev_priv->display.get_display_clock_speed(dev) * 9 / 10) pipeconf |= PIPECONF_DOUBLE_WIDE; else pipeconf &= ~PIPECONF_DOUBLE_WIDE; } /* default to 8bpc */ pipeconf &= ~(PIPECONF_BPP_MASK | PIPECONF_DITHER_EN); if (is_dp) { if (mode->private_flags & INTEL_MODE_DP_FORCE_6BPC) { pipeconf |= PIPECONF_BPP_6 | PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP; } } DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B'); drm_mode_debug_printmodeline(mode); if (HAS_PIPE_CXSR(dev)) { if (intel_crtc->lowfreq_avail) { DRM_DEBUG_KMS("enabling CxSR downclocking\n"); pipeconf |= PIPECONF_CXSR_DOWNCLOCK; } else { DRM_DEBUG_KMS("disabling CxSR downclocking\n"); pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK; } } pipeconf &= ~PIPECONF_INTERLACE_MASK; if (!IS_GEN2(dev) && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; /* the chip adds 2 halflines automatically */ adjusted_mode->crtc_vtotal -= 1; adjusted_mode->crtc_vblank_end -= 1; vsyncshift = adjusted_mode->crtc_hsync_start - adjusted_mode->crtc_htotal/2; } else { pipeconf |= PIPECONF_PROGRESSIVE; vsyncshift = 0; } if (!IS_GEN3(dev)) I915_WRITE(VSYNCSHIFT(pipe), vsyncshift); I915_WRITE(HTOTAL(pipe), (adjusted_mode->crtc_hdisplay - 1) | ((adjusted_mode->crtc_htotal - 1) << 16)); I915_WRITE(HBLANK(pipe), (adjusted_mode->crtc_hblank_start - 1) | ((adjusted_mode->crtc_hblank_end - 1) << 16)); I915_WRITE(HSYNC(pipe), (adjusted_mode->crtc_hsync_start - 1) | ((adjusted_mode->crtc_hsync_end - 1) << 16)); I915_WRITE(VTOTAL(pipe), (adjusted_mode->crtc_vdisplay - 1) | ((adjusted_mode->crtc_vtotal - 1) << 16)); I915_WRITE(VBLANK(pipe), (adjusted_mode->crtc_vblank_start - 1) | ((adjusted_mode->crtc_vblank_end - 1) << 16)); I915_WRITE(VSYNC(pipe), (adjusted_mode->crtc_vsync_start - 1) | ((adjusted_mode->crtc_vsync_end - 1) << 16)); /* pipesrc and dspsize control the size that is scaled from, * which should always be the user's requested size. */ I915_WRITE(DSPSIZE(plane), ((mode->vdisplay - 1) << 16) | (mode->hdisplay - 1)); I915_WRITE(DSPPOS(plane), 0); I915_WRITE(PIPESRC(pipe), ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1)); I915_WRITE(PIPECONF(pipe), pipeconf); POSTING_READ(PIPECONF(pipe)); intel_enable_pipe(dev_priv, pipe, false); intel_wait_for_vblank(dev, pipe); I915_WRITE(DSPCNTR(plane), dspcntr); POSTING_READ(DSPCNTR(plane)); ret = intel_pipe_set_base(crtc, x, y, old_fb); intel_update_watermarks(dev); return ret; } /* * Initialize reference clocks when the driver loads */ void ironlake_init_pch_refclk(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder; u32 temp; bool has_lvds = false; bool has_cpu_edp = false; bool has_pch_edp = false; bool has_panel = false; bool has_ck505 = false; bool can_ssc = false; /* We need to take the global config into account */ list_for_each_entry(encoder, &mode_config->encoder_list, base.head) { switch (encoder->type) { case INTEL_OUTPUT_LVDS: has_panel = true; has_lvds = true; break; case INTEL_OUTPUT_EDP: has_panel = true; if (intel_encoder_is_pch_edp(&encoder->base)) has_pch_edp = true; else has_cpu_edp = true; break; } } if (HAS_PCH_IBX(dev)) { has_ck505 = dev_priv->display_clock_mode; can_ssc = has_ck505; } else { has_ck505 = false; can_ssc = true; } DRM_DEBUG_KMS("has_panel %d has_lvds %d has_pch_edp %d has_cpu_edp %d has_ck505 %d\n", has_panel, has_lvds, has_pch_edp, has_cpu_edp, has_ck505); /* Ironlake: try to setup display ref clock before DPLL * enabling. This is only under driver's control after * PCH B stepping, previous chipset stepping should be * ignoring this setting. */ temp = I915_READ(PCH_DREF_CONTROL); /* Always enable nonspread source */ temp &= ~DREF_NONSPREAD_SOURCE_MASK; if (has_ck505) temp |= DREF_NONSPREAD_CK505_ENABLE; else temp |= DREF_NONSPREAD_SOURCE_ENABLE; if (has_panel) { temp &= ~DREF_SSC_SOURCE_MASK; temp |= DREF_SSC_SOURCE_ENABLE; /* SSC must be turned on before enabling the CPU output */ if (intel_panel_use_ssc(dev_priv) && can_ssc) { DRM_DEBUG_KMS("Using SSC on panel\n"); temp |= DREF_SSC1_ENABLE; } else temp &= ~DREF_SSC1_ENABLE; /* Get SSC going before enabling the outputs */ I915_WRITE(PCH_DREF_CONTROL, temp); POSTING_READ(PCH_DREF_CONTROL); DELAY(200); temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK; /* Enable CPU source on CPU attached eDP */ if (has_cpu_edp) { if (intel_panel_use_ssc(dev_priv) && can_ssc) { DRM_DEBUG_KMS("Using SSC on eDP\n"); temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD; } else temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD; } else temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE; I915_WRITE(PCH_DREF_CONTROL, temp); POSTING_READ(PCH_DREF_CONTROL); DELAY(200); } else { DRM_DEBUG_KMS("Disabling SSC entirely\n"); temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK; /* Turn off CPU output */ temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE; I915_WRITE(PCH_DREF_CONTROL, temp); POSTING_READ(PCH_DREF_CONTROL); DELAY(200); /* Turn off the SSC source */ temp &= ~DREF_SSC_SOURCE_MASK; temp |= DREF_SSC_SOURCE_DISABLE; /* Turn off SSC1 */ temp &= ~ DREF_SSC1_ENABLE; I915_WRITE(PCH_DREF_CONTROL, temp); POSTING_READ(PCH_DREF_CONTROL); DELAY(200); } } static int ironlake_get_refclk(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_encoder *encoder; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *edp_encoder = NULL; int num_connectors = 0; bool is_lvds = false; list_for_each_entry(encoder, &mode_config->encoder_list, base.head) { if (encoder->base.crtc != crtc) continue; switch (encoder->type) { case INTEL_OUTPUT_LVDS: is_lvds = true; break; case INTEL_OUTPUT_EDP: edp_encoder = encoder; break; } num_connectors++; } if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) { DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n", dev_priv->lvds_ssc_freq); return dev_priv->lvds_ssc_freq * 1000; } return 120000; } static int ironlake_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode, int x, int y, struct drm_framebuffer *old_fb) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; int refclk, num_connectors = 0; intel_clock_t clock, reduced_clock; u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf; bool ok, has_reduced_clock = false, is_sdvo = false; bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder, *edp_encoder = NULL; const intel_limit_t *limit; int ret; struct fdi_m_n m_n = {0}; u32 temp; int target_clock, pixel_multiplier, lane, link_bw, factor; unsigned int pipe_bpp; bool dither; bool is_cpu_edp = false, is_pch_edp = false; list_for_each_entry(encoder, &mode_config->encoder_list, base.head) { if (encoder->base.crtc != crtc) continue; switch (encoder->type) { case INTEL_OUTPUT_LVDS: is_lvds = true; break; case INTEL_OUTPUT_SDVO: case INTEL_OUTPUT_HDMI: is_sdvo = true; if (encoder->needs_tv_clock) is_tv = true; break; case INTEL_OUTPUT_TVOUT: is_tv = true; break; case INTEL_OUTPUT_ANALOG: is_crt = true; break; case INTEL_OUTPUT_DISPLAYPORT: is_dp = true; break; case INTEL_OUTPUT_EDP: is_dp = true; if (intel_encoder_is_pch_edp(&encoder->base)) is_pch_edp = true; else is_cpu_edp = true; edp_encoder = encoder; break; } num_connectors++; } refclk = ironlake_get_refclk(crtc); /* * Returns a set of divisors for the desired target clock with the given * refclk, or false. The returned values represent the clock equation: * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. */ limit = intel_limit(crtc, refclk); ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL, &clock); if (!ok) { DRM_ERROR("Couldn't find PLL settings for mode!\n"); return -EINVAL; } /* Ensure that the cursor is valid for the new mode before changing... */ intel_crtc_update_cursor(crtc, true); if (is_lvds && dev_priv->lvds_downclock_avail) { /* * Ensure we match the reduced clock's P to the target clock. * If the clocks don't match, we can't switch the display clock * by using the FP0/FP1. In such case we will disable the LVDS * downclock feature. */ has_reduced_clock = limit->find_pll(limit, crtc, dev_priv->lvds_downclock, refclk, &clock, &reduced_clock); } /* SDVO TV has fixed PLL values depend on its clock range, this mirrors vbios setting. */ if (is_sdvo && is_tv) { if (adjusted_mode->clock >= 100000 && adjusted_mode->clock < 140500) { clock.p1 = 2; clock.p2 = 10; clock.n = 3; clock.m1 = 16; clock.m2 = 8; } else if (adjusted_mode->clock >= 140500 && adjusted_mode->clock <= 200000) { clock.p1 = 1; clock.p2 = 10; clock.n = 6; clock.m1 = 12; clock.m2 = 8; } } /* FDI link */ pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode); lane = 0; /* CPU eDP doesn't require FDI link, so just set DP M/N according to current link config */ if (is_cpu_edp) { target_clock = mode->clock; intel_edp_link_config(edp_encoder, &lane, &link_bw); } else { /* [e]DP over FDI requires target mode clock instead of link clock */ if (is_dp) target_clock = mode->clock; else target_clock = adjusted_mode->clock; /* FDI is a binary signal running at ~2.7GHz, encoding * each output octet as 10 bits. The actual frequency * is stored as a divider into a 100MHz clock, and the * mode pixel clock is stored in units of 1KHz. * Hence the bw of each lane in terms of the mode signal * is: */ link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10; } /* determine panel color depth */ temp = I915_READ(PIPECONF(pipe)); temp &= ~PIPE_BPC_MASK; dither = intel_choose_pipe_bpp_dither(crtc, &pipe_bpp, mode); switch (pipe_bpp) { case 18: temp |= PIPE_6BPC; break; case 24: temp |= PIPE_8BPC; break; case 30: temp |= PIPE_10BPC; break; case 36: temp |= PIPE_12BPC; break; default: printf("intel_choose_pipe_bpp returned invalid value %d\n", pipe_bpp); temp |= PIPE_8BPC; pipe_bpp = 24; break; } intel_crtc->bpp = pipe_bpp; I915_WRITE(PIPECONF(pipe), temp); if (!lane) { /* * Account for spread spectrum to avoid * oversubscribing the link. Max center spread * is 2.5%; use 5% for safety's sake. */ u32 bps = target_clock * intel_crtc->bpp * 21 / 20; lane = bps / (link_bw * 8) + 1; } intel_crtc->fdi_lanes = lane; if (pixel_multiplier > 1) link_bw *= pixel_multiplier; ironlake_compute_m_n(intel_crtc->bpp, lane, target_clock, link_bw, &m_n); fp = clock.n << 16 | clock.m1 << 8 | clock.m2; if (has_reduced_clock) fp2 = reduced_clock.n << 16 | reduced_clock.m1 << 8 | reduced_clock.m2; /* Enable autotuning of the PLL clock (if permissible) */ factor = 21; if (is_lvds) { if ((intel_panel_use_ssc(dev_priv) && dev_priv->lvds_ssc_freq == 100) || (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP) factor = 25; } else if (is_sdvo && is_tv) factor = 20; if (clock.m < factor * clock.n) fp |= FP_CB_TUNE; dpll = 0; if (is_lvds) dpll |= DPLLB_MODE_LVDS; else dpll |= DPLLB_MODE_DAC_SERIAL; if (is_sdvo) { int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode); if (pixel_multiplier > 1) { dpll |= (pixel_multiplier - 1) << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT; } dpll |= DPLL_DVO_HIGH_SPEED; } if (is_dp && !is_cpu_edp) dpll |= DPLL_DVO_HIGH_SPEED; /* compute bitmask from p1 value */ dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; /* also FPA1 */ dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; switch (clock.p2) { case 5: dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5; break; case 7: dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7; break; case 10: dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10; break; case 14: dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; break; } if (is_sdvo && is_tv) dpll |= PLL_REF_INPUT_TVCLKINBC; else if (is_tv) /* XXX: just matching BIOS for now */ /* dpll |= PLL_REF_INPUT_TVCLKINBC; */ dpll |= 3; else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; else dpll |= PLL_REF_INPUT_DREFCLK; /* setup pipeconf */ pipeconf = I915_READ(PIPECONF(pipe)); /* Set up the display plane register */ dspcntr = DISPPLANE_GAMMA_ENABLE; DRM_DEBUG_KMS("Mode for pipe %d:\n", pipe); drm_mode_debug_printmodeline(mode); /* CPU eDP is the only output that doesn't need a PCH PLL of its own on * pre-Haswell/LPT generation */ if (HAS_PCH_LPT(dev)) { DRM_DEBUG_KMS("LPT detected: no PLL for pipe %d necessary\n", pipe); } else if (!is_cpu_edp) { struct intel_pch_pll *pll; pll = intel_get_pch_pll(intel_crtc, dpll, fp); if (pll == NULL) { DRM_DEBUG_DRIVER("failed to find PLL for pipe %d\n", pipe); return -EINVAL; } } else intel_put_pch_pll(intel_crtc); /* The LVDS pin pair needs to be on before the DPLLs are enabled. * This is an exception to the general rule that mode_set doesn't turn * things on. */ if (is_lvds) { temp = I915_READ(PCH_LVDS); temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP; if (HAS_PCH_CPT(dev)) { temp &= ~PORT_TRANS_SEL_MASK; temp |= PORT_TRANS_SEL_CPT(pipe); } else { if (pipe == 1) temp |= LVDS_PIPEB_SELECT; else temp &= ~LVDS_PIPEB_SELECT; } /* set the corresponsding LVDS_BORDER bit */ temp |= dev_priv->lvds_border_bits; /* Set the B0-B3 data pairs corresponding to whether we're going to * set the DPLLs for dual-channel mode or not. */ if (clock.p2 == 7) temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP; else temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP); /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP) * appropriately here, but we need to look more thoroughly into how * panels behave in the two modes. */ temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY); if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC) temp |= LVDS_HSYNC_POLARITY; if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC) temp |= LVDS_VSYNC_POLARITY; I915_WRITE(PCH_LVDS, temp); } pipeconf &= ~PIPECONF_DITHER_EN; pipeconf &= ~PIPECONF_DITHER_TYPE_MASK; if ((is_lvds && dev_priv->lvds_dither) || dither) { pipeconf |= PIPECONF_DITHER_EN; pipeconf |= PIPECONF_DITHER_TYPE_SP; } if (is_dp && !is_cpu_edp) { intel_dp_set_m_n(crtc, mode, adjusted_mode); } else { /* For non-DP output, clear any trans DP clock recovery setting.*/ I915_WRITE(TRANSDATA_M1(pipe), 0); I915_WRITE(TRANSDATA_N1(pipe), 0); I915_WRITE(TRANSDPLINK_M1(pipe), 0); I915_WRITE(TRANSDPLINK_N1(pipe), 0); } if (intel_crtc->pch_pll) { I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll); /* Wait for the clocks to stabilize. */ POSTING_READ(intel_crtc->pch_pll->pll_reg); DELAY(150); /* The pixel multiplier can only be updated once the * DPLL is enabled and the clocks are stable. * * So write it again. */ I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll); } intel_crtc->lowfreq_avail = false; if (intel_crtc->pch_pll) { if (is_lvds && has_reduced_clock && i915_powersave) { I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp2); intel_crtc->lowfreq_avail = true; if (HAS_PIPE_CXSR(dev)) { DRM_DEBUG_KMS("enabling CxSR downclocking\n"); pipeconf |= PIPECONF_CXSR_DOWNCLOCK; } } else { I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp); if (HAS_PIPE_CXSR(dev)) { DRM_DEBUG_KMS("disabling CxSR downclocking\n"); pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK; } } } pipeconf &= ~PIPECONF_INTERLACE_MASK; if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { pipeconf |= PIPECONF_INTERLACED_ILK; /* the chip adds 2 halflines automatically */ adjusted_mode->crtc_vtotal -= 1; adjusted_mode->crtc_vblank_end -= 1; I915_WRITE(VSYNCSHIFT(pipe), adjusted_mode->crtc_hsync_start - adjusted_mode->crtc_htotal/2); } else { pipeconf |= PIPECONF_PROGRESSIVE; I915_WRITE(VSYNCSHIFT(pipe), 0); } I915_WRITE(HTOTAL(pipe), (adjusted_mode->crtc_hdisplay - 1) | ((adjusted_mode->crtc_htotal - 1) << 16)); I915_WRITE(HBLANK(pipe), (adjusted_mode->crtc_hblank_start - 1) | ((adjusted_mode->crtc_hblank_end - 1) << 16)); I915_WRITE(HSYNC(pipe), (adjusted_mode->crtc_hsync_start - 1) | ((adjusted_mode->crtc_hsync_end - 1) << 16)); I915_WRITE(VTOTAL(pipe), (adjusted_mode->crtc_vdisplay - 1) | ((adjusted_mode->crtc_vtotal - 1) << 16)); I915_WRITE(VBLANK(pipe), (adjusted_mode->crtc_vblank_start - 1) | ((adjusted_mode->crtc_vblank_end - 1) << 16)); I915_WRITE(VSYNC(pipe), (adjusted_mode->crtc_vsync_start - 1) | ((adjusted_mode->crtc_vsync_end - 1) << 16)); /* pipesrc controls the size that is scaled from, which should * always be the user's requested size. */ I915_WRITE(PIPESRC(pipe), ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1)); I915_WRITE(PIPE_DATA_M1(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m); I915_WRITE(PIPE_DATA_N1(pipe), m_n.gmch_n); I915_WRITE(PIPE_LINK_M1(pipe), m_n.link_m); I915_WRITE(PIPE_LINK_N1(pipe), m_n.link_n); if (is_cpu_edp) ironlake_set_pll_edp(crtc, adjusted_mode->clock); I915_WRITE(PIPECONF(pipe), pipeconf); POSTING_READ(PIPECONF(pipe)); intel_wait_for_vblank(dev, pipe); I915_WRITE(DSPCNTR(plane), dspcntr); POSTING_READ(DSPCNTR(plane)); ret = intel_pipe_set_base(crtc, x, y, old_fb); intel_update_watermarks(dev); intel_update_linetime_watermarks(dev, pipe, adjusted_mode); return ret; } static int intel_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode, int x, int y, struct drm_framebuffer *old_fb) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int ret; drm_vblank_pre_modeset(dev, pipe); ret = dev_priv->display.crtc_mode_set(crtc, mode, adjusted_mode, x, y, old_fb); drm_vblank_post_modeset(dev, pipe); if (ret) intel_crtc->dpms_mode = DRM_MODE_DPMS_OFF; else intel_crtc->dpms_mode = DRM_MODE_DPMS_ON; return ret; } static bool intel_eld_uptodate(struct drm_connector *connector, int reg_eldv, uint32_t bits_eldv, int reg_elda, uint32_t bits_elda, int reg_edid) { struct drm_i915_private *dev_priv = connector->dev->dev_private; uint8_t *eld = connector->eld; uint32_t i; i = I915_READ(reg_eldv); i &= bits_eldv; if (!eld[0]) return !i; if (!i) return false; i = I915_READ(reg_elda); i &= ~bits_elda; I915_WRITE(reg_elda, i); for (i = 0; i < eld[2]; i++) if (I915_READ(reg_edid) != *((uint32_t *)eld + i)) return false; return true; } static void g4x_write_eld(struct drm_connector *connector, struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = connector->dev->dev_private; uint8_t *eld = connector->eld; uint32_t eldv; uint32_t len; uint32_t i; i = I915_READ(G4X_AUD_VID_DID); if (i == INTEL_AUDIO_DEVBLC || i == INTEL_AUDIO_DEVCL) eldv = G4X_ELDV_DEVCL_DEVBLC; else eldv = G4X_ELDV_DEVCTG; if (intel_eld_uptodate(connector, G4X_AUD_CNTL_ST, eldv, G4X_AUD_CNTL_ST, G4X_ELD_ADDR, G4X_HDMIW_HDMIEDID)) return; i = I915_READ(G4X_AUD_CNTL_ST); i &= ~(eldv | G4X_ELD_ADDR); len = (i >> 9) & 0x1f; /* ELD buffer size */ I915_WRITE(G4X_AUD_CNTL_ST, i); if (!eld[0]) return; if (eld[2] < (uint8_t)len) len = eld[2]; DRM_DEBUG_KMS("ELD size %d\n", len); for (i = 0; i < len; i++) I915_WRITE(G4X_HDMIW_HDMIEDID, *((uint32_t *)eld + i)); i = I915_READ(G4X_AUD_CNTL_ST); i |= eldv; I915_WRITE(G4X_AUD_CNTL_ST, i); } static void ironlake_write_eld(struct drm_connector *connector, struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = connector->dev->dev_private; uint8_t *eld = connector->eld; uint32_t eldv; uint32_t i; int len; int hdmiw_hdmiedid; int aud_config; int aud_cntl_st; int aud_cntrl_st2; if (HAS_PCH_IBX(connector->dev)) { hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID_A; aud_config = IBX_AUD_CONFIG_A; aud_cntl_st = IBX_AUD_CNTL_ST_A; aud_cntrl_st2 = IBX_AUD_CNTL_ST2; } else { hdmiw_hdmiedid = CPT_HDMIW_HDMIEDID_A; aud_config = CPT_AUD_CONFIG_A; aud_cntl_st = CPT_AUD_CNTL_ST_A; aud_cntrl_st2 = CPT_AUD_CNTRL_ST2; } i = to_intel_crtc(crtc)->pipe; hdmiw_hdmiedid += i * 0x100; aud_cntl_st += i * 0x100; aud_config += i * 0x100; DRM_DEBUG_KMS("ELD on pipe %c\n", pipe_name(i)); i = I915_READ(aud_cntl_st); i = (i >> 29) & 0x3; /* DIP_Port_Select, 0x1 = PortB */ if (!i) { DRM_DEBUG_KMS("Audio directed to unknown port\n"); /* operate blindly on all ports */ eldv = IBX_ELD_VALIDB; eldv |= IBX_ELD_VALIDB << 4; eldv |= IBX_ELD_VALIDB << 8; } else { DRM_DEBUG_KMS("ELD on port %c\n", 'A' + i); eldv = IBX_ELD_VALIDB << ((i - 1) * 4); } if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) { DRM_DEBUG_DRIVER("ELD: DisplayPort detected\n"); eld[5] |= (1 << 2); /* Conn_Type, 0x1 = DisplayPort */ I915_WRITE(aud_config, AUD_CONFIG_N_VALUE_INDEX); /* 0x1 = DP */ } else I915_WRITE(aud_config, 0); if (intel_eld_uptodate(connector, aud_cntrl_st2, eldv, aud_cntl_st, IBX_ELD_ADDRESS, hdmiw_hdmiedid)) return; i = I915_READ(aud_cntrl_st2); i &= ~eldv; I915_WRITE(aud_cntrl_st2, i); if (!eld[0]) return; i = I915_READ(aud_cntl_st); i &= ~IBX_ELD_ADDRESS; I915_WRITE(aud_cntl_st, i); /* 84 bytes of hw ELD buffer */ len = 21; if (eld[2] < (uint8_t)len) len = eld[2]; DRM_DEBUG_KMS("ELD size %d\n", len); for (i = 0; i < len; i++) I915_WRITE(hdmiw_hdmiedid, *((uint32_t *)eld + i)); i = I915_READ(aud_cntrl_st2); i |= eldv; I915_WRITE(aud_cntrl_st2, i); } void intel_write_eld(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_crtc *crtc = encoder->crtc; struct drm_connector *connector; struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; connector = drm_select_eld(encoder, mode); if (!connector) return; DRM_DEBUG_KMS("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", connector->base.id, drm_get_connector_name(connector), connector->encoder->base.id, drm_get_encoder_name(connector->encoder)); connector->eld[6] = drm_av_sync_delay(connector, mode) / 2; if (dev_priv->display.write_eld) dev_priv->display.write_eld(connector, crtc); } /** Loads the palette/gamma unit for the CRTC with the prepared values */ void intel_crtc_load_lut(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int palreg = PALETTE(intel_crtc->pipe); int i; /* The clocks have to be on to load the palette. */ if (!crtc->enabled || !intel_crtc->active) return; /* use legacy palette for Ironlake */ if (HAS_PCH_SPLIT(dev)) palreg = LGC_PALETTE(intel_crtc->pipe); for (i = 0; i < 256; i++) { I915_WRITE(palreg + 4 * i, (intel_crtc->lut_r[i] << 16) | (intel_crtc->lut_g[i] << 8) | intel_crtc->lut_b[i]); } } static void i845_update_cursor(struct drm_crtc *crtc, u32 base) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); bool visible = base != 0; u32 cntl; if (intel_crtc->cursor_visible == visible) return; cntl = I915_READ(_CURACNTR); if (visible) { /* On these chipsets we can only modify the base whilst * the cursor is disabled. */ I915_WRITE(_CURABASE, base); cntl &= ~(CURSOR_FORMAT_MASK); /* XXX width must be 64, stride 256 => 0x00 << 28 */ cntl |= CURSOR_ENABLE | CURSOR_GAMMA_ENABLE | CURSOR_FORMAT_ARGB; } else cntl &= ~(CURSOR_ENABLE | CURSOR_GAMMA_ENABLE); I915_WRITE(_CURACNTR, cntl); intel_crtc->cursor_visible = visible; } static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; bool visible = base != 0; if (intel_crtc->cursor_visible != visible) { uint32_t cntl = I915_READ(CURCNTR(pipe)); if (base) { cntl &= ~(CURSOR_MODE | MCURSOR_PIPE_SELECT); cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE; cntl |= pipe << 28; /* Connect to correct pipe */ } else { cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE); cntl |= CURSOR_MODE_DISABLE; } I915_WRITE(CURCNTR(pipe), cntl); intel_crtc->cursor_visible = visible; } /* and commit changes on next vblank */ I915_WRITE(CURBASE(pipe), base); } static void ivb_update_cursor(struct drm_crtc *crtc, u32 base) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; bool visible = base != 0; if (intel_crtc->cursor_visible != visible) { uint32_t cntl = I915_READ(CURCNTR_IVB(pipe)); if (base) { cntl &= ~CURSOR_MODE; cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE; } else { cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE); cntl |= CURSOR_MODE_DISABLE; } I915_WRITE(CURCNTR_IVB(pipe), cntl); intel_crtc->cursor_visible = visible; } /* and commit changes on next vblank */ I915_WRITE(CURBASE_IVB(pipe), base); } /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */ static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int x = intel_crtc->cursor_x; int y = intel_crtc->cursor_y; u32 base, pos; bool visible; pos = 0; if (on && crtc->enabled && crtc->fb) { base = intel_crtc->cursor_addr; if (x > (int) crtc->fb->width) base = 0; if (y > (int) crtc->fb->height) base = 0; } else base = 0; if (x < 0) { if (x + intel_crtc->cursor_width < 0) base = 0; pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT; x = -x; } pos |= x << CURSOR_X_SHIFT; if (y < 0) { if (y + intel_crtc->cursor_height < 0) base = 0; pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT; y = -y; } pos |= y << CURSOR_Y_SHIFT; visible = base != 0; if (!visible && !intel_crtc->cursor_visible) return; if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) { I915_WRITE(CURPOS_IVB(pipe), pos); ivb_update_cursor(crtc, base); } else { I915_WRITE(CURPOS(pipe), pos); if (IS_845G(dev) || IS_I865G(dev)) i845_update_cursor(crtc, base); else i9xx_update_cursor(crtc, base); } } static int intel_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file, uint32_t handle, uint32_t width, uint32_t height) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_i915_gem_object *obj; uint32_t addr; int ret; DRM_DEBUG_KMS("\n"); /* if we want to turn off the cursor ignore width and height */ if (!handle) { DRM_DEBUG_KMS("cursor off\n"); addr = 0; obj = NULL; DRM_LOCK(dev); goto finish; } /* Currently we only support 64x64 cursors */ if (width != 64 || height != 64) { DRM_ERROR("we currently only support 64x64 cursors\n"); return -EINVAL; } obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); if (&obj->base == NULL) return -ENOENT; if (obj->base.size < width * height * 4) { DRM_ERROR("buffer is to small\n"); ret = -ENOMEM; goto fail; } /* we only need to pin inside GTT if cursor is non-phy */ DRM_LOCK(dev); if (!dev_priv->info->cursor_needs_physical) { if (obj->tiling_mode) { DRM_ERROR("cursor cannot be tiled\n"); ret = -EINVAL; goto fail_locked; } ret = i915_gem_object_pin_to_display_plane(obj, 0, NULL); if (ret) { DRM_ERROR("failed to move cursor bo into the GTT\n"); goto fail_locked; } ret = i915_gem_object_put_fence(obj); if (ret) { DRM_ERROR("failed to release fence for cursor\n"); goto fail_unpin; } addr = obj->gtt_offset; } else { int align = IS_I830(dev) ? 16 * 1024 : 256; ret = i915_gem_attach_phys_object(dev, obj, (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1, align); if (ret) { DRM_ERROR("failed to attach phys object\n"); goto fail_locked; } addr = obj->phys_obj->handle->busaddr; } if (IS_GEN2(dev)) I915_WRITE(CURSIZE, (height << 12) | width); finish: if (intel_crtc->cursor_bo) { if (dev_priv->info->cursor_needs_physical) { if (intel_crtc->cursor_bo != obj) i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo); } else i915_gem_object_unpin_from_display_plane(intel_crtc->cursor_bo); drm_gem_object_unreference(&intel_crtc->cursor_bo->base); } DRM_UNLOCK(dev); intel_crtc->cursor_addr = addr; intel_crtc->cursor_bo = obj; intel_crtc->cursor_width = width; intel_crtc->cursor_height = height; intel_crtc_update_cursor(crtc, true); return 0; fail_unpin: i915_gem_object_unpin_from_display_plane(obj); fail_locked: DRM_UNLOCK(dev); fail: drm_gem_object_unreference_unlocked(&obj->base); return ret; } static int intel_crtc_cursor_move(struct drm_crtc *crtc, int x, int y) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); intel_crtc->cursor_x = x; intel_crtc->cursor_y = y; intel_crtc_update_cursor(crtc, true); return 0; } /** Sets the color ramps on behalf of RandR */ void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green, u16 blue, int regno) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); intel_crtc->lut_r[regno] = red >> 8; intel_crtc->lut_g[regno] = green >> 8; intel_crtc->lut_b[regno] = blue >> 8; } void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue, int regno) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); *red = intel_crtc->lut_r[regno] << 8; *green = intel_crtc->lut_g[regno] << 8; *blue = intel_crtc->lut_b[regno] << 8; } static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue, uint32_t start, uint32_t size) { int end = (start + size > 256) ? 256 : start + size, i; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); for (i = start; i < end; i++) { intel_crtc->lut_r[i] = red[i] >> 8; intel_crtc->lut_g[i] = green[i] >> 8; intel_crtc->lut_b[i] = blue[i] >> 8; } intel_crtc_load_lut(crtc); } /** * Get a pipe with a simple mode set on it for doing load-based monitor * detection. * * It will be up to the load-detect code to adjust the pipe as appropriate for * its requirements. The pipe will be connected to no other encoders. * * Currently this code will only succeed if there is a pipe with no encoders * configured for it. In the future, it could choose to temporarily disable * some outputs to free up a pipe for its use. * * \return crtc, or NULL if no pipes are available. */ /* VESA 640x480x72Hz mode to set on the pipe */ static struct drm_display_mode load_detect_mode = { DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664, 704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), }; static int intel_framebuffer_create(struct drm_device *dev, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_i915_gem_object *obj, struct drm_framebuffer **res) { struct intel_framebuffer *intel_fb; int ret; intel_fb = malloc(sizeof(*intel_fb), DRM_MEM_KMS, M_WAITOK | M_ZERO); ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj); if (ret) { drm_gem_object_unreference_unlocked(&obj->base); free(intel_fb, DRM_MEM_KMS); return (ret); } *res = &intel_fb->base; return (0); } static u32 intel_framebuffer_pitch_for_width(int width, int bpp) { u32 pitch = howmany(width * bpp, 8); return roundup2(pitch, 64); } static u32 intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp) { u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp); return roundup2(pitch * mode->vdisplay, PAGE_SIZE); } static int intel_framebuffer_create_for_mode(struct drm_device *dev, struct drm_display_mode *mode, int depth, int bpp, struct drm_framebuffer **res) { struct drm_i915_gem_object *obj; struct drm_mode_fb_cmd2 mode_cmd; obj = i915_gem_alloc_object(dev, intel_framebuffer_size_for_mode(mode, bpp)); if (obj == NULL) return (-ENOMEM); mode_cmd.width = mode->hdisplay; mode_cmd.height = mode->vdisplay; mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width, bpp); mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth); return (intel_framebuffer_create(dev, &mode_cmd, obj, res)); } static int mode_fits_in_fbdev(struct drm_device *dev, struct drm_display_mode *mode, struct drm_framebuffer **res) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; struct drm_framebuffer *fb; if (dev_priv->fbdev == NULL) { *res = NULL; return (0); } obj = dev_priv->fbdev->ifb.obj; if (obj == NULL) { *res = NULL; return (0); } fb = &dev_priv->fbdev->ifb.base; if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay, fb->bits_per_pixel)) { *res = NULL; return (0); } if (obj->base.size < mode->vdisplay * fb->pitches[0]) { *res = NULL; return (0); } *res = fb; return (0); } bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder, struct drm_connector *connector, struct drm_display_mode *mode, struct intel_load_detect_pipe *old) { struct intel_crtc *intel_crtc; struct drm_crtc *possible_crtc; struct drm_encoder *encoder = &intel_encoder->base; struct drm_crtc *crtc = NULL; struct drm_device *dev = encoder->dev; struct drm_framebuffer *old_fb; int i = -1, r; DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", connector->base.id, drm_get_connector_name(connector), encoder->base.id, drm_get_encoder_name(encoder)); /* * Algorithm gets a little messy: * * - if the connector already has an assigned crtc, use it (but make * sure it's on first) * * - try to find the first unused crtc that can drive this connector, * and use that if we find one */ /* See if we already have a CRTC for this connector */ if (encoder->crtc) { crtc = encoder->crtc; intel_crtc = to_intel_crtc(crtc); old->dpms_mode = intel_crtc->dpms_mode; old->load_detect_temp = false; /* Make sure the crtc and connector are running */ if (intel_crtc->dpms_mode != DRM_MODE_DPMS_ON) { struct drm_encoder_helper_funcs *encoder_funcs; struct drm_crtc_helper_funcs *crtc_funcs; crtc_funcs = crtc->helper_private; crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON); encoder_funcs = encoder->helper_private; encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON); } return true; } /* Find an unused one (if possible) */ list_for_each_entry(possible_crtc, &dev->mode_config.crtc_list, head) { i++; if (!(encoder->possible_crtcs & (1 << i))) continue; if (!possible_crtc->enabled) { crtc = possible_crtc; break; } } /* * If we didn't find an unused CRTC, don't use any. */ if (!crtc) { DRM_DEBUG_KMS("no pipe available for load-detect\n"); return false; } encoder->crtc = crtc; connector->encoder = encoder; intel_crtc = to_intel_crtc(crtc); old->dpms_mode = intel_crtc->dpms_mode; old->load_detect_temp = true; old->release_fb = NULL; if (!mode) mode = &load_detect_mode; old_fb = crtc->fb; /* We need a framebuffer large enough to accommodate all accesses * that the plane may generate whilst we perform load detection. * We can not rely on the fbcon either being present (we get called * during its initialisation to detect all boot displays, or it may * not even exist) or that it is large enough to satisfy the * requested mode. */ r = mode_fits_in_fbdev(dev, mode, &crtc->fb); if (crtc->fb == NULL) { DRM_DEBUG_KMS("creating tmp fb for load-detection\n"); r = intel_framebuffer_create_for_mode(dev, mode, 24, 32, &crtc->fb); old->release_fb = crtc->fb; } else DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n"); if (r != 0) { DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n"); crtc->fb = old_fb; return false; } if (!drm_crtc_helper_set_mode(crtc, mode, 0, 0, old_fb)) { DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n"); if (old->release_fb) old->release_fb->funcs->destroy(old->release_fb); crtc->fb = old_fb; return false; } /* let the connector get through one full cycle before testing */ intel_wait_for_vblank(dev, intel_crtc->pipe); return true; } void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder, struct drm_connector *connector, struct intel_load_detect_pipe *old) { struct drm_encoder *encoder = &intel_encoder->base; struct drm_device *dev = encoder->dev; struct drm_crtc *crtc = encoder->crtc; struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private; struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private; DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", connector->base.id, drm_get_connector_name(connector), encoder->base.id, drm_get_encoder_name(encoder)); if (old->load_detect_temp) { connector->encoder = NULL; drm_helper_disable_unused_functions(dev); if (old->release_fb) old->release_fb->funcs->destroy(old->release_fb); return; } /* Switch crtc and encoder back off if necessary */ if (old->dpms_mode != DRM_MODE_DPMS_ON) { encoder_funcs->dpms(encoder, old->dpms_mode); crtc_funcs->dpms(crtc, old->dpms_mode); } } /* Returns the clock of the currently programmed mode of the given pipe. */ static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 dpll = I915_READ(DPLL(pipe)); u32 fp; intel_clock_t clock; if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0) fp = I915_READ(FP0(pipe)); else fp = I915_READ(FP1(pipe)); clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT; if (IS_PINEVIEW(dev)) { clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1; clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT; } else { clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT; clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT; } if (!IS_GEN2(dev)) { if (IS_PINEVIEW(dev)) clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >> DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW); else clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >> DPLL_FPA01_P1_POST_DIV_SHIFT); switch (dpll & DPLL_MODE_MASK) { case DPLLB_MODE_DAC_SERIAL: clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ? 5 : 10; break; case DPLLB_MODE_LVDS: clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ? 7 : 14; break; default: DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed " "mode\n", (int)(dpll & DPLL_MODE_MASK)); return 0; } /* XXX: Handle the 100Mhz refclk */ intel_clock(dev, 96000, &clock); } else { bool is_lvds = (pipe == 1) && (I915_READ(LVDS) & LVDS_PORT_EN); if (is_lvds) { clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >> DPLL_FPA01_P1_POST_DIV_SHIFT); clock.p2 = 14; if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN) { /* XXX: might not be 66MHz */ intel_clock(dev, 66000, &clock); } else intel_clock(dev, 48000, &clock); } else { if (dpll & PLL_P1_DIVIDE_BY_TWO) clock.p1 = 2; else { clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >> DPLL_FPA01_P1_POST_DIV_SHIFT) + 2; } if (dpll & PLL_P2_DIVIDE_BY_4) clock.p2 = 4; else clock.p2 = 2; intel_clock(dev, 48000, &clock); } } /* XXX: It would be nice to validate the clocks, but we can't reuse * i830PllIsValid() because it relies on the xf86_config connector * configuration being accurate, which it isn't necessarily. */ return clock.dot; } /** Returns the currently programmed mode of the given pipe. */ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev, struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; struct drm_display_mode *mode; int htot = I915_READ(HTOTAL(pipe)); int hsync = I915_READ(HSYNC(pipe)); int vtot = I915_READ(VTOTAL(pipe)); int vsync = I915_READ(VSYNC(pipe)); mode = malloc(sizeof(*mode), DRM_MEM_KMS, M_WAITOK | M_ZERO); mode->clock = intel_crtc_clock_get(dev, crtc); mode->hdisplay = (htot & 0xffff) + 1; mode->htotal = ((htot & 0xffff0000) >> 16) + 1; mode->hsync_start = (hsync & 0xffff) + 1; mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1; mode->vdisplay = (vtot & 0xffff) + 1; mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1; mode->vsync_start = (vsync & 0xffff) + 1; mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1; drm_mode_set_name(mode); return mode; } #define GPU_IDLE_TIMEOUT (500 /* ms */ * 1000 / hz) /* When this timer fires, we've been idle for awhile */ static void intel_gpu_idle_timer(void *arg) { struct drm_device *dev = arg; drm_i915_private_t *dev_priv = dev->dev_private; if (!list_empty(&dev_priv->mm.active_list)) { /* Still processing requests, so just re-arm the timer. */ callout_schedule(&dev_priv->idle_callout, GPU_IDLE_TIMEOUT); return; } dev_priv->busy = false; taskqueue_enqueue(dev_priv->tq, &dev_priv->idle_task); } #define CRTC_IDLE_TIMEOUT (1000 /* ms */ * 1000 / hz) static void intel_crtc_idle_timer(void *arg) { struct intel_crtc *intel_crtc = arg; struct drm_crtc *crtc = &intel_crtc->base; drm_i915_private_t *dev_priv = crtc->dev->dev_private; struct intel_framebuffer *intel_fb; intel_fb = to_intel_framebuffer(crtc->fb); if (intel_fb && intel_fb->obj->active) { /* The framebuffer is still being accessed by the GPU. */ callout_schedule(&intel_crtc->idle_callout, CRTC_IDLE_TIMEOUT); return; } intel_crtc->busy = false; taskqueue_enqueue(dev_priv->tq, &dev_priv->idle_task); } static void intel_increase_pllclock(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; drm_i915_private_t *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int dpll_reg = DPLL(pipe); int dpll; if (HAS_PCH_SPLIT(dev)) return; if (!dev_priv->lvds_downclock_avail) return; dpll = I915_READ(dpll_reg); if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) { DRM_DEBUG_DRIVER("upclocking LVDS\n"); assert_panel_unlocked(dev_priv, pipe); dpll &= ~DISPLAY_RATE_SELECT_FPA1; I915_WRITE(dpll_reg, dpll); intel_wait_for_vblank(dev, pipe); dpll = I915_READ(dpll_reg); if (dpll & DISPLAY_RATE_SELECT_FPA1) DRM_DEBUG_DRIVER("failed to upclock LVDS!\n"); } /* Schedule downclock */ callout_reset(&intel_crtc->idle_callout, CRTC_IDLE_TIMEOUT, intel_crtc_idle_timer, intel_crtc); } static void intel_decrease_pllclock(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; drm_i915_private_t *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); if (HAS_PCH_SPLIT(dev)) return; if (!dev_priv->lvds_downclock_avail) return; /* * Since this is called by a timer, we should never get here in * the manual case. */ if (!HAS_PIPE_CXSR(dev) && intel_crtc->lowfreq_avail) { int pipe = intel_crtc->pipe; int dpll_reg = DPLL(pipe); u32 dpll; DRM_DEBUG_DRIVER("downclocking LVDS\n"); assert_panel_unlocked(dev_priv, pipe); dpll = I915_READ(dpll_reg); dpll |= DISPLAY_RATE_SELECT_FPA1; I915_WRITE(dpll_reg, dpll); intel_wait_for_vblank(dev, pipe); dpll = I915_READ(dpll_reg); if (!(dpll & DISPLAY_RATE_SELECT_FPA1)) DRM_DEBUG_DRIVER("failed to downclock LVDS!\n"); } } /** * intel_idle_update - adjust clocks for idleness * @work: work struct * * Either the GPU or display (or both) went idle. Check the busy status * here and adjust the CRTC and GPU clocks as necessary. */ static void intel_idle_update(void *arg, int pending) { drm_i915_private_t *dev_priv = arg; struct drm_device *dev = dev_priv->dev; struct drm_crtc *crtc; struct intel_crtc *intel_crtc; if (!i915_powersave) return; DRM_LOCK(dev); i915_update_gfx_val(dev_priv); list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { /* Skip inactive CRTCs */ if (!crtc->fb) continue; intel_crtc = to_intel_crtc(crtc); if (!intel_crtc->busy) intel_decrease_pllclock(crtc); } DRM_UNLOCK(dev); } /** * intel_mark_busy - mark the GPU and possibly the display busy * @dev: drm device * @obj: object we're operating on * * Callers can use this function to indicate that the GPU is busy processing * commands. If @obj matches one of the CRTC objects (i.e. it's a scanout * buffer), we'll also mark the display as busy, so we know to increase its * clock frequency. */ void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_crtc *crtc = NULL; struct intel_framebuffer *intel_fb; struct intel_crtc *intel_crtc; if (!drm_core_check_feature(dev, DRIVER_MODESET)) return; if (!dev_priv->busy) { intel_sanitize_pm(dev); dev_priv->busy = true; } else callout_reset(&dev_priv->idle_callout, GPU_IDLE_TIMEOUT, intel_gpu_idle_timer, dev); if (obj == NULL) return; list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { if (!crtc->fb) continue; intel_crtc = to_intel_crtc(crtc); intel_fb = to_intel_framebuffer(crtc->fb); if (intel_fb->obj == obj) { if (!intel_crtc->busy) { /* Non-busy -> busy, upclock */ intel_increase_pllclock(crtc); intel_crtc->busy = true; } else { /* Busy -> busy, put off timer */ callout_reset(&intel_crtc->idle_callout, CRTC_IDLE_TIMEOUT, intel_crtc_idle_timer, intel_crtc); } } } } static void intel_crtc_destroy(struct drm_crtc *crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_unpin_work *work; mtx_lock(&dev->event_lock); work = intel_crtc->unpin_work; intel_crtc->unpin_work = NULL; mtx_unlock(&dev->event_lock); if (work) { taskqueue_cancel(dev_priv->tq, &work->task, NULL); taskqueue_drain(dev_priv->tq, &work->task); free(work, DRM_MEM_KMS); } drm_crtc_cleanup(crtc); free(intel_crtc, DRM_MEM_KMS); } static void intel_unpin_work_fn(void *arg, int pending) { struct intel_unpin_work *work = arg; struct drm_device *dev; dev = work->dev; DRM_LOCK(dev); intel_unpin_fb_obj(work->old_fb_obj); drm_gem_object_unreference(&work->pending_flip_obj->base); drm_gem_object_unreference(&work->old_fb_obj->base); intel_update_fbc(work->dev); DRM_UNLOCK(dev); free(work, DRM_MEM_KMS); } static void do_intel_finish_page_flip(struct drm_device *dev, struct drm_crtc *crtc) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_unpin_work *work; struct drm_i915_gem_object *obj; struct drm_pending_vblank_event *e; struct timeval tnow, tvbl; /* Ignore early vblank irqs */ if (intel_crtc == NULL) return; microtime(&tnow); mtx_lock(&dev->event_lock); work = intel_crtc->unpin_work; if (work == NULL || !work->pending) { mtx_unlock(&dev->event_lock); return; } intel_crtc->unpin_work = NULL; if (work->event) { e = work->event; e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl); /* Called before vblank count and timestamps have * been updated for the vblank interval of flip * completion? Need to increment vblank count and * add one videorefresh duration to returned timestamp * to account for this. We assume this happened if we * get called over 0.9 frame durations after the last * timestamped vblank. * * This calculation can not be used with vrefresh rates * below 5Hz (10Hz to be on the safe side) without * promoting to 64 integers. */ if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) > 9 * crtc->framedur_ns) { e->event.sequence++; tvbl = ns_to_timeval(timeval_to_ns(&tvbl) + crtc->framedur_ns); } e->event.tv_sec = tvbl.tv_sec; e->event.tv_usec = tvbl.tv_usec; list_add_tail(&e->base.link, &e->base.file_priv->event_list); drm_event_wakeup(&e->base); } drm_vblank_put(dev, intel_crtc->pipe); obj = work->old_fb_obj; atomic_clear_int(&obj->pending_flip, 1 << intel_crtc->plane); if (atomic_load_acq_int(&obj->pending_flip) == 0) wakeup(&obj->pending_flip); mtx_unlock(&dev->event_lock); taskqueue_enqueue(dev_priv->tq, &work->task); CTR2(KTR_DRM, "i915_flip_complete %d %p", intel_crtc->plane, work->pending_flip_obj); } void intel_finish_page_flip(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; do_intel_finish_page_flip(dev, crtc); } void intel_finish_page_flip_plane(struct drm_device *dev, int plane) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane]; do_intel_finish_page_flip(dev, crtc); } void intel_prepare_page_flip(struct drm_device *dev, int plane) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]); mtx_lock(&dev->event_lock); if (intel_crtc->unpin_work) { if ((++intel_crtc->unpin_work->pending) > 1) DRM_ERROR("Prepared flip multiple times\n"); } else { DRM_DEBUG("preparing flip with no unpin work?\n"); } mtx_unlock(&dev->event_lock); } static int intel_gen2_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); unsigned long offset; u32 flip_mask; struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; int ret; ret = intel_pin_and_fence_fb_obj(dev, obj, ring); if (ret) goto err; /* Offset into the new buffer for cases of shared fbs between CRTCs */ offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8; ret = intel_ring_begin(ring, 6); if (ret) goto err_unpin; /* Can't queue multiple flips, so wait for the previous * one to finish before executing the next. */ if (intel_crtc->plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; else flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); intel_ring_emit(ring, obj->gtt_offset + offset); intel_ring_emit(ring, 0); /* aux display base address, unused */ intel_ring_advance(ring); return 0; err_unpin: intel_unpin_fb_obj(obj); err: return ret; } static int intel_gen3_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); unsigned long offset; u32 flip_mask; struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; int ret; ret = intel_pin_and_fence_fb_obj(dev, obj, ring); if (ret) goto err; /* Offset into the new buffer for cases of shared fbs between CRTCs */ offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8; ret = intel_ring_begin(ring, 6); if (ret) goto err_unpin; if (intel_crtc->plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; else flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); intel_ring_emit(ring, obj->gtt_offset + offset); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; err_unpin: intel_unpin_fb_obj(obj); err: return ret; } static int intel_gen4_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pf, pipesrc; struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; int ret; ret = intel_pin_and_fence_fb_obj(dev, obj, ring); if (ret) goto err; ret = intel_ring_begin(ring, 4); if (ret) goto err_unpin; /* i965+ uses the linear or tiled offsets from the * Display Registers (which do not change across a page-flip) * so we need only reprogram the base address. */ intel_ring_emit(ring, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); intel_ring_emit(ring, obj->gtt_offset | obj->tiling_mode); /* XXX Enabling the panel-fitter across page-flip is so far * untested on non-native modes, so ignore it for now. * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE; */ pf = 0; pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; intel_ring_emit(ring, pf | pipesrc); intel_ring_advance(ring); return 0; err_unpin: intel_unpin_fb_obj(obj); err: return ret; } static int intel_gen6_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; uint32_t pf, pipesrc; int ret; ret = intel_pin_and_fence_fb_obj(dev, obj, ring); if (ret) goto err; ret = intel_ring_begin(ring, 4); if (ret) goto err_unpin; intel_ring_emit(ring, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode); intel_ring_emit(ring, obj->gtt_offset); /* Contrary to the suggestions in the documentation, * "Enable Panel Fitter" does not seem to be required when page * flipping with a non-native mode, and worse causes a normal * modeset to fail. * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE; */ pf = 0; pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; intel_ring_emit(ring, pf | pipesrc); intel_ring_advance(ring); return 0; err_unpin: intel_unpin_fb_obj(obj); err: return ret; } /* * On gen7 we currently use the blit ring because (in early silicon at least) * the render ring doesn't give us interrpts for page flip completion, which * means clients will hang after the first flip is queued. Fortunately the * blit ring generates interrupts properly, so use it instead. */ static int intel_gen7_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_ring_buffer *ring = &dev_priv->rings[BCS]; int ret; ret = intel_pin_and_fence_fb_obj(dev, obj, ring); if (ret) goto err; ret = intel_ring_begin(ring, 4); if (ret) goto err_unpin; intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | (intel_crtc->plane << 19)); intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode)); intel_ring_emit(ring, (obj->gtt_offset)); intel_ring_emit(ring, (MI_NOOP)); intel_ring_advance(ring); return 0; err_unpin: intel_unpin_fb_obj(obj); err: return ret; } static int intel_default_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj) { return -ENODEV; } static int intel_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_pending_vblank_event *event) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_framebuffer *intel_fb; struct drm_i915_gem_object *obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_unpin_work *work; int ret; work = malloc(sizeof *work, DRM_MEM_KMS, M_WAITOK | M_ZERO); work->event = event; work->dev = crtc->dev; intel_fb = to_intel_framebuffer(crtc->fb); work->old_fb_obj = intel_fb->obj; TASK_INIT(&work->task, 0, intel_unpin_work_fn, work); ret = drm_vblank_get(dev, intel_crtc->pipe); if (ret) goto free_work; /* We borrow the event spin lock for protecting unpin_work */ mtx_lock(&dev->event_lock); if (intel_crtc->unpin_work) { mtx_unlock(&dev->event_lock); free(work, DRM_MEM_KMS); drm_vblank_put(dev, intel_crtc->pipe); DRM_DEBUG("flip queue: crtc already busy\n"); return -EBUSY; } intel_crtc->unpin_work = work; mtx_unlock(&dev->event_lock); intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; DRM_LOCK(dev); /* Reference the objects for the scheduled work. */ drm_gem_object_reference(&work->old_fb_obj->base); drm_gem_object_reference(&obj->base); crtc->fb = fb; work->pending_flip_obj = obj; work->enable_stall_check = true; /* Block clients from rendering to the new back buffer until * the flip occurs and the object is no longer visible. */ atomic_set_int(&work->old_fb_obj->pending_flip, 1 << intel_crtc->plane); ret = dev_priv->display.queue_flip(dev, crtc, fb, obj); if (ret) goto cleanup_pending; intel_disable_fbc(dev); intel_mark_busy(dev, obj); DRM_UNLOCK(dev); CTR2(KTR_DRM, "i915_flip_request %d %p", intel_crtc->plane, obj); return 0; cleanup_pending: atomic_clear_int(&work->old_fb_obj->pending_flip, 1 << intel_crtc->plane); drm_gem_object_unreference(&work->old_fb_obj->base); drm_gem_object_unreference(&obj->base); DRM_UNLOCK(dev); mtx_lock(&dev->event_lock); intel_crtc->unpin_work = NULL; mtx_unlock(&dev->event_lock); drm_vblank_put(dev, intel_crtc->pipe); free_work: free(work, DRM_MEM_KMS); return ret; } static void intel_sanitize_modesetting(struct drm_device *dev, int pipe, int plane) { struct drm_i915_private *dev_priv = dev->dev_private; u32 reg, val; int i; /* Clear any frame start delays used for debugging left by the BIOS */ for_each_pipe(i) { reg = PIPECONF(i); I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK); } if (HAS_PCH_SPLIT(dev)) return; /* Who knows what state these registers were left in by the BIOS or * grub? * * If we leave the registers in a conflicting state (e.g. with the * display plane reading from the other pipe than the one we intend * to use) then when we attempt to teardown the active mode, we will * not disable the pipes and planes in the correct order -- leaving * a plane reading from a disabled pipe and possibly leading to * undefined behaviour. */ reg = DSPCNTR(plane); val = I915_READ(reg); if ((val & DISPLAY_PLANE_ENABLE) == 0) return; if (!!(val & DISPPLANE_SEL_PIPE_MASK) == pipe) return; /* This display plane is active and attached to the other CPU pipe. */ pipe = !pipe; /* Disable the plane and wait for it to stop reading from the pipe. */ intel_disable_plane(dev_priv, plane, pipe); intel_disable_pipe(dev_priv, pipe); } static void intel_crtc_reset(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); /* Reset flags back to the 'unknown' status so that they * will be correctly set on the initial modeset. */ intel_crtc->dpms_mode = -1; /* We need to fix up any BIOS configuration that conflicts with * our expectations. */ intel_sanitize_modesetting(dev, intel_crtc->pipe, intel_crtc->plane); } static struct drm_crtc_helper_funcs intel_helper_funcs = { .dpms = intel_crtc_dpms, .mode_fixup = intel_crtc_mode_fixup, .mode_set = intel_crtc_mode_set, .mode_set_base = intel_pipe_set_base, .mode_set_base_atomic = intel_pipe_set_base_atomic, .load_lut = intel_crtc_load_lut, .disable = intel_crtc_disable, }; static const struct drm_crtc_funcs intel_crtc_funcs = { .reset = intel_crtc_reset, .cursor_set = intel_crtc_cursor_set, .cursor_move = intel_crtc_cursor_move, .gamma_set = intel_crtc_gamma_set, .set_config = drm_crtc_helper_set_config, .destroy = intel_crtc_destroy, .page_flip = intel_crtc_page_flip, }; static void intel_pch_pll_init(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; int i; if (dev_priv->num_pch_pll == 0) { DRM_DEBUG_KMS("No PCH PLLs on this hardware, skipping initialisation\n"); return; } for (i = 0; i < dev_priv->num_pch_pll; i++) { dev_priv->pch_plls[i].pll_reg = _PCH_DPLL(i); dev_priv->pch_plls[i].fp0_reg = _PCH_FP0(i); dev_priv->pch_plls[i].fp1_reg = _PCH_FP1(i); } } static void intel_crtc_init(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc; int i; intel_crtc = malloc(sizeof(struct intel_crtc) + (INTELFB_CONN_LIMIT * sizeof(struct drm_connector *)), DRM_MEM_KMS, M_WAITOK | M_ZERO); drm_crtc_init(dev, &intel_crtc->base, &intel_crtc_funcs); drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256); for (i = 0; i < 256; i++) { intel_crtc->lut_r[i] = i; intel_crtc->lut_g[i] = i; intel_crtc->lut_b[i] = i; } /* Swap pipes & planes for FBC on pre-965 */ intel_crtc->pipe = pipe; intel_crtc->plane = pipe; if (IS_MOBILE(dev) && IS_GEN3(dev)) { DRM_DEBUG_KMS("swapping pipes & planes for FBC\n"); intel_crtc->plane = !pipe; } KASSERT(pipe < DRM_ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) && dev_priv->plane_to_crtc_mapping[intel_crtc->plane] == NULL, ("plane_to_crtc is already initialized")); dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base; dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base; intel_crtc_reset(&intel_crtc->base); intel_crtc->active = true; /* force the pipe off on setup_init_config */ intel_crtc->bpp = 24; /* default for pre-Ironlake */ if (HAS_PCH_SPLIT(dev)) { intel_helper_funcs.prepare = ironlake_crtc_prepare; intel_helper_funcs.commit = ironlake_crtc_commit; } else { intel_helper_funcs.prepare = i9xx_crtc_prepare; intel_helper_funcs.commit = i9xx_crtc_commit; } drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs); intel_crtc->busy = false; callout_init(&intel_crtc->idle_callout, CALLOUT_MPSAFE); } int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data; struct drm_mode_object *drmmode_obj; struct intel_crtc *crtc; if (!drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; drmmode_obj = drm_mode_object_find(dev, pipe_from_crtc_id->crtc_id, DRM_MODE_OBJECT_CRTC); if (!drmmode_obj) { DRM_ERROR("no such CRTC id\n"); return -EINVAL; } crtc = to_intel_crtc(obj_to_crtc(drmmode_obj)); pipe_from_crtc_id->pipe = crtc->pipe; return 0; } static int intel_encoder_clones(struct drm_device *dev, int type_mask) { struct intel_encoder *encoder; int index_mask = 0; int entry = 0; list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { if (type_mask & encoder->clone_mask) index_mask |= (1 << entry); entry++; } return index_mask; } static bool has_edp_a(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; if (!IS_MOBILE(dev)) return false; if ((I915_READ(DP_A) & DP_DETECTED) == 0) return false; if (IS_GEN5(dev) && (I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE)) return false; return true; } static void intel_setup_outputs(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_encoder *encoder; bool dpd_is_edp = false; bool has_lvds; has_lvds = intel_lvds_init(dev); if (!has_lvds && !HAS_PCH_SPLIT(dev)) { /* disable the panel fitter on everything but LVDS */ I915_WRITE(PFIT_CONTROL, 0); } if (HAS_PCH_SPLIT(dev)) { dpd_is_edp = intel_dpd_is_edp(dev); if (has_edp_a(dev)) intel_dp_init(dev, DP_A); if (dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED)) intel_dp_init(dev, PCH_DP_D); } intel_crt_init(dev); if (IS_HASWELL(dev)) { int found; /* Haswell uses DDI functions to detect digital outputs */ found = I915_READ(DDI_BUF_CTL_A) & DDI_INIT_DISPLAY_DETECTED; /* DDI A only supports eDP */ if (found) intel_ddi_init(dev, PORT_A); /* DDI B, C and D detection is indicated by the SFUSE_STRAP * register */ found = I915_READ(SFUSE_STRAP); if (found & SFUSE_STRAP_DDIB_DETECTED) intel_ddi_init(dev, PORT_B); if (found & SFUSE_STRAP_DDIC_DETECTED) intel_ddi_init(dev, PORT_C); if (found & SFUSE_STRAP_DDID_DETECTED) intel_ddi_init(dev, PORT_D); } else if (HAS_PCH_SPLIT(dev)) { int found; DRM_DEBUG_KMS( "HDMIB %d PCH_DP_B %d HDMIC %d HDMID %d PCH_DP_C %d PCH_DP_D %d LVDS %d\n", (I915_READ(HDMIB) & PORT_DETECTED) != 0, (I915_READ(PCH_DP_B) & DP_DETECTED) != 0, (I915_READ(HDMIC) & PORT_DETECTED) != 0, (I915_READ(HDMID) & PORT_DETECTED) != 0, (I915_READ(PCH_DP_C) & DP_DETECTED) != 0, (I915_READ(PCH_DP_D) & DP_DETECTED) != 0, (I915_READ(PCH_LVDS) & LVDS_DETECTED) != 0); if (I915_READ(HDMIB) & PORT_DETECTED) { /* PCH SDVOB multiplex with HDMIB */ found = intel_sdvo_init(dev, PCH_SDVOB, true); if (!found) intel_hdmi_init(dev, HDMIB); if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED)) intel_dp_init(dev, PCH_DP_B); } if (I915_READ(HDMIC) & PORT_DETECTED) intel_hdmi_init(dev, HDMIC); if (I915_READ(HDMID) & PORT_DETECTED) intel_hdmi_init(dev, HDMID); if (I915_READ(PCH_DP_C) & DP_DETECTED) intel_dp_init(dev, PCH_DP_C); if (!dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED)) intel_dp_init(dev, PCH_DP_D); } else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) { bool found = false; if (I915_READ(SDVOB) & SDVO_DETECTED) { DRM_DEBUG_KMS("probing SDVOB\n"); found = intel_sdvo_init(dev, SDVOB, true); if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) { DRM_DEBUG_KMS("probing HDMI on SDVOB\n"); intel_hdmi_init(dev, SDVOB); } if (!found && SUPPORTS_INTEGRATED_DP(dev)) { DRM_DEBUG_KMS("probing DP_B\n"); intel_dp_init(dev, DP_B); } } /* Before G4X SDVOC doesn't have its own detect register */ if (I915_READ(SDVOB) & SDVO_DETECTED) { DRM_DEBUG_KMS("probing SDVOC\n"); found = intel_sdvo_init(dev, SDVOC, false); } if (!found && (I915_READ(SDVOC) & SDVO_DETECTED)) { if (SUPPORTS_INTEGRATED_HDMI(dev)) { DRM_DEBUG_KMS("probing HDMI on SDVOC\n"); intel_hdmi_init(dev, SDVOC); } if (SUPPORTS_INTEGRATED_DP(dev)) { DRM_DEBUG_KMS("probing DP_C\n"); intel_dp_init(dev, DP_C); } } if (SUPPORTS_INTEGRATED_DP(dev) && (I915_READ(DP_D) & DP_DETECTED)) { DRM_DEBUG_KMS("probing DP_D\n"); intel_dp_init(dev, DP_D); } } else if (IS_GEN2(dev)) { #if 1 KIB_NOTYET(); #else intel_dvo_init(dev); #endif } if (SUPPORTS_TV(dev)) intel_tv_init(dev); list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { encoder->base.possible_crtcs = encoder->crtc_mask; encoder->base.possible_clones = intel_encoder_clones(dev, encoder->clone_mask); } /* disable all the possible outputs/crtcs before entering KMS mode */ drm_helper_disable_unused_functions(dev); if (HAS_PCH_SPLIT(dev)) ironlake_init_pch_refclk(dev); } static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb) { struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); drm_framebuffer_cleanup(fb); drm_gem_object_unreference_unlocked(&intel_fb->obj->base); free(intel_fb, DRM_MEM_KMS); } static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, struct drm_file *file, unsigned int *handle) { struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; return drm_gem_handle_create(file, &obj->base, handle); } static const struct drm_framebuffer_funcs intel_fb_funcs = { .destroy = intel_user_framebuffer_destroy, .create_handle = intel_user_framebuffer_create_handle, }; int intel_framebuffer_init(struct drm_device *dev, struct intel_framebuffer *intel_fb, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_i915_gem_object *obj) { int ret; if (obj->tiling_mode == I915_TILING_Y) return -EINVAL; if (mode_cmd->pitches[0] & 63) return -EINVAL; switch (mode_cmd->pixel_format) { case DRM_FORMAT_RGB332: case DRM_FORMAT_RGB565: case DRM_FORMAT_XRGB8888: case DRM_FORMAT_XBGR8888: case DRM_FORMAT_ARGB8888: case DRM_FORMAT_XRGB2101010: case DRM_FORMAT_ARGB2101010: /* RGB formats are common across chipsets */ break; case DRM_FORMAT_YUYV: case DRM_FORMAT_UYVY: case DRM_FORMAT_YVYU: case DRM_FORMAT_VYUY: break; default: DRM_DEBUG_KMS("unsupported pixel format %u\n", mode_cmd->pixel_format); return -EINVAL; } ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs); if (ret) { DRM_ERROR("framebuffer init failed %d\n", ret); return ret; } drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd); intel_fb->obj = obj; return 0; } static int intel_user_framebuffer_create(struct drm_device *dev, struct drm_file *filp, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_framebuffer **res) { struct drm_i915_gem_object *obj; obj = to_intel_bo(drm_gem_object_lookup(dev, filp, mode_cmd->handles[0])); if (&obj->base == NULL) return (-ENOENT); return (intel_framebuffer_create(dev, mode_cmd, obj, res)); } static const struct drm_mode_config_funcs intel_mode_funcs = { .fb_create = intel_user_framebuffer_create, .output_poll_changed = intel_fb_output_poll_changed, }; /* Set up chip specific display functions */ static void intel_init_display(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; /* We always want a DPMS function */ if (HAS_PCH_SPLIT(dev)) { dev_priv->display.dpms = ironlake_crtc_dpms; dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set; dev_priv->display.off = ironlake_crtc_off; dev_priv->display.update_plane = ironlake_update_plane; } else { dev_priv->display.dpms = i9xx_crtc_dpms; dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set; dev_priv->display.off = i9xx_crtc_off; dev_priv->display.update_plane = i9xx_update_plane; } /* Returns the core display clock speed */ if (IS_VALLEYVIEW(dev)) dev_priv->display.get_display_clock_speed = valleyview_get_display_clock_speed; else if (IS_I945G(dev) || (IS_G33(dev) && !IS_PINEVIEW_M(dev))) dev_priv->display.get_display_clock_speed = i945_get_display_clock_speed; else if (IS_I915G(dev)) dev_priv->display.get_display_clock_speed = i915_get_display_clock_speed; else if (IS_I945GM(dev) || IS_845G(dev) || IS_PINEVIEW_M(dev)) dev_priv->display.get_display_clock_speed = i9xx_misc_get_display_clock_speed; else if (IS_I915GM(dev)) dev_priv->display.get_display_clock_speed = i915gm_get_display_clock_speed; else if (IS_I865G(dev)) dev_priv->display.get_display_clock_speed = i865_get_display_clock_speed; else if (IS_I85X(dev)) dev_priv->display.get_display_clock_speed = i855_get_display_clock_speed; else /* 852, 830 */ dev_priv->display.get_display_clock_speed = i830_get_display_clock_speed; if (HAS_PCH_SPLIT(dev)) { if (IS_GEN5(dev)) { dev_priv->display.fdi_link_train = ironlake_fdi_link_train; dev_priv->display.write_eld = ironlake_write_eld; } else if (IS_GEN6(dev)) { dev_priv->display.fdi_link_train = gen6_fdi_link_train; dev_priv->display.write_eld = ironlake_write_eld; } else if (IS_IVYBRIDGE(dev)) { /* FIXME: detect B0+ stepping and use auto training */ dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; dev_priv->display.write_eld = ironlake_write_eld; } else if (IS_HASWELL(dev)) { dev_priv->display.fdi_link_train = hsw_fdi_link_train; dev_priv->display.write_eld = ironlake_write_eld; } else dev_priv->display.update_wm = NULL; } else if (IS_VALLEYVIEW(dev)) { dev_priv->display.force_wake_get = vlv_force_wake_get; dev_priv->display.force_wake_put = vlv_force_wake_put; } else if (IS_G4X(dev)) { dev_priv->display.write_eld = g4x_write_eld; } /* Default just returns -ENODEV to indicate unsupported */ dev_priv->display.queue_flip = intel_default_queue_flip; switch (INTEL_INFO(dev)->gen) { case 2: dev_priv->display.queue_flip = intel_gen2_queue_flip; break; case 3: dev_priv->display.queue_flip = intel_gen3_queue_flip; break; case 4: case 5: dev_priv->display.queue_flip = intel_gen4_queue_flip; break; case 6: dev_priv->display.queue_flip = intel_gen6_queue_flip; break; case 7: dev_priv->display.queue_flip = intel_gen7_queue_flip; break; } } /* * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend, * resume, or other times. This quirk makes sure that's the case for * affected systems. */ static void quirk_pipea_force(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; dev_priv->quirks |= QUIRK_PIPEA_FORCE; DRM_INFO("applying pipe a force quirk\n"); } /* * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason */ static void quirk_ssc_force_disable(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE; DRM_INFO("applying lvds SSC disable quirk\n"); } /* * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight * brightness value */ static void quirk_invert_brightness(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS; DRM_INFO("applying inverted panel brightness quirk\n"); } struct intel_quirk { int device; int subsystem_vendor; int subsystem_device; void (*hook)(struct drm_device *dev); }; #define PCI_ANY_ID (~0u) static struct intel_quirk intel_quirks[] = { /* HP Mini needs pipe A force quirk (LP: #322104) */ { 0x27ae, 0x103c, 0x361a, quirk_pipea_force }, /* Thinkpad R31 needs pipe A force quirk */ { 0x3577, 0x1014, 0x0505, quirk_pipea_force }, /* Toshiba Protege R-205, S-209 needs pipe A force quirk */ { 0x2592, 0x1179, 0x0001, quirk_pipea_force }, /* ThinkPad X30 needs pipe A force quirk (LP: #304614) */ { 0x3577, 0x1014, 0x0513, quirk_pipea_force }, /* ThinkPad X40 needs pipe A force quirk */ /* ThinkPad T60 needs pipe A force quirk (bug #16494) */ { 0x2782, 0x17aa, 0x201a, quirk_pipea_force }, /* 855 & before need to leave pipe A & dpll A up */ { 0x3582, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force }, { 0x2562, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force }, /* Lenovo U160 cannot use SSC on LVDS */ { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable }, /* Sony Vaio Y cannot use SSC on LVDS */ { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable }, /* Acer Aspire 5734Z must invert backlight brightness */ { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness }, }; static void intel_init_quirks(struct drm_device *dev) { struct intel_quirk *q; device_t d; int i; d = dev->device; for (i = 0; i < DRM_ARRAY_SIZE(intel_quirks); i++) { q = &intel_quirks[i]; if (pci_get_device(d) == q->device && (pci_get_subvendor(d) == q->subsystem_vendor || q->subsystem_vendor == PCI_ANY_ID) && (pci_get_subdevice(d) == q->subsystem_device || q->subsystem_device == PCI_ANY_ID)) q->hook(dev); } } /* Disable the VGA plane that we never use */ static void i915_disable_vga(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; u8 sr1; u32 vga_reg; if (HAS_PCH_SPLIT(dev)) vga_reg = CPU_VGACNTRL; else vga_reg = VGACNTRL; #if 0 vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO); #endif outb(VGA_SR_INDEX, SR01); sr1 = inb(VGA_SR_DATA); outb(VGA_SR_DATA, sr1 | 1 << 5); #if 0 vga_put(dev->pdev, VGA_RSRC_LEGACY_IO); #endif DELAY(300); I915_WRITE(vga_reg, VGA_DISP_DISABLE); POSTING_READ(vga_reg); } static void ivb_pch_pwm_override(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; /* * IVB has CPU eDP backlight regs too, set things up to let the * PCH regs control the backlight */ I915_WRITE(BLC_PWM_CPU_CTL2, PWM_ENABLE); I915_WRITE(BLC_PWM_CPU_CTL, 0); - I915_WRITE(BLC_PWM_PCH_CTL1, PWM_ENABLE | (1<<30)); + I915_WRITE(BLC_PWM_PCH_CTL1, PWM_ENABLE); } void intel_modeset_init_hw(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; intel_init_clock_gating(dev); if (IS_IRONLAKE_M(dev)) { ironlake_enable_drps(dev); ironlake_enable_rc6(dev); intel_init_emon(dev); } if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) { gen6_enable_rps(dev_priv); gen6_update_ring_freq(dev_priv); } if (IS_IVYBRIDGE(dev)) ivb_pch_pwm_override(dev); } void intel_modeset_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int i, ret; drm_mode_config_init(dev); dev->mode_config.min_width = 0; dev->mode_config.min_height = 0; dev->mode_config.preferred_depth = 24; dev->mode_config.prefer_shadow = 1; dev->mode_config.funcs = &intel_mode_funcs; intel_init_quirks(dev); intel_init_pm(dev); intel_prepare_ddi(dev); intel_init_display(dev); if (IS_GEN2(dev)) { dev->mode_config.max_width = 2048; dev->mode_config.max_height = 2048; } else if (IS_GEN3(dev)) { dev->mode_config.max_width = 4096; dev->mode_config.max_height = 4096; } else { dev->mode_config.max_width = 8192; dev->mode_config.max_height = 8192; } dev->mode_config.fb_base = dev->agp->base; DRM_DEBUG_KMS("%d display pipe%s available.\n", dev_priv->num_pipe, dev_priv->num_pipe > 1 ? "s" : ""); for (i = 0; i < dev_priv->num_pipe; i++) { intel_crtc_init(dev, i); ret = intel_plane_init(dev, i); if (ret) DRM_DEBUG_KMS("plane %d init failed: %d\n", i, ret); } intel_pch_pll_init(dev); /* Just disable it once at startup */ i915_disable_vga(dev); intel_setup_outputs(dev); TASK_INIT(&dev_priv->idle_task, 0, intel_idle_update, dev_priv); callout_init(&dev_priv->idle_callout, CALLOUT_MPSAFE); } void intel_modeset_gem_init(struct drm_device *dev) { intel_modeset_init_hw(dev); intel_setup_overlay(dev); } void intel_modeset_cleanup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc; struct intel_crtc *intel_crtc; drm_kms_helper_poll_fini(dev); DRM_LOCK(dev); #if 0 intel_unregister_dsm_handler(); #endif list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { /* Skip inactive CRTCs */ if (!crtc->fb) continue; intel_crtc = to_intel_crtc(crtc); intel_increase_pllclock(crtc); } intel_disable_fbc(dev); if (IS_IRONLAKE_M(dev)) ironlake_disable_drps(dev); if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) gen6_disable_rps(dev); if (IS_IRONLAKE_M(dev)) ironlake_disable_rc6(dev); if (IS_VALLEYVIEW(dev)) vlv_init_dpio(dev); /* Disable the irq before mode object teardown, for the irq might * enqueue unpin/hotplug work. */ drm_irq_uninstall(dev); DRM_UNLOCK(dev); if (taskqueue_cancel(dev_priv->tq, &dev_priv->hotplug_task, NULL)) taskqueue_drain(dev_priv->tq, &dev_priv->hotplug_task); if (taskqueue_cancel(dev_priv->tq, &dev_priv->rps_task, NULL)) taskqueue_drain(dev_priv->tq, &dev_priv->rps_task); /* Shut off idle work before the crtcs get freed. */ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { intel_crtc = to_intel_crtc(crtc); callout_drain(&intel_crtc->idle_callout); } callout_drain(&dev_priv->idle_callout); if (taskqueue_cancel(dev_priv->tq, &dev_priv->idle_task, NULL)) taskqueue_drain(dev_priv->tq, &dev_priv->idle_task); drm_mode_config_cleanup(dev); } /* * Return which encoder is currently attached for connector. */ struct drm_encoder *intel_best_encoder(struct drm_connector *connector) { return &intel_attached_encoder(connector)->base; } void intel_connector_attach_encoder(struct intel_connector *connector, struct intel_encoder *encoder) { connector->encoder = encoder; drm_mode_connector_attach_encoder(&connector->base, &encoder->base); } /* * set vga decode state - true == enable VGA decode */ int intel_modeset_vga_set_state(struct drm_device *dev, bool state) { struct drm_i915_private *dev_priv; device_t bridge_dev; u16 gmch_ctrl; dev_priv = dev->dev_private; bridge_dev = intel_gtt_get_bridge_device(); gmch_ctrl = pci_read_config(bridge_dev, INTEL_GMCH_CTRL, 2); if (state) gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE; else gmch_ctrl |= INTEL_GMCH_VGA_DISABLE; pci_write_config(bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl, 2); return (0); } struct intel_display_error_state { struct intel_cursor_error_state { u32 control; u32 position; u32 base; u32 size; } cursor[2]; struct intel_pipe_error_state { u32 conf; u32 source; u32 htotal; u32 hblank; u32 hsync; u32 vtotal; u32 vblank; u32 vsync; } pipe[2]; struct intel_plane_error_state { u32 control; u32 stride; u32 size; u32 pos; u32 addr; u32 surface; u32 tile_offset; } plane[2]; }; struct intel_display_error_state * intel_display_capture_error_state(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_display_error_state *error; int i; error = malloc(sizeof(*error), DRM_MEM_KMS, M_NOWAIT); if (error == NULL) return NULL; for (i = 0; i < 2; i++) { error->cursor[i].control = I915_READ(CURCNTR(i)); error->cursor[i].position = I915_READ(CURPOS(i)); error->cursor[i].base = I915_READ(CURBASE(i)); error->plane[i].control = I915_READ(DSPCNTR(i)); error->plane[i].stride = I915_READ(DSPSTRIDE(i)); error->plane[i].size = I915_READ(DSPSIZE(i)); error->plane[i].pos = I915_READ(DSPPOS(i)); error->plane[i].addr = I915_READ(DSPADDR(i)); if (INTEL_INFO(dev)->gen >= 4) { error->plane[i].surface = I915_READ(DSPSURF(i)); error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i)); } error->pipe[i].conf = I915_READ(PIPECONF(i)); error->pipe[i].source = I915_READ(PIPESRC(i)); error->pipe[i].htotal = I915_READ(HTOTAL(i)); error->pipe[i].hblank = I915_READ(HBLANK(i)); error->pipe[i].hsync = I915_READ(HSYNC(i)); error->pipe[i].vtotal = I915_READ(VTOTAL(i)); error->pipe[i].vblank = I915_READ(VBLANK(i)); error->pipe[i].vsync = I915_READ(VSYNC(i)); } return error; } void intel_display_print_error_state(struct sbuf *m, struct drm_device *dev, struct intel_display_error_state *error) { int i; for (i = 0; i < 2; i++) { sbuf_printf(m, "Pipe [%d]:\n", i); sbuf_printf(m, " CONF: %08x\n", error->pipe[i].conf); sbuf_printf(m, " SRC: %08x\n", error->pipe[i].source); sbuf_printf(m, " HTOTAL: %08x\n", error->pipe[i].htotal); sbuf_printf(m, " HBLANK: %08x\n", error->pipe[i].hblank); sbuf_printf(m, " HSYNC: %08x\n", error->pipe[i].hsync); sbuf_printf(m, " VTOTAL: %08x\n", error->pipe[i].vtotal); sbuf_printf(m, " VBLANK: %08x\n", error->pipe[i].vblank); sbuf_printf(m, " VSYNC: %08x\n", error->pipe[i].vsync); sbuf_printf(m, "Plane [%d]:\n", i); sbuf_printf(m, " CNTR: %08x\n", error->plane[i].control); sbuf_printf(m, " STRIDE: %08x\n", error->plane[i].stride); sbuf_printf(m, " SIZE: %08x\n", error->plane[i].size); sbuf_printf(m, " POS: %08x\n", error->plane[i].pos); sbuf_printf(m, " ADDR: %08x\n", error->plane[i].addr); if (INTEL_INFO(dev)->gen >= 4) { sbuf_printf(m, " SURF: %08x\n", error->plane[i].surface); sbuf_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset); } sbuf_printf(m, "Cursor [%d]:\n", i); sbuf_printf(m, " CNTR: %08x\n", error->cursor[i].control); sbuf_printf(m, " POS: %08x\n", error->cursor[i].position); sbuf_printf(m, " BASE: %08x\n", error->cursor[i].base); } } diff --git a/sys/dev/iscsi/icl.c b/sys/dev/iscsi/icl.c index 7f5d01f5332e..a60313cf857b 100644 --- a/sys/dev/iscsi/icl.c +++ b/sys/dev/iscsi/icl.c @@ -1,1462 +1,247 @@ /*- * Copyright (c) 2012 The FreeBSD Foundation * All rights reserved. * * This software was developed by Edward Tomasz Napierala under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * iSCSI Common Layer. It's used by both the initiator and target to send * and receive iSCSI PDUs. */ #include __FBSDID("$FreeBSD$"); #include -#include #include #include -#include -#include -#include #include -#include +#include +#include #include #include -#include -#include -#include +#include #include #include #include -#include -#include -#include -#include #include -#include +#include + +struct icl_module { + TAILQ_ENTRY(icl_module) im_next; + char *im_name; + int im_priority; + int (*im_limits)(size_t *limitp); + struct icl_conn *(*im_new_conn)(const char *name, + struct mtx *lock); +}; + +struct icl_softc { + struct sx sc_lock; + TAILQ_HEAD(, icl_module) sc_modules; +}; SYSCTL_NODE(_kern, OID_AUTO, icl, CTLFLAG_RD, 0, "iSCSI Common Layer"); -static int debug = 1; +int icl_debug = 1; SYSCTL_INT(_kern_icl, OID_AUTO, debug, CTLFLAG_RWTUN, - &debug, 0, "Enable debug messages"); -static int coalesce = 1; -SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN, - &coalesce, 0, "Try to coalesce PDUs before sending"); -static int partial_receive_len = 128 * 1024; -SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, - &partial_receive_len, 0, "Minimum read size for partially received " - "data segment"); -static int sendspace = 1048576; -SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN, - &sendspace, 0, "Default send socket buffer size"); -static int recvspace = 1048576; -SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN, - &recvspace, 0, "Default receive socket buffer size"); - -static uma_zone_t icl_conn_zone; -static uma_zone_t icl_pdu_zone; - -static volatile u_int icl_ncons; - -#define ICL_DEBUG(X, ...) \ - do { \ - if (debug > 1) \ - printf("%s: " X "\n", __func__, ## __VA_ARGS__);\ - } while (0) - -#define ICL_WARN(X, ...) \ - do { \ - if (debug > 0) { \ - printf("WARNING: %s: " X "\n", \ - __func__, ## __VA_ARGS__); \ - } \ - } while (0) - -#define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) -#define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) -#define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) -#define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) - -STAILQ_HEAD(icl_pdu_stailq, icl_pdu); - -static void -icl_conn_fail(struct icl_conn *ic) -{ - if (ic->ic_socket == NULL) - return; - - /* - * XXX - */ - ic->ic_socket->so_error = EDOOFUS; - (ic->ic_error)(ic); -} - -static struct mbuf * -icl_conn_receive(struct icl_conn *ic, size_t len) -{ - struct uio uio; - struct socket *so; - struct mbuf *m; - int error, flags; - - so = ic->ic_socket; - - memset(&uio, 0, sizeof(uio)); - uio.uio_resid = len; - - flags = MSG_DONTWAIT; - error = soreceive(so, NULL, &uio, &m, NULL, &flags); - if (error != 0) { - ICL_DEBUG("soreceive error %d", error); - return (NULL); - } - if (uio.uio_resid != 0) { - m_freem(m); - ICL_DEBUG("short read"); - return (NULL); - } - - return (m); -} - -static struct icl_pdu * -icl_pdu_new_empty(struct icl_conn *ic, int flags) -{ - struct icl_pdu *ip; - -#ifdef DIAGNOSTIC - refcount_acquire(&ic->ic_outstanding_pdus); -#endif - ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO); - if (ip == NULL) { - ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); -#ifdef DIAGNOSTIC - refcount_release(&ic->ic_outstanding_pdus); -#endif - return (NULL); - } - - ip->ip_conn = ic; - - return (ip); -} - -void -icl_pdu_free(struct icl_pdu *ip) -{ - struct icl_conn *ic; - - ic = ip->ip_conn; - - m_freem(ip->ip_bhs_mbuf); - m_freem(ip->ip_ahs_mbuf); - m_freem(ip->ip_data_mbuf); - uma_zfree(icl_pdu_zone, ip); -#ifdef DIAGNOSTIC - refcount_release(&ic->ic_outstanding_pdus); -#endif -} - -/* - * Allocate icl_pdu with empty BHS to fill up by the caller. - */ -struct icl_pdu * -icl_pdu_new(struct icl_conn *ic, int flags) -{ - struct icl_pdu *ip; - - ip = icl_pdu_new_empty(ic, flags); - if (ip == NULL) - return (NULL); - - ip->ip_bhs_mbuf = m_getm2(NULL, sizeof(struct iscsi_bhs), - flags, MT_DATA, M_PKTHDR); - if (ip->ip_bhs_mbuf == NULL) { - ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); - icl_pdu_free(ip); - return (NULL); - } - ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); - memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); - ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); - - return (ip); -} - -static int -icl_pdu_ahs_length(const struct icl_pdu *request) -{ - - return (request->ip_bhs->bhs_total_ahs_len * 4); -} - -size_t -icl_pdu_data_segment_length(const struct icl_pdu *request) -{ - uint32_t len = 0; - - len += request->ip_bhs->bhs_data_segment_len[0]; - len <<= 8; - len += request->ip_bhs->bhs_data_segment_len[1]; - len <<= 8; - len += request->ip_bhs->bhs_data_segment_len[2]; - - return (len); -} - -static void -icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) -{ - - response->ip_bhs->bhs_data_segment_len[2] = len; - response->ip_bhs->bhs_data_segment_len[1] = len >> 8; - response->ip_bhs->bhs_data_segment_len[0] = len >> 16; -} - -static size_t -icl_pdu_padding(const struct icl_pdu *ip) -{ - - if ((ip->ip_data_len % 4) != 0) - return (4 - (ip->ip_data_len % 4)); - - return (0); -} - -static size_t -icl_pdu_size(const struct icl_pdu *response) -{ - size_t len; - - KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); - - len = sizeof(struct iscsi_bhs) + response->ip_data_len + - icl_pdu_padding(response); - if (response->ip_conn->ic_header_crc32c) - len += ISCSI_HEADER_DIGEST_SIZE; - if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) - len += ISCSI_DATA_DIGEST_SIZE; - - return (len); -} - -static int -icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep) -{ - struct mbuf *m; - - m = icl_conn_receive(request->ip_conn, sizeof(struct iscsi_bhs)); - if (m == NULL) { - ICL_DEBUG("failed to receive BHS"); - return (-1); - } - - request->ip_bhs_mbuf = m_pullup(m, sizeof(struct iscsi_bhs)); - if (request->ip_bhs_mbuf == NULL) { - ICL_WARN("m_pullup failed"); - return (-1); - } - request->ip_bhs = mtod(request->ip_bhs_mbuf, struct iscsi_bhs *); - - /* - * XXX: For architectures with strict alignment requirements - * we may need to allocate ip_bhs and copy the data into it. - * For some reason, though, not doing this doesn't seem - * to cause problems; tested on sparc64. - */ - - *availablep -= sizeof(struct iscsi_bhs); - return (0); -} - -static int -icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep) -{ - - request->ip_ahs_len = icl_pdu_ahs_length(request); - if (request->ip_ahs_len == 0) - return (0); - - request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn, - request->ip_ahs_len); - if (request->ip_ahs_mbuf == NULL) { - ICL_DEBUG("failed to receive AHS"); - return (-1); - } - - *availablep -= request->ip_ahs_len; - return (0); -} - -static uint32_t -icl_mbuf_to_crc32c(const struct mbuf *m0) -{ - uint32_t digest = 0xffffffff; - const struct mbuf *m; - - for (m = m0; m != NULL; m = m->m_next) - digest = calculate_crc32c(digest, - mtod(m, const void *), m->m_len); - - digest = digest ^ 0xffffffff; + &icl_debug, 0, "Enable debug messages"); - return (digest); -} +static MALLOC_DEFINE(M_ICL, "icl", "iSCSI Common Layer"); +static struct icl_softc *sc; -static int -icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep) +static struct icl_module * +icl_find(const char *name) { - struct mbuf *m; - uint32_t received_digest, valid_digest; + struct icl_module *im, *im_max; - if (request->ip_conn->ic_header_crc32c == false) - return (0); - - m = icl_conn_receive(request->ip_conn, ISCSI_HEADER_DIGEST_SIZE); - if (m == NULL) { - ICL_DEBUG("failed to receive header digest"); - return (-1); - } - - CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); - m_copydata(m, 0, ISCSI_HEADER_DIGEST_SIZE, (void *)&received_digest); - m_freem(m); - - *availablep -= ISCSI_HEADER_DIGEST_SIZE; + sx_assert(&sc->sc_lock, SA_LOCKED); /* - * XXX: Handle AHS. + * If the name was not specified, pick a module with highest + * priority. */ - valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); - if (received_digest != valid_digest) { - ICL_WARN("header digest check failed; got 0x%x, " - "should be 0x%x", received_digest, valid_digest); - return (-1); - } - - return (0); -} - -/* - * Return the number of bytes that should be waiting in the receive socket - * before icl_pdu_receive_data_segment() gets called. - */ -static size_t -icl_pdu_data_segment_receive_len(const struct icl_pdu *request) -{ - size_t len; - - len = icl_pdu_data_segment_length(request); - if (len == 0) - return (0); - - /* - * Account for the parts of data segment already read from - * the socket buffer. - */ - KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); - len -= request->ip_data_len; - - /* - * Don't always wait for the full data segment to be delivered - * to the socket; this might badly affect performance due to - * TCP window scaling. - */ - if (len > partial_receive_len) { -#if 0 - ICL_DEBUG("need %zd bytes of data, limiting to %zd", - len, partial_receive_len)); -#endif - len = partial_receive_len; - - return (len); - } - - /* - * Account for padding. Note that due to the way code is written, - * the icl_pdu_receive_data_segment() must always receive padding - * along with the last part of data segment, because it would be - * impossible to tell whether we've already received the full data - * segment including padding, or without it. - */ - if ((len % 4) != 0) - len += 4 - (len % 4); - -#if 0 - ICL_DEBUG("need %zd bytes of data", len)); -#endif - - return (len); -} - -static int -icl_pdu_receive_data_segment(struct icl_pdu *request, - size_t *availablep, bool *more_neededp) -{ - struct icl_conn *ic; - size_t len, padding = 0; - struct mbuf *m; - - ic = request->ip_conn; - - *more_neededp = false; - ic->ic_receive_len = 0; - - len = icl_pdu_data_segment_length(request); - if (len == 0) - return (0); - - if ((len % 4) != 0) - padding = 4 - (len % 4); - - /* - * Account for already received parts of data segment. - */ - KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); - len -= request->ip_data_len; - - if (len + padding > *availablep) { - /* - * Not enough data in the socket buffer. Receive as much - * as we can. Don't receive padding, since, obviously, it's - * not the end of data segment yet. - */ -#if 0 - ICL_DEBUG("limited from %zd to %zd", - len + padding, *availablep - padding)); -#endif - len = *availablep - padding; - *more_neededp = true; - padding = 0; - } - - /* - * Must not try to receive padding without at least one byte - * of actual data segment. - */ - if (len > 0) { - m = icl_conn_receive(request->ip_conn, len + padding); - if (m == NULL) { - ICL_DEBUG("failed to receive data segment"); - return (-1); + if (name == NULL || name[0] == '\0') { + im_max = TAILQ_FIRST(&sc->sc_modules); + TAILQ_FOREACH(im, &sc->sc_modules, im_next) { + if (im->im_priority > im_max->im_priority) + im_max = im; } - if (request->ip_data_mbuf == NULL) - request->ip_data_mbuf = m; - else - m_cat(request->ip_data_mbuf, m); - - request->ip_data_len += len; - *availablep -= len + padding; - } else - ICL_DEBUG("len 0"); - - if (*more_neededp) - ic->ic_receive_len = - icl_pdu_data_segment_receive_len(request); - - return (0); -} - -static int -icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep) -{ - struct mbuf *m; - uint32_t received_digest, valid_digest; - - if (request->ip_conn->ic_data_crc32c == false) - return (0); - - if (request->ip_data_len == 0) - return (0); - - m = icl_conn_receive(request->ip_conn, ISCSI_DATA_DIGEST_SIZE); - if (m == NULL) { - ICL_DEBUG("failed to receive data digest"); - return (-1); - } - - CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); - m_copydata(m, 0, ISCSI_DATA_DIGEST_SIZE, (void *)&received_digest); - m_freem(m); - - *availablep -= ISCSI_DATA_DIGEST_SIZE; - - /* - * Note that ip_data_mbuf also contains padding; since digest - * calculation is supposed to include that, we iterate over - * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. - */ - valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); - if (received_digest != valid_digest) { - ICL_WARN("data digest check failed; got 0x%x, " - "should be 0x%x", received_digest, valid_digest); - return (-1); - } - - return (0); -} - -/* - * Somewhat contrary to the name, this attempts to receive only one - * "part" of PDU at a time; call it repeatedly until it returns non-NULL. - */ -static struct icl_pdu * -icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep) -{ - struct icl_pdu *request; - struct socket *so; - size_t len; - int error; - bool more_needed; - - so = ic->ic_socket; - - if (ic->ic_receive_state == ICL_CONN_STATE_BHS) { - KASSERT(ic->ic_receive_pdu == NULL, - ("ic->ic_receive_pdu != NULL")); - request = icl_pdu_new_empty(ic, M_NOWAIT); - if (request == NULL) { - ICL_DEBUG("failed to allocate PDU; " - "dropping connection"); - icl_conn_fail(ic); - return (NULL); - } - ic->ic_receive_pdu = request; - } else { - KASSERT(ic->ic_receive_pdu != NULL, - ("ic->ic_receive_pdu == NULL")); - request = ic->ic_receive_pdu; - } - - if (*availablep < ic->ic_receive_len) { -#if 0 - ICL_DEBUG("not enough data; need %zd, " - "have %zd", ic->ic_receive_len, *availablep); -#endif - return (NULL); - } - - switch (ic->ic_receive_state) { - case ICL_CONN_STATE_BHS: - //ICL_DEBUG("receiving BHS"); - error = icl_pdu_receive_bhs(request, availablep); - if (error != 0) { - ICL_DEBUG("failed to receive BHS; " - "dropping connection"); - break; - } - - /* - * We don't enforce any limit for AHS length; - * its length is stored in 8 bit field. - */ - - len = icl_pdu_data_segment_length(request); - if (len > ic->ic_max_data_segment_length) { - ICL_WARN("received data segment " - "length %zd is larger than negotiated " - "MaxDataSegmentLength %zd; " - "dropping connection", - len, ic->ic_max_data_segment_length); - error = EINVAL; - break; - } - - ic->ic_receive_state = ICL_CONN_STATE_AHS; - ic->ic_receive_len = icl_pdu_ahs_length(request); - break; - - case ICL_CONN_STATE_AHS: - //ICL_DEBUG("receiving AHS"); - error = icl_pdu_receive_ahs(request, availablep); - if (error != 0) { - ICL_DEBUG("failed to receive AHS; " - "dropping connection"); - break; - } - ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST; - if (ic->ic_header_crc32c == false) - ic->ic_receive_len = 0; - else - ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE; - break; - - case ICL_CONN_STATE_HEADER_DIGEST: - //ICL_DEBUG("receiving header digest"); - error = icl_pdu_check_header_digest(request, availablep); - if (error != 0) { - ICL_DEBUG("header digest failed; " - "dropping connection"); - break; - } - - ic->ic_receive_state = ICL_CONN_STATE_DATA; - ic->ic_receive_len = - icl_pdu_data_segment_receive_len(request); - break; - - case ICL_CONN_STATE_DATA: - //ICL_DEBUG("receiving data segment"); - error = icl_pdu_receive_data_segment(request, availablep, - &more_needed); - if (error != 0) { - ICL_DEBUG("failed to receive data segment;" - "dropping connection"); - break; - } - - if (more_needed) - break; - - ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST; - if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) - ic->ic_receive_len = 0; - else - ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE; - break; - - case ICL_CONN_STATE_DATA_DIGEST: - //ICL_DEBUG("receiving data digest"); - error = icl_pdu_check_data_digest(request, availablep); - if (error != 0) { - ICL_DEBUG("data digest failed; " - "dropping connection"); - break; - } - - /* - * We've received complete PDU; reset the receive state machine - * and return the PDU. - */ - ic->ic_receive_state = ICL_CONN_STATE_BHS; - ic->ic_receive_len = sizeof(struct iscsi_bhs); - ic->ic_receive_pdu = NULL; - return (request); - - default: - panic("invalid ic_receive_state %d\n", ic->ic_receive_state); + return (im_max); } - if (error != 0) { - /* - * Don't free the PDU; it's pointed to by ic->ic_receive_pdu - * and will get freed in icl_conn_close(). - */ - icl_conn_fail(ic); + TAILQ_FOREACH(im, &sc->sc_modules, im_next) { + if (strcmp(im->im_name, name) == 0) + return (im); } return (NULL); } -static void -icl_conn_receive_pdus(struct icl_conn *ic, size_t available) -{ - struct icl_pdu *response; - struct socket *so; - - so = ic->ic_socket; - - /* - * This can never happen; we're careful to only mess with ic->ic_socket - * pointer when the send/receive threads are not running. - */ - KASSERT(so != NULL, ("NULL socket")); - - for (;;) { - if (ic->ic_disconnecting) - return; - - if (so->so_error != 0) { - ICL_DEBUG("connection error %d; " - "dropping connection", so->so_error); - icl_conn_fail(ic); - return; - } - - /* - * Loop until we have a complete PDU or there is not enough - * data in the socket buffer. - */ - if (available < ic->ic_receive_len) { -#if 0 - ICL_DEBUG("not enough data; have %zd, " - "need %zd", available, - ic->ic_receive_len); -#endif - return; - } - - response = icl_conn_receive_pdu(ic, &available); - if (response == NULL) - continue; - - if (response->ip_ahs_len > 0) { - ICL_WARN("received PDU with unsupported " - "AHS; opcode 0x%x; dropping connection", - response->ip_bhs->bhs_opcode); - icl_pdu_free(response); - icl_conn_fail(ic); - return; - } - - (ic->ic_receive)(response); - } -} - -static void -icl_receive_thread(void *arg) -{ - struct icl_conn *ic; - size_t available; - struct socket *so; - - ic = arg; - so = ic->ic_socket; - - ICL_CONN_LOCK(ic); - ic->ic_receive_running = true; - ICL_CONN_UNLOCK(ic); - - for (;;) { - if (ic->ic_disconnecting) { - //ICL_DEBUG("terminating"); - break; - } - - /* - * Set the low watermark, to be checked by - * soreadable() in icl_soupcall_receive() - * to avoid unneccessary wakeups until there - * is enough data received to read the PDU. - */ - SOCKBUF_LOCK(&so->so_rcv); - available = sbavail(&so->so_rcv); - if (available < ic->ic_receive_len) { - so->so_rcv.sb_lowat = ic->ic_receive_len; - cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx); - } else - so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; - SOCKBUF_UNLOCK(&so->so_rcv); - - icl_conn_receive_pdus(ic, available); - } - - ICL_CONN_LOCK(ic); - ic->ic_receive_running = false; - cv_signal(&ic->ic_send_cv); - ICL_CONN_UNLOCK(ic); - kthread_exit(); -} - -static int -icl_soupcall_receive(struct socket *so, void *arg, int waitflag) -{ - struct icl_conn *ic; - - if (!soreadable(so)) - return (SU_OK); - - ic = arg; - cv_signal(&ic->ic_receive_cv); - return (SU_OK); -} - -static int -icl_pdu_finalize(struct icl_pdu *request) +struct icl_conn * +icl_new_conn(const char *offload, const char *name, struct mtx *lock) { - size_t padding, pdu_len; - uint32_t digest, zero = 0; - int ok; + struct icl_module *im; struct icl_conn *ic; - ic = request->ip_conn; - - icl_pdu_set_data_segment_length(request, request->ip_data_len); - - pdu_len = icl_pdu_size(request); - - if (ic->ic_header_crc32c) { - digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); - ok = m_append(request->ip_bhs_mbuf, sizeof(digest), - (void *)&digest); - if (ok != 1) { - ICL_WARN("failed to append header digest"); - return (1); - } - } - - if (request->ip_data_len != 0) { - padding = icl_pdu_padding(request); - if (padding > 0) { - ok = m_append(request->ip_data_mbuf, padding, - (void *)&zero); - if (ok != 1) { - ICL_WARN("failed to append padding"); - return (1); - } - } - - if (ic->ic_data_crc32c) { - digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); - - ok = m_append(request->ip_data_mbuf, sizeof(digest), - (void *)&digest); - if (ok != 1) { - ICL_WARN("failed to append data digest"); - return (1); - } - } + sx_slock(&sc->sc_lock); + im = icl_find(offload); - m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); - request->ip_data_mbuf = NULL; + if (im == NULL) { + ICL_WARN("offload \"%s\" not found", offload); + sx_sunlock(&sc->sc_lock); + return (NULL); } - request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; - - return (0); -} - -static void -icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue) -{ - struct icl_pdu *request, *request2; - struct socket *so; - size_t available, size, size2; - int coalesced, error; + ic = im->im_new_conn(name, lock); + sx_sunlock(&sc->sc_lock); - ICL_CONN_LOCK_ASSERT_NOT(ic); - - so = ic->ic_socket; - - SOCKBUF_LOCK(&so->so_snd); - /* - * Check how much space do we have for transmit. We can't just - * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, - * as it always frees the mbuf chain passed to it, even in case - * of error. - */ - available = sbspace(&so->so_snd); - - /* - * Notify the socket upcall that we don't need wakeups - * for the time being. - */ - so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; - SOCKBUF_UNLOCK(&so->so_snd); - - while (!STAILQ_EMPTY(queue)) { - request = STAILQ_FIRST(queue); - size = icl_pdu_size(request); - if (available < size) { - - /* - * Set the low watermark, to be checked by - * sowriteable() in icl_soupcall_send() - * to avoid unneccessary wakeups until there - * is enough space for the PDU to fit. - */ - SOCKBUF_LOCK(&so->so_snd); - available = sbspace(&so->so_snd); - if (available < size) { -#if 1 - ICL_DEBUG("no space to send; " - "have %zd, need %zd", - available, size); -#endif - so->so_snd.sb_lowat = size; - SOCKBUF_UNLOCK(&so->so_snd); - return; - } - SOCKBUF_UNLOCK(&so->so_snd); - } - STAILQ_REMOVE_HEAD(queue, ip_next); - error = icl_pdu_finalize(request); - if (error != 0) { - ICL_DEBUG("failed to finalize PDU; " - "dropping connection"); - icl_conn_fail(ic); - icl_pdu_free(request); - return; - } - if (coalesce) { - coalesced = 1; - for (;;) { - request2 = STAILQ_FIRST(queue); - if (request2 == NULL) - break; - size2 = icl_pdu_size(request2); - if (available < size + size2) - break; - STAILQ_REMOVE_HEAD(queue, ip_next); - error = icl_pdu_finalize(request2); - if (error != 0) { - ICL_DEBUG("failed to finalize PDU; " - "dropping connection"); - icl_conn_fail(ic); - icl_pdu_free(request); - icl_pdu_free(request2); - return; - } - m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf); - request2->ip_bhs_mbuf = NULL; - request->ip_bhs_mbuf->m_pkthdr.len += size2; - size += size2; - STAILQ_REMOVE_AFTER(queue, request, ip_next); - icl_pdu_free(request2); - coalesced++; - } -#if 0 - if (coalesced > 1) { - ICL_DEBUG("coalesced %d PDUs into %zd bytes", - coalesced, size); - } -#endif - } - available -= size; - error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, - NULL, MSG_DONTWAIT, curthread); - request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ - if (error != 0) { - ICL_DEBUG("failed to send PDU, error %d; " - "dropping connection", error); - icl_conn_fail(ic); - icl_pdu_free(request); - return; - } - icl_pdu_free(request); - } + return (ic); } -static void -icl_send_thread(void *arg) +int +icl_limits(const char *offload, size_t *limitp) { - struct icl_conn *ic; - struct icl_pdu_stailq queue; - - ic = arg; - - STAILQ_INIT(&queue); - - ICL_CONN_LOCK(ic); - ic->ic_send_running = true; - - for (;;) { - for (;;) { - /* - * If the local queue is empty, populate it from - * the main one. This way the icl_conn_send_pdus() - * can go through all the queued PDUs without holding - * any locks. - */ - if (STAILQ_EMPTY(&queue)) - STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu); - - ic->ic_check_send_space = false; - ICL_CONN_UNLOCK(ic); - icl_conn_send_pdus(ic, &queue); - ICL_CONN_LOCK(ic); - - /* - * The icl_soupcall_send() was called since the last - * call to sbspace(); go around; - */ - if (ic->ic_check_send_space) - continue; - - /* - * Local queue is empty, but we still have PDUs - * in the main one; go around. - */ - if (STAILQ_EMPTY(&queue) && - !STAILQ_EMPTY(&ic->ic_to_send)) - continue; - - /* - * There might be some stuff in the local queue, - * which didn't get sent due to not having enough send - * space. Wait for socket upcall. - */ - break; - } + struct icl_module *im; + int error; - if (ic->ic_disconnecting) { - //ICL_DEBUG("terminating"); - break; - } + sx_slock(&sc->sc_lock); + im = icl_find(offload); - cv_wait(&ic->ic_send_cv, ic->ic_lock); + if (im == NULL) { + ICL_WARN("offload \"%s\" not found", offload); + sx_sunlock(&sc->sc_lock); + return (ENXIO); } - /* - * We're exiting; move PDUs back to the main queue, so they can - * get freed properly. At this point ordering doesn't matter. - */ - STAILQ_CONCAT(&ic->ic_to_send, &queue); + error = im->im_limits(limitp); + sx_sunlock(&sc->sc_lock); - ic->ic_send_running = false; - cv_signal(&ic->ic_send_cv); - ICL_CONN_UNLOCK(ic); - kthread_exit(); + return (error); } -static int -icl_soupcall_send(struct socket *so, void *arg, int waitflag) -{ - struct icl_conn *ic; - - if (!sowriteable(so)) - return (SU_OK); - - ic = arg; - - ICL_CONN_LOCK(ic); - ic->ic_check_send_space = true; - ICL_CONN_UNLOCK(ic); - - cv_signal(&ic->ic_send_cv); - - return (SU_OK); -} int -icl_pdu_append_data(struct icl_pdu *request, const void *addr, size_t len, - int flags) +icl_register(const char *offload, int priority, int (*limits)(size_t *), + struct icl_conn *(*new_conn)(const char *, struct mtx *)) { - struct mbuf *mb, *newmb; - size_t copylen, off = 0; + struct icl_module *im; - KASSERT(len > 0, ("len == 0")); + sx_xlock(&sc->sc_lock); + im = icl_find(offload); - newmb = m_getm2(NULL, len, flags, MT_DATA, M_PKTHDR); - if (newmb == NULL) { - ICL_WARN("failed to allocate mbuf for %zd bytes", len); - return (ENOMEM); + if (im != NULL) { + ICL_WARN("offload \"%s\" already registered", offload); + sx_xunlock(&sc->sc_lock); + return (EBUSY); } - for (mb = newmb; mb != NULL; mb = mb->m_next) { - copylen = min(M_TRAILINGSPACE(mb), len - off); - memcpy(mtod(mb, char *), (const char *)addr + off, copylen); - mb->m_len = copylen; - off += copylen; - } - KASSERT(off == len, ("%s: off != len", __func__)); + im = malloc(sizeof(*im), M_ICL, M_ZERO | M_WAITOK); + im->im_name = strdup(offload, M_ICL); + im->im_priority = priority; + im->im_limits = limits; + im->im_new_conn = new_conn; - if (request->ip_data_mbuf == NULL) { - request->ip_data_mbuf = newmb; - request->ip_data_len = len; - } else { - m_cat(request->ip_data_mbuf, newmb); - request->ip_data_len += len; - } + TAILQ_INSERT_HEAD(&sc->sc_modules, im, im_next); + sx_xunlock(&sc->sc_lock); + ICL_DEBUG("offload \"%s\" registered", offload); return (0); } -void -icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len) -{ - - m_copydata(ip->ip_data_mbuf, off, len, addr); -} - -void -icl_pdu_queue(struct icl_pdu *ip) +int +icl_unregister(const char *offload) { - struct icl_conn *ic; + struct icl_module *im; - ic = ip->ip_conn; - - ICL_CONN_LOCK_ASSERT(ic); - - if (ic->ic_disconnecting || ic->ic_socket == NULL) { - ICL_DEBUG("icl_pdu_queue on closed connection"); - icl_pdu_free(ip); - return; - } + sx_xlock(&sc->sc_lock); + im = icl_find(offload); - if (!STAILQ_EMPTY(&ic->ic_to_send)) { - STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); - /* - * If the queue is not empty, someone else had already - * signaled the send thread; no need to do that again, - * just return. - */ - return; + if (im == NULL) { + ICL_WARN("offload \"%s\" not registered", offload); + sx_xunlock(&sc->sc_lock); + return (ENXIO); } - STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); - cv_signal(&ic->ic_send_cv); -} - -struct icl_conn * -icl_conn_new(const char *name, struct mtx *lock) -{ - struct icl_conn *ic; - - refcount_acquire(&icl_ncons); + TAILQ_REMOVE(&sc->sc_modules, im, im_next); + sx_xunlock(&sc->sc_lock); - ic = uma_zalloc(icl_conn_zone, M_WAITOK | M_ZERO); + free(im->im_name, M_ICL); + free(im, M_ICL); - STAILQ_INIT(&ic->ic_to_send); - ic->ic_lock = lock; - cv_init(&ic->ic_send_cv, "icl_tx"); - cv_init(&ic->ic_receive_cv, "icl_rx"); -#ifdef DIAGNOSTIC - refcount_init(&ic->ic_outstanding_pdus, 0); -#endif - ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH; - ic->ic_name = name; - - return (ic); -} - -void -icl_conn_free(struct icl_conn *ic) -{ - - cv_destroy(&ic->ic_send_cv); - cv_destroy(&ic->ic_receive_cv); - uma_zfree(icl_conn_zone, ic); - refcount_release(&icl_ncons); + ICL_DEBUG("offload \"%s\" unregistered", offload); + return (0); } static int -icl_conn_start(struct icl_conn *ic) +icl_load(void) { - size_t minspace; - struct sockopt opt; - int error, one = 1; - - ICL_CONN_LOCK(ic); - - /* - * XXX: Ugly hack. - */ - if (ic->ic_socket == NULL) { - ICL_CONN_UNLOCK(ic); - return (EINVAL); - } - ic->ic_receive_state = ICL_CONN_STATE_BHS; - ic->ic_receive_len = sizeof(struct iscsi_bhs); - ic->ic_disconnecting = false; - - ICL_CONN_UNLOCK(ic); - - /* - * For sendspace, this is required because the current code cannot - * send a PDU in pieces; thus, the minimum buffer size is equal - * to the maximum PDU size. "+4" is to account for possible padding. - * - * What we should actually do here is to use autoscaling, but set - * some minimal buffer size to "minspace". I don't know a way to do - * that, though. - */ - minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + - ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; - if (sendspace < minspace) { - ICL_WARN("kern.icl.sendspace too low; must be at least %zd", - minspace); - sendspace = minspace; - } - if (recvspace < minspace) { - ICL_WARN("kern.icl.recvspace too low; must be at least %zd", - minspace); - recvspace = minspace; - } - - error = soreserve(ic->ic_socket, sendspace, recvspace); - if (error != 0) { - ICL_WARN("soreserve failed with error %d", error); - icl_conn_close(ic); - return (error); - } - ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; - ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; - - /* - * Disable Nagle. - */ - bzero(&opt, sizeof(opt)); - opt.sopt_dir = SOPT_SET; - opt.sopt_level = IPPROTO_TCP; - opt.sopt_name = TCP_NODELAY; - opt.sopt_val = &one; - opt.sopt_valsize = sizeof(one); - error = sosetopt(ic->ic_socket, &opt); - if (error != 0) { - ICL_WARN("disabling TCP_NODELAY failed with error %d", error); - icl_conn_close(ic); - return (error); - } - - /* - * Start threads. - */ - error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", - ic->ic_name); - if (error != 0) { - ICL_WARN("kthread_add(9) failed with error %d", error); - icl_conn_close(ic); - return (error); - } - - error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", - ic->ic_name); - if (error != 0) { - ICL_WARN("kthread_add(9) failed with error %d", error); - icl_conn_close(ic); - return (error); - } - - /* - * Register socket upcall, to get notified about incoming PDUs - * and free space to send outgoing ones. - */ - SOCKBUF_LOCK(&ic->ic_socket->so_snd); - soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic); - SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); - SOCKBUF_LOCK(&ic->ic_socket->so_rcv); - soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic); - SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); + sc = malloc(sizeof(*sc), M_ICL, M_ZERO | M_WAITOK); + sx_init(&sc->sc_lock, "icl"); + TAILQ_INIT(&sc->sc_modules); return (0); } -int -icl_conn_handoff(struct icl_conn *ic, int fd) -{ - struct file *fp; - struct socket *so; - cap_rights_t rights; - int error; - - ICL_CONN_LOCK_ASSERT_NOT(ic); - - /* - * Steal the socket from userland. - */ - error = fget(curthread, fd, - cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); - if (error != 0) - return (error); - if (fp->f_type != DTYPE_SOCKET) { - fdrop(fp, curthread); - return (EINVAL); - } - so = fp->f_data; - if (so->so_type != SOCK_STREAM) { - fdrop(fp, curthread); - return (EINVAL); - } - - ICL_CONN_LOCK(ic); - - if (ic->ic_socket != NULL) { - ICL_CONN_UNLOCK(ic); - fdrop(fp, curthread); - return (EBUSY); - } - - ic->ic_socket = fp->f_data; - fp->f_ops = &badfileops; - fp->f_data = NULL; - fdrop(fp, curthread); - ICL_CONN_UNLOCK(ic); - - error = icl_conn_start(ic); - - return (error); -} - -void -icl_conn_close(struct icl_conn *ic) -{ - struct icl_pdu *pdu; - - ICL_CONN_LOCK_ASSERT_NOT(ic); - - ICL_CONN_LOCK(ic); - if (ic->ic_socket == NULL) { - ICL_CONN_UNLOCK(ic); - return; - } - - /* - * Deregister socket upcalls. - */ - ICL_CONN_UNLOCK(ic); - SOCKBUF_LOCK(&ic->ic_socket->so_snd); - if (ic->ic_socket->so_snd.sb_upcall != NULL) - soupcall_clear(ic->ic_socket, SO_SND); - SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); - SOCKBUF_LOCK(&ic->ic_socket->so_rcv); - if (ic->ic_socket->so_rcv.sb_upcall != NULL) - soupcall_clear(ic->ic_socket, SO_RCV); - SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); - ICL_CONN_LOCK(ic); - - ic->ic_disconnecting = true; - - /* - * Wake up the threads, so they can properly terminate. - */ - while (ic->ic_receive_running || ic->ic_send_running) { - //ICL_DEBUG("waiting for send/receive threads to terminate"); - cv_signal(&ic->ic_receive_cv); - cv_signal(&ic->ic_send_cv); - cv_wait(&ic->ic_send_cv, ic->ic_lock); - } - //ICL_DEBUG("send/receive threads terminated"); - - ICL_CONN_UNLOCK(ic); - soclose(ic->ic_socket); - ICL_CONN_LOCK(ic); - ic->ic_socket = NULL; - - if (ic->ic_receive_pdu != NULL) { - //ICL_DEBUG("freeing partially received PDU"); - icl_pdu_free(ic->ic_receive_pdu); - ic->ic_receive_pdu = NULL; - } - - /* - * Remove any outstanding PDUs from the send queue. - */ - while (!STAILQ_EMPTY(&ic->ic_to_send)) { - pdu = STAILQ_FIRST(&ic->ic_to_send); - STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next); - icl_pdu_free(pdu); - } - - KASSERT(STAILQ_EMPTY(&ic->ic_to_send), - ("destroying session with non-empty send queue")); -#ifdef DIAGNOSTIC - KASSERT(ic->ic_outstanding_pdus == 0, - ("destroying session with %d outstanding PDUs", - ic->ic_outstanding_pdus)); -#endif - ICL_CONN_UNLOCK(ic); -} - -bool -icl_conn_connected(struct icl_conn *ic) -{ - ICL_CONN_LOCK_ASSERT_NOT(ic); - - ICL_CONN_LOCK(ic); - if (ic->ic_socket == NULL) { - ICL_CONN_UNLOCK(ic); - return (false); - } - if (ic->ic_socket->so_error != 0) { - ICL_CONN_UNLOCK(ic); - return (false); - } - ICL_CONN_UNLOCK(ic); - return (true); -} - -#ifdef ICL_KERNEL_PROXY -int -icl_conn_handoff_sock(struct icl_conn *ic, struct socket *so) -{ - int error; - - ICL_CONN_LOCK_ASSERT_NOT(ic); - - if (so->so_type != SOCK_STREAM) - return (EINVAL); - - ICL_CONN_LOCK(ic); - if (ic->ic_socket != NULL) { - ICL_CONN_UNLOCK(ic); - return (EBUSY); - } - ic->ic_socket = so; - ICL_CONN_UNLOCK(ic); - - error = icl_conn_start(ic); - - return (error); -} -#endif /* ICL_KERNEL_PROXY */ - static int icl_unload(void) { - if (icl_ncons != 0) - return (EBUSY); + sx_slock(&sc->sc_lock); + KASSERT(TAILQ_EMPTY(&sc->sc_modules), ("still have modules")); + sx_sunlock(&sc->sc_lock); - uma_zdestroy(icl_conn_zone); - uma_zdestroy(icl_pdu_zone); + sx_destroy(&sc->sc_lock); + free(sc, M_ICL); return (0); } -static void -icl_load(void) -{ - - icl_conn_zone = uma_zcreate("icl_conn", - sizeof(struct icl_conn), NULL, NULL, NULL, NULL, - UMA_ALIGN_PTR, 0); - icl_pdu_zone = uma_zcreate("icl_pdu", - sizeof(struct icl_pdu), NULL, NULL, NULL, NULL, - UMA_ALIGN_PTR, 0); - - refcount_init(&icl_ncons, 0); -} - static int icl_modevent(module_t mod, int what, void *arg) { switch (what) { case MOD_LOAD: - icl_load(); - return (0); + return (icl_load()); case MOD_UNLOAD: return (icl_unload()); default: return (EINVAL); } } moduledata_t icl_data = { "icl", icl_modevent, 0 }; DECLARE_MODULE(icl, icl_data, SI_SUB_DRIVERS, SI_ORDER_FIRST); MODULE_VERSION(icl, 1); diff --git a/sys/dev/iscsi/icl.h b/sys/dev/iscsi/icl.h index 50b231ef6996..f667ad66a899 100644 --- a/sys/dev/iscsi/icl.h +++ b/sys/dev/iscsi/icl.h @@ -1,158 +1,180 @@ /*- * Copyright (c) 2012 The FreeBSD Foundation * All rights reserved. * * This software was developed by Edward Tomasz Napierala under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef ICL_H #define ICL_H /* * iSCSI Common Layer. It's used by both the initiator and target to send * and receive iSCSI PDUs. */ +#include +#include +#include +#include + +SYSCTL_DECL(_kern_icl); + +extern int icl_debug; + +#define ICL_DEBUG(X, ...) \ + do { \ + if (icl_debug > 1) \ + printf("%s: " X "\n", __func__, ## __VA_ARGS__);\ + } while (0) + +#define ICL_WARN(X, ...) \ + do { \ + if (icl_debug > 0) { \ + printf("WARNING: %s: " X "\n", \ + __func__, ## __VA_ARGS__); \ + } \ + } while (0) + struct icl_conn; +struct ccb_scsiio; +union ctl_io; struct icl_pdu { STAILQ_ENTRY(icl_pdu) ip_next; struct icl_conn *ip_conn; struct iscsi_bhs *ip_bhs; struct mbuf *ip_bhs_mbuf; size_t ip_ahs_len; struct mbuf *ip_ahs_mbuf; size_t ip_data_len; struct mbuf *ip_data_mbuf; /* * User (initiator or provider) private fields. */ uint32_t ip_prv0; uint32_t ip_prv1; uint32_t ip_prv2; }; -struct icl_pdu *icl_pdu_new(struct icl_conn *ic, int flags); -size_t icl_pdu_data_segment_length(const struct icl_pdu *ip); -int icl_pdu_append_data(struct icl_pdu *ip, const void *addr, size_t len, int flags); -void icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len); -void icl_pdu_queue(struct icl_pdu *ip); -void icl_pdu_free(struct icl_pdu *ip); - #define ICL_CONN_STATE_INVALID 0 #define ICL_CONN_STATE_BHS 1 #define ICL_CONN_STATE_AHS 2 #define ICL_CONN_STATE_HEADER_DIGEST 3 #define ICL_CONN_STATE_DATA 4 #define ICL_CONN_STATE_DATA_DIGEST 5 #define ICL_MAX_DATA_SEGMENT_LENGTH (128 * 1024) struct icl_conn { + KOBJ_FIELDS; struct mtx *ic_lock; struct socket *ic_socket; #ifdef DIAGNOSTIC volatile u_int ic_outstanding_pdus; #endif STAILQ_HEAD(, icl_pdu) ic_to_send; bool ic_check_send_space; size_t ic_receive_len; int ic_receive_state; struct icl_pdu *ic_receive_pdu; struct cv ic_send_cv; struct cv ic_receive_cv; bool ic_header_crc32c; bool ic_data_crc32c; bool ic_send_running; bool ic_receive_running; size_t ic_max_data_segment_length; bool ic_disconnecting; bool ic_iser; const char *ic_name; void (*ic_receive)(struct icl_pdu *); void (*ic_error)(struct icl_conn *); /* * User (initiator or provider) private fields. */ void *ic_prv0; }; -struct icl_conn *icl_conn_new(const char *name, struct mtx *lock); -void icl_conn_free(struct icl_conn *ic); -int icl_conn_handoff(struct icl_conn *ic, int fd); -void icl_conn_close(struct icl_conn *ic); -bool icl_conn_connected(struct icl_conn *ic); +struct icl_conn *icl_new_conn(const char *offload, const char *name, + struct mtx *lock); +int icl_limits(const char *offload, size_t *limitp); + +int icl_register(const char *offload, int priority, + int (*limits)(size_t *), + struct icl_conn *(*new_conn)(const char *, struct mtx *)); +int icl_unregister(const char *offload); #ifdef ICL_KERNEL_PROXY struct sockaddr; struct icl_listen; struct icl_listen_sock { TAILQ_ENTRY(icl_listen_sock) ils_next; struct icl_listen *ils_listen; struct socket *ils_socket; bool ils_running; bool ils_disconnecting; int ils_id; }; struct icl_listen { TAILQ_HEAD(, icl_listen_sock) il_sockets; struct sx il_lock; void (*il_accept)(struct socket *, struct sockaddr *, int); }; /* * Initiator part. */ int icl_conn_connect(struct icl_conn *ic, bool rdma, int domain, int socktype, int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa); /* * Target part. */ struct icl_listen *icl_listen_new(void (*accept_cb)(struct socket *, struct sockaddr *, int)); void icl_listen_free(struct icl_listen *il); int icl_listen_add(struct icl_listen *il, bool rdma, int domain, int socktype, int protocol, struct sockaddr *sa, int portal_id); int icl_listen_remove(struct icl_listen *il, struct sockaddr *sa); /* * This one is not a public API; only to be used by icl_proxy.c. */ int icl_conn_handoff_sock(struct icl_conn *ic, struct socket *so); #endif /* ICL_KERNEL_PROXY */ #endif /* !ICL_H */ diff --git a/sys/dev/iscsi/icl_conn_if.m b/sys/dev/iscsi/icl_conn_if.m new file mode 100644 index 000000000000..d3ac57f0420d --- /dev/null +++ b/sys/dev/iscsi/icl_conn_if.m @@ -0,0 +1,87 @@ +#- +# Copyright (c) 2014 The FreeBSD Foundation +# All rights reserved. +# +# This software was developed by Edward Tomasz Napierala under sponsorship +# from the FreeBSD Foundation. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +#include + +INTERFACE icl_conn; + +METHOD size_t pdu_data_segment_length { + struct icl_conn *_ic; + const struct icl_pdu *_ip; +}; + +METHOD int pdu_append_data { + struct icl_conn *_ic; + struct icl_pdu *_ip; + const void *_addr; + size_t _len; + int _flags; +}; + +METHOD void pdu_get_data { + struct icl_conn *_ic; + struct icl_pdu *_ip; + size_t _off; + void *_addr; + size_t _len; +}; + +METHOD void pdu_queue { + struct icl_conn *_ic; + struct icl_pdu *_ip; +}; + +METHOD void pdu_free { + struct icl_conn *_ic; + struct icl_pdu *_ip; +}; + +METHOD struct icl_pdu * new_pdu { + struct icl_conn *_ic; + int _flags; +}; + +METHOD void free { + struct icl_conn *_ic; +}; + +METHOD int handoff { + struct icl_conn *_ic; + int _fd; +}; + +METHOD void close { + struct icl_conn *_ic; +}; + +METHOD bool connected { + struct icl_conn *_ic; +}; diff --git a/sys/dev/iscsi/icl.c b/sys/dev/iscsi/icl_soft.c similarity index 90% copy from sys/dev/iscsi/icl.c copy to sys/dev/iscsi/icl_soft.c index 7f5d01f5332e..9b6c695eb7c3 100644 --- a/sys/dev/iscsi/icl.c +++ b/sys/dev/iscsi/icl_soft.c @@ -1,1462 +1,1537 @@ /*- * Copyright (c) 2012 The FreeBSD Foundation * All rights reserved. * * This software was developed by Edward Tomasz Napierala under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * iSCSI Common Layer. It's used by both the initiator and target to send * and receive iSCSI PDUs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include -SYSCTL_NODE(_kern, OID_AUTO, icl, CTLFLAG_RD, 0, "iSCSI Common Layer"); -static int debug = 1; -SYSCTL_INT(_kern_icl, OID_AUTO, debug, CTLFLAG_RWTUN, - &debug, 0, "Enable debug messages"); static int coalesce = 1; SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN, &coalesce, 0, "Try to coalesce PDUs before sending"); static int partial_receive_len = 128 * 1024; SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, &partial_receive_len, 0, "Minimum read size for partially received " "data segment"); static int sendspace = 1048576; SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN, &sendspace, 0, "Default send socket buffer size"); static int recvspace = 1048576; SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN, &recvspace, 0, "Default receive socket buffer size"); -static uma_zone_t icl_conn_zone; +static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend"); static uma_zone_t icl_pdu_zone; static volatile u_int icl_ncons; -#define ICL_DEBUG(X, ...) \ - do { \ - if (debug > 1) \ - printf("%s: " X "\n", __func__, ## __VA_ARGS__);\ - } while (0) - -#define ICL_WARN(X, ...) \ - do { \ - if (debug > 0) { \ - printf("WARNING: %s: " X "\n", \ - __func__, ## __VA_ARGS__); \ - } \ - } while (0) - #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) STAILQ_HEAD(icl_pdu_stailq, icl_pdu); +static icl_conn_new_pdu_t icl_soft_conn_new_pdu; +static icl_conn_pdu_free_t icl_soft_conn_pdu_free; +static icl_conn_pdu_data_segment_length_t + icl_soft_conn_pdu_data_segment_length; +static icl_conn_pdu_append_data_t icl_soft_conn_pdu_append_data; +static icl_conn_pdu_get_data_t icl_soft_conn_pdu_get_data; +static icl_conn_pdu_queue_t icl_soft_conn_pdu_queue; +static icl_conn_handoff_t icl_soft_conn_handoff; +static icl_conn_free_t icl_soft_conn_free; +static icl_conn_close_t icl_soft_conn_close; +static icl_conn_connected_t icl_soft_conn_connected; + +static kobj_method_t icl_soft_methods[] = { + KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu), + KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free), + KOBJMETHOD(icl_conn_pdu_data_segment_length, + icl_soft_conn_pdu_data_segment_length), + KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data), + KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data), + KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue), + KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff), + KOBJMETHOD(icl_conn_free, icl_soft_conn_free), + KOBJMETHOD(icl_conn_close, icl_soft_conn_close), + KOBJMETHOD(icl_conn_connected, icl_soft_conn_connected), + { 0, 0 } +}; + +DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn)); + +static void icl_conn_close(struct icl_conn *ic); + static void icl_conn_fail(struct icl_conn *ic) { if (ic->ic_socket == NULL) return; /* * XXX */ ic->ic_socket->so_error = EDOOFUS; (ic->ic_error)(ic); } static struct mbuf * icl_conn_receive(struct icl_conn *ic, size_t len) { struct uio uio; struct socket *so; struct mbuf *m; int error, flags; so = ic->ic_socket; memset(&uio, 0, sizeof(uio)); uio.uio_resid = len; flags = MSG_DONTWAIT; error = soreceive(so, NULL, &uio, &m, NULL, &flags); if (error != 0) { ICL_DEBUG("soreceive error %d", error); return (NULL); } if (uio.uio_resid != 0) { m_freem(m); ICL_DEBUG("short read"); return (NULL); } return (m); } static struct icl_pdu * icl_pdu_new_empty(struct icl_conn *ic, int flags) { struct icl_pdu *ip; #ifdef DIAGNOSTIC refcount_acquire(&ic->ic_outstanding_pdus); #endif ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO); if (ip == NULL) { ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); #ifdef DIAGNOSTIC refcount_release(&ic->ic_outstanding_pdus); #endif return (NULL); } ip->ip_conn = ic; return (ip); } -void +static void icl_pdu_free(struct icl_pdu *ip) { struct icl_conn *ic; ic = ip->ip_conn; m_freem(ip->ip_bhs_mbuf); m_freem(ip->ip_ahs_mbuf); m_freem(ip->ip_data_mbuf); uma_zfree(icl_pdu_zone, ip); #ifdef DIAGNOSTIC refcount_release(&ic->ic_outstanding_pdus); #endif } +void +icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) +{ + icl_pdu_free(ip); +} + /* * Allocate icl_pdu with empty BHS to fill up by the caller. */ struct icl_pdu * -icl_pdu_new(struct icl_conn *ic, int flags) +icl_soft_conn_new_pdu(struct icl_conn *ic, int flags) { struct icl_pdu *ip; ip = icl_pdu_new_empty(ic, flags); if (ip == NULL) return (NULL); ip->ip_bhs_mbuf = m_getm2(NULL, sizeof(struct iscsi_bhs), flags, MT_DATA, M_PKTHDR); if (ip->ip_bhs_mbuf == NULL) { ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); icl_pdu_free(ip); return (NULL); } ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); return (ip); } static int icl_pdu_ahs_length(const struct icl_pdu *request) { return (request->ip_bhs->bhs_total_ahs_len * 4); } -size_t +static size_t icl_pdu_data_segment_length(const struct icl_pdu *request) { uint32_t len = 0; len += request->ip_bhs->bhs_data_segment_len[0]; len <<= 8; len += request->ip_bhs->bhs_data_segment_len[1]; len <<= 8; len += request->ip_bhs->bhs_data_segment_len[2]; return (len); } +size_t +icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic, + const struct icl_pdu *request) +{ + + return (icl_pdu_data_segment_length(request)); +} + static void icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) { response->ip_bhs->bhs_data_segment_len[2] = len; response->ip_bhs->bhs_data_segment_len[1] = len >> 8; response->ip_bhs->bhs_data_segment_len[0] = len >> 16; } static size_t icl_pdu_padding(const struct icl_pdu *ip) { if ((ip->ip_data_len % 4) != 0) return (4 - (ip->ip_data_len % 4)); return (0); } static size_t icl_pdu_size(const struct icl_pdu *response) { size_t len; KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); len = sizeof(struct iscsi_bhs) + response->ip_data_len + icl_pdu_padding(response); if (response->ip_conn->ic_header_crc32c) len += ISCSI_HEADER_DIGEST_SIZE; if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) len += ISCSI_DATA_DIGEST_SIZE; return (len); } static int icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep) { struct mbuf *m; m = icl_conn_receive(request->ip_conn, sizeof(struct iscsi_bhs)); if (m == NULL) { ICL_DEBUG("failed to receive BHS"); return (-1); } request->ip_bhs_mbuf = m_pullup(m, sizeof(struct iscsi_bhs)); if (request->ip_bhs_mbuf == NULL) { ICL_WARN("m_pullup failed"); return (-1); } request->ip_bhs = mtod(request->ip_bhs_mbuf, struct iscsi_bhs *); /* * XXX: For architectures with strict alignment requirements * we may need to allocate ip_bhs and copy the data into it. * For some reason, though, not doing this doesn't seem * to cause problems; tested on sparc64. */ *availablep -= sizeof(struct iscsi_bhs); return (0); } static int icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep) { request->ip_ahs_len = icl_pdu_ahs_length(request); if (request->ip_ahs_len == 0) return (0); request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn, request->ip_ahs_len); if (request->ip_ahs_mbuf == NULL) { ICL_DEBUG("failed to receive AHS"); return (-1); } *availablep -= request->ip_ahs_len; return (0); } static uint32_t icl_mbuf_to_crc32c(const struct mbuf *m0) { uint32_t digest = 0xffffffff; const struct mbuf *m; for (m = m0; m != NULL; m = m->m_next) digest = calculate_crc32c(digest, mtod(m, const void *), m->m_len); digest = digest ^ 0xffffffff; return (digest); } static int icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep) { struct mbuf *m; uint32_t received_digest, valid_digest; if (request->ip_conn->ic_header_crc32c == false) return (0); m = icl_conn_receive(request->ip_conn, ISCSI_HEADER_DIGEST_SIZE); if (m == NULL) { ICL_DEBUG("failed to receive header digest"); return (-1); } CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); m_copydata(m, 0, ISCSI_HEADER_DIGEST_SIZE, (void *)&received_digest); m_freem(m); *availablep -= ISCSI_HEADER_DIGEST_SIZE; /* * XXX: Handle AHS. */ valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); if (received_digest != valid_digest) { ICL_WARN("header digest check failed; got 0x%x, " "should be 0x%x", received_digest, valid_digest); return (-1); } return (0); } /* * Return the number of bytes that should be waiting in the receive socket * before icl_pdu_receive_data_segment() gets called. */ static size_t icl_pdu_data_segment_receive_len(const struct icl_pdu *request) { size_t len; len = icl_pdu_data_segment_length(request); if (len == 0) return (0); /* * Account for the parts of data segment already read from * the socket buffer. */ KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); len -= request->ip_data_len; /* * Don't always wait for the full data segment to be delivered * to the socket; this might badly affect performance due to * TCP window scaling. */ if (len > partial_receive_len) { #if 0 ICL_DEBUG("need %zd bytes of data, limiting to %zd", len, partial_receive_len)); #endif len = partial_receive_len; return (len); } /* * Account for padding. Note that due to the way code is written, * the icl_pdu_receive_data_segment() must always receive padding * along with the last part of data segment, because it would be * impossible to tell whether we've already received the full data * segment including padding, or without it. */ if ((len % 4) != 0) len += 4 - (len % 4); #if 0 ICL_DEBUG("need %zd bytes of data", len)); #endif return (len); } static int icl_pdu_receive_data_segment(struct icl_pdu *request, size_t *availablep, bool *more_neededp) { struct icl_conn *ic; size_t len, padding = 0; struct mbuf *m; ic = request->ip_conn; *more_neededp = false; ic->ic_receive_len = 0; len = icl_pdu_data_segment_length(request); if (len == 0) return (0); if ((len % 4) != 0) padding = 4 - (len % 4); /* * Account for already received parts of data segment. */ KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); len -= request->ip_data_len; if (len + padding > *availablep) { /* * Not enough data in the socket buffer. Receive as much * as we can. Don't receive padding, since, obviously, it's * not the end of data segment yet. */ #if 0 ICL_DEBUG("limited from %zd to %zd", len + padding, *availablep - padding)); #endif len = *availablep - padding; *more_neededp = true; padding = 0; } /* * Must not try to receive padding without at least one byte * of actual data segment. */ if (len > 0) { m = icl_conn_receive(request->ip_conn, len + padding); if (m == NULL) { ICL_DEBUG("failed to receive data segment"); return (-1); } if (request->ip_data_mbuf == NULL) request->ip_data_mbuf = m; else m_cat(request->ip_data_mbuf, m); request->ip_data_len += len; *availablep -= len + padding; } else ICL_DEBUG("len 0"); if (*more_neededp) ic->ic_receive_len = icl_pdu_data_segment_receive_len(request); return (0); } static int icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep) { struct mbuf *m; uint32_t received_digest, valid_digest; if (request->ip_conn->ic_data_crc32c == false) return (0); if (request->ip_data_len == 0) return (0); m = icl_conn_receive(request->ip_conn, ISCSI_DATA_DIGEST_SIZE); if (m == NULL) { ICL_DEBUG("failed to receive data digest"); return (-1); } CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); m_copydata(m, 0, ISCSI_DATA_DIGEST_SIZE, (void *)&received_digest); m_freem(m); *availablep -= ISCSI_DATA_DIGEST_SIZE; /* * Note that ip_data_mbuf also contains padding; since digest * calculation is supposed to include that, we iterate over * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. */ valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); if (received_digest != valid_digest) { ICL_WARN("data digest check failed; got 0x%x, " "should be 0x%x", received_digest, valid_digest); return (-1); } return (0); } /* * Somewhat contrary to the name, this attempts to receive only one * "part" of PDU at a time; call it repeatedly until it returns non-NULL. */ static struct icl_pdu * icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep) { struct icl_pdu *request; struct socket *so; size_t len; int error; bool more_needed; so = ic->ic_socket; if (ic->ic_receive_state == ICL_CONN_STATE_BHS) { KASSERT(ic->ic_receive_pdu == NULL, ("ic->ic_receive_pdu != NULL")); request = icl_pdu_new_empty(ic, M_NOWAIT); if (request == NULL) { ICL_DEBUG("failed to allocate PDU; " "dropping connection"); icl_conn_fail(ic); return (NULL); } ic->ic_receive_pdu = request; } else { KASSERT(ic->ic_receive_pdu != NULL, ("ic->ic_receive_pdu == NULL")); request = ic->ic_receive_pdu; } if (*availablep < ic->ic_receive_len) { #if 0 ICL_DEBUG("not enough data; need %zd, " "have %zd", ic->ic_receive_len, *availablep); #endif return (NULL); } switch (ic->ic_receive_state) { case ICL_CONN_STATE_BHS: //ICL_DEBUG("receiving BHS"); error = icl_pdu_receive_bhs(request, availablep); if (error != 0) { ICL_DEBUG("failed to receive BHS; " "dropping connection"); break; } /* * We don't enforce any limit for AHS length; * its length is stored in 8 bit field. */ len = icl_pdu_data_segment_length(request); if (len > ic->ic_max_data_segment_length) { ICL_WARN("received data segment " "length %zd is larger than negotiated " "MaxDataSegmentLength %zd; " "dropping connection", len, ic->ic_max_data_segment_length); error = EINVAL; break; } ic->ic_receive_state = ICL_CONN_STATE_AHS; ic->ic_receive_len = icl_pdu_ahs_length(request); break; case ICL_CONN_STATE_AHS: //ICL_DEBUG("receiving AHS"); error = icl_pdu_receive_ahs(request, availablep); if (error != 0) { ICL_DEBUG("failed to receive AHS; " "dropping connection"); break; } ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST; if (ic->ic_header_crc32c == false) ic->ic_receive_len = 0; else ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE; break; case ICL_CONN_STATE_HEADER_DIGEST: //ICL_DEBUG("receiving header digest"); error = icl_pdu_check_header_digest(request, availablep); if (error != 0) { ICL_DEBUG("header digest failed; " "dropping connection"); break; } ic->ic_receive_state = ICL_CONN_STATE_DATA; ic->ic_receive_len = icl_pdu_data_segment_receive_len(request); break; case ICL_CONN_STATE_DATA: //ICL_DEBUG("receiving data segment"); error = icl_pdu_receive_data_segment(request, availablep, &more_needed); if (error != 0) { ICL_DEBUG("failed to receive data segment;" "dropping connection"); break; } if (more_needed) break; ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST; if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) ic->ic_receive_len = 0; else ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE; break; case ICL_CONN_STATE_DATA_DIGEST: //ICL_DEBUG("receiving data digest"); error = icl_pdu_check_data_digest(request, availablep); if (error != 0) { ICL_DEBUG("data digest failed; " "dropping connection"); break; } /* * We've received complete PDU; reset the receive state machine * and return the PDU. */ ic->ic_receive_state = ICL_CONN_STATE_BHS; ic->ic_receive_len = sizeof(struct iscsi_bhs); ic->ic_receive_pdu = NULL; return (request); default: panic("invalid ic_receive_state %d\n", ic->ic_receive_state); } if (error != 0) { /* * Don't free the PDU; it's pointed to by ic->ic_receive_pdu * and will get freed in icl_conn_close(). */ icl_conn_fail(ic); } return (NULL); } static void icl_conn_receive_pdus(struct icl_conn *ic, size_t available) { struct icl_pdu *response; struct socket *so; so = ic->ic_socket; /* * This can never happen; we're careful to only mess with ic->ic_socket * pointer when the send/receive threads are not running. */ KASSERT(so != NULL, ("NULL socket")); for (;;) { if (ic->ic_disconnecting) return; if (so->so_error != 0) { ICL_DEBUG("connection error %d; " "dropping connection", so->so_error); icl_conn_fail(ic); return; } /* * Loop until we have a complete PDU or there is not enough * data in the socket buffer. */ if (available < ic->ic_receive_len) { #if 0 ICL_DEBUG("not enough data; have %zd, " "need %zd", available, ic->ic_receive_len); #endif return; } response = icl_conn_receive_pdu(ic, &available); if (response == NULL) continue; if (response->ip_ahs_len > 0) { ICL_WARN("received PDU with unsupported " "AHS; opcode 0x%x; dropping connection", response->ip_bhs->bhs_opcode); icl_pdu_free(response); icl_conn_fail(ic); return; } (ic->ic_receive)(response); } } static void icl_receive_thread(void *arg) { struct icl_conn *ic; size_t available; struct socket *so; ic = arg; so = ic->ic_socket; ICL_CONN_LOCK(ic); ic->ic_receive_running = true; ICL_CONN_UNLOCK(ic); for (;;) { if (ic->ic_disconnecting) { //ICL_DEBUG("terminating"); break; } /* * Set the low watermark, to be checked by * soreadable() in icl_soupcall_receive() * to avoid unneccessary wakeups until there * is enough data received to read the PDU. */ SOCKBUF_LOCK(&so->so_rcv); available = sbavail(&so->so_rcv); if (available < ic->ic_receive_len) { so->so_rcv.sb_lowat = ic->ic_receive_len; cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx); } else so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; SOCKBUF_UNLOCK(&so->so_rcv); icl_conn_receive_pdus(ic, available); } ICL_CONN_LOCK(ic); ic->ic_receive_running = false; cv_signal(&ic->ic_send_cv); ICL_CONN_UNLOCK(ic); kthread_exit(); } static int icl_soupcall_receive(struct socket *so, void *arg, int waitflag) { struct icl_conn *ic; if (!soreadable(so)) return (SU_OK); ic = arg; cv_signal(&ic->ic_receive_cv); return (SU_OK); } static int icl_pdu_finalize(struct icl_pdu *request) { size_t padding, pdu_len; uint32_t digest, zero = 0; int ok; struct icl_conn *ic; ic = request->ip_conn; icl_pdu_set_data_segment_length(request, request->ip_data_len); pdu_len = icl_pdu_size(request); if (ic->ic_header_crc32c) { digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); ok = m_append(request->ip_bhs_mbuf, sizeof(digest), (void *)&digest); if (ok != 1) { ICL_WARN("failed to append header digest"); return (1); } } if (request->ip_data_len != 0) { padding = icl_pdu_padding(request); if (padding > 0) { ok = m_append(request->ip_data_mbuf, padding, (void *)&zero); if (ok != 1) { ICL_WARN("failed to append padding"); return (1); } } if (ic->ic_data_crc32c) { digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); ok = m_append(request->ip_data_mbuf, sizeof(digest), (void *)&digest); if (ok != 1) { ICL_WARN("failed to append data digest"); return (1); } } m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); request->ip_data_mbuf = NULL; } request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; return (0); } static void icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue) { struct icl_pdu *request, *request2; struct socket *so; size_t available, size, size2; int coalesced, error; ICL_CONN_LOCK_ASSERT_NOT(ic); so = ic->ic_socket; SOCKBUF_LOCK(&so->so_snd); /* * Check how much space do we have for transmit. We can't just * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, * as it always frees the mbuf chain passed to it, even in case * of error. */ available = sbspace(&so->so_snd); /* * Notify the socket upcall that we don't need wakeups * for the time being. */ so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; SOCKBUF_UNLOCK(&so->so_snd); while (!STAILQ_EMPTY(queue)) { request = STAILQ_FIRST(queue); size = icl_pdu_size(request); if (available < size) { /* * Set the low watermark, to be checked by * sowriteable() in icl_soupcall_send() * to avoid unneccessary wakeups until there * is enough space for the PDU to fit. */ SOCKBUF_LOCK(&so->so_snd); available = sbspace(&so->so_snd); if (available < size) { #if 1 ICL_DEBUG("no space to send; " "have %zd, need %zd", available, size); #endif so->so_snd.sb_lowat = size; SOCKBUF_UNLOCK(&so->so_snd); return; } SOCKBUF_UNLOCK(&so->so_snd); } STAILQ_REMOVE_HEAD(queue, ip_next); error = icl_pdu_finalize(request); if (error != 0) { ICL_DEBUG("failed to finalize PDU; " "dropping connection"); icl_conn_fail(ic); icl_pdu_free(request); return; } if (coalesce) { coalesced = 1; for (;;) { request2 = STAILQ_FIRST(queue); if (request2 == NULL) break; size2 = icl_pdu_size(request2); if (available < size + size2) break; STAILQ_REMOVE_HEAD(queue, ip_next); error = icl_pdu_finalize(request2); if (error != 0) { ICL_DEBUG("failed to finalize PDU; " "dropping connection"); icl_conn_fail(ic); icl_pdu_free(request); icl_pdu_free(request2); return; } m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf); request2->ip_bhs_mbuf = NULL; request->ip_bhs_mbuf->m_pkthdr.len += size2; size += size2; STAILQ_REMOVE_AFTER(queue, request, ip_next); icl_pdu_free(request2); coalesced++; } #if 0 if (coalesced > 1) { ICL_DEBUG("coalesced %d PDUs into %zd bytes", coalesced, size); } #endif } available -= size; error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, NULL, MSG_DONTWAIT, curthread); request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ if (error != 0) { ICL_DEBUG("failed to send PDU, error %d; " "dropping connection", error); icl_conn_fail(ic); icl_pdu_free(request); return; } icl_pdu_free(request); } } static void icl_send_thread(void *arg) { struct icl_conn *ic; struct icl_pdu_stailq queue; ic = arg; STAILQ_INIT(&queue); ICL_CONN_LOCK(ic); ic->ic_send_running = true; for (;;) { for (;;) { /* * If the local queue is empty, populate it from * the main one. This way the icl_conn_send_pdus() * can go through all the queued PDUs without holding * any locks. */ if (STAILQ_EMPTY(&queue)) STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu); ic->ic_check_send_space = false; ICL_CONN_UNLOCK(ic); icl_conn_send_pdus(ic, &queue); ICL_CONN_LOCK(ic); /* * The icl_soupcall_send() was called since the last * call to sbspace(); go around; */ if (ic->ic_check_send_space) continue; /* * Local queue is empty, but we still have PDUs * in the main one; go around. */ if (STAILQ_EMPTY(&queue) && !STAILQ_EMPTY(&ic->ic_to_send)) continue; /* * There might be some stuff in the local queue, * which didn't get sent due to not having enough send * space. Wait for socket upcall. */ break; } if (ic->ic_disconnecting) { //ICL_DEBUG("terminating"); break; } cv_wait(&ic->ic_send_cv, ic->ic_lock); } /* * We're exiting; move PDUs back to the main queue, so they can * get freed properly. At this point ordering doesn't matter. */ STAILQ_CONCAT(&ic->ic_to_send, &queue); ic->ic_send_running = false; cv_signal(&ic->ic_send_cv); ICL_CONN_UNLOCK(ic); kthread_exit(); } static int icl_soupcall_send(struct socket *so, void *arg, int waitflag) { struct icl_conn *ic; if (!sowriteable(so)) return (SU_OK); ic = arg; ICL_CONN_LOCK(ic); ic->ic_check_send_space = true; ICL_CONN_UNLOCK(ic); cv_signal(&ic->ic_send_cv); return (SU_OK); } -int +static int icl_pdu_append_data(struct icl_pdu *request, const void *addr, size_t len, int flags) { struct mbuf *mb, *newmb; size_t copylen, off = 0; KASSERT(len > 0, ("len == 0")); newmb = m_getm2(NULL, len, flags, MT_DATA, M_PKTHDR); if (newmb == NULL) { ICL_WARN("failed to allocate mbuf for %zd bytes", len); return (ENOMEM); } for (mb = newmb; mb != NULL; mb = mb->m_next) { copylen = min(M_TRAILINGSPACE(mb), len - off); memcpy(mtod(mb, char *), (const char *)addr + off, copylen); mb->m_len = copylen; off += copylen; } KASSERT(off == len, ("%s: off != len", __func__)); if (request->ip_data_mbuf == NULL) { request->ip_data_mbuf = newmb; request->ip_data_len = len; } else { m_cat(request->ip_data_mbuf, newmb); request->ip_data_len += len; } return (0); } -void +int +icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, + const void *addr, size_t len, int flags) +{ + + return (icl_pdu_append_data(request, addr, len, flags)); +} + +static void icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len) { m_copydata(ip->ip_data_mbuf, off, len, addr); } void +icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, + size_t off, void *addr, size_t len) +{ + + return (icl_pdu_get_data(ip, off, addr, len)); +} + +static void icl_pdu_queue(struct icl_pdu *ip) { struct icl_conn *ic; ic = ip->ip_conn; ICL_CONN_LOCK_ASSERT(ic); if (ic->ic_disconnecting || ic->ic_socket == NULL) { ICL_DEBUG("icl_pdu_queue on closed connection"); icl_pdu_free(ip); return; } if (!STAILQ_EMPTY(&ic->ic_to_send)) { STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); /* * If the queue is not empty, someone else had already * signaled the send thread; no need to do that again, * just return. */ return; } STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); cv_signal(&ic->ic_send_cv); } -struct icl_conn * -icl_conn_new(const char *name, struct mtx *lock) +void +icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) +{ + + icl_pdu_queue(ip); +} + +static struct icl_conn * +icl_soft_new_conn(const char *name, struct mtx *lock) { struct icl_conn *ic; refcount_acquire(&icl_ncons); - ic = uma_zalloc(icl_conn_zone, M_WAITOK | M_ZERO); + ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO); STAILQ_INIT(&ic->ic_to_send); ic->ic_lock = lock; cv_init(&ic->ic_send_cv, "icl_tx"); cv_init(&ic->ic_receive_cv, "icl_rx"); #ifdef DIAGNOSTIC refcount_init(&ic->ic_outstanding_pdus, 0); #endif ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH; ic->ic_name = name; return (ic); } void -icl_conn_free(struct icl_conn *ic) +icl_soft_conn_free(struct icl_conn *ic) { cv_destroy(&ic->ic_send_cv); cv_destroy(&ic->ic_receive_cv); - uma_zfree(icl_conn_zone, ic); + kobj_delete((struct kobj *)ic, M_ICL_SOFT); refcount_release(&icl_ncons); } static int icl_conn_start(struct icl_conn *ic) { size_t minspace; struct sockopt opt; int error, one = 1; ICL_CONN_LOCK(ic); /* * XXX: Ugly hack. */ if (ic->ic_socket == NULL) { ICL_CONN_UNLOCK(ic); return (EINVAL); } ic->ic_receive_state = ICL_CONN_STATE_BHS; ic->ic_receive_len = sizeof(struct iscsi_bhs); ic->ic_disconnecting = false; ICL_CONN_UNLOCK(ic); /* * For sendspace, this is required because the current code cannot * send a PDU in pieces; thus, the minimum buffer size is equal * to the maximum PDU size. "+4" is to account for possible padding. * * What we should actually do here is to use autoscaling, but set * some minimal buffer size to "minspace". I don't know a way to do * that, though. */ minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; if (sendspace < minspace) { ICL_WARN("kern.icl.sendspace too low; must be at least %zd", minspace); sendspace = minspace; } if (recvspace < minspace) { ICL_WARN("kern.icl.recvspace too low; must be at least %zd", minspace); recvspace = minspace; } error = soreserve(ic->ic_socket, sendspace, recvspace); if (error != 0) { ICL_WARN("soreserve failed with error %d", error); icl_conn_close(ic); return (error); } ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; /* * Disable Nagle. */ bzero(&opt, sizeof(opt)); opt.sopt_dir = SOPT_SET; opt.sopt_level = IPPROTO_TCP; opt.sopt_name = TCP_NODELAY; opt.sopt_val = &one; opt.sopt_valsize = sizeof(one); error = sosetopt(ic->ic_socket, &opt); if (error != 0) { ICL_WARN("disabling TCP_NODELAY failed with error %d", error); icl_conn_close(ic); return (error); } /* * Start threads. */ error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", ic->ic_name); if (error != 0) { ICL_WARN("kthread_add(9) failed with error %d", error); icl_conn_close(ic); return (error); } error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", ic->ic_name); if (error != 0) { ICL_WARN("kthread_add(9) failed with error %d", error); icl_conn_close(ic); return (error); } /* * Register socket upcall, to get notified about incoming PDUs * and free space to send outgoing ones. */ SOCKBUF_LOCK(&ic->ic_socket->so_snd); soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic); SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); SOCKBUF_LOCK(&ic->ic_socket->so_rcv); soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic); SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); return (0); } int -icl_conn_handoff(struct icl_conn *ic, int fd) +icl_soft_conn_handoff(struct icl_conn *ic, int fd) { struct file *fp; struct socket *so; cap_rights_t rights; int error; ICL_CONN_LOCK_ASSERT_NOT(ic); /* * Steal the socket from userland. */ error = fget(curthread, fd, cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); if (error != 0) return (error); if (fp->f_type != DTYPE_SOCKET) { fdrop(fp, curthread); return (EINVAL); } so = fp->f_data; if (so->so_type != SOCK_STREAM) { fdrop(fp, curthread); return (EINVAL); } ICL_CONN_LOCK(ic); if (ic->ic_socket != NULL) { ICL_CONN_UNLOCK(ic); fdrop(fp, curthread); return (EBUSY); } ic->ic_socket = fp->f_data; fp->f_ops = &badfileops; fp->f_data = NULL; fdrop(fp, curthread); ICL_CONN_UNLOCK(ic); error = icl_conn_start(ic); return (error); } void icl_conn_close(struct icl_conn *ic) { struct icl_pdu *pdu; ICL_CONN_LOCK_ASSERT_NOT(ic); ICL_CONN_LOCK(ic); if (ic->ic_socket == NULL) { ICL_CONN_UNLOCK(ic); return; } /* * Deregister socket upcalls. */ ICL_CONN_UNLOCK(ic); SOCKBUF_LOCK(&ic->ic_socket->so_snd); if (ic->ic_socket->so_snd.sb_upcall != NULL) soupcall_clear(ic->ic_socket, SO_SND); SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); SOCKBUF_LOCK(&ic->ic_socket->so_rcv); if (ic->ic_socket->so_rcv.sb_upcall != NULL) soupcall_clear(ic->ic_socket, SO_RCV); SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); ICL_CONN_LOCK(ic); ic->ic_disconnecting = true; /* * Wake up the threads, so they can properly terminate. */ while (ic->ic_receive_running || ic->ic_send_running) { //ICL_DEBUG("waiting for send/receive threads to terminate"); cv_signal(&ic->ic_receive_cv); cv_signal(&ic->ic_send_cv); cv_wait(&ic->ic_send_cv, ic->ic_lock); } //ICL_DEBUG("send/receive threads terminated"); ICL_CONN_UNLOCK(ic); soclose(ic->ic_socket); ICL_CONN_LOCK(ic); ic->ic_socket = NULL; if (ic->ic_receive_pdu != NULL) { //ICL_DEBUG("freeing partially received PDU"); icl_pdu_free(ic->ic_receive_pdu); ic->ic_receive_pdu = NULL; } /* * Remove any outstanding PDUs from the send queue. */ while (!STAILQ_EMPTY(&ic->ic_to_send)) { pdu = STAILQ_FIRST(&ic->ic_to_send); STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next); icl_pdu_free(pdu); } KASSERT(STAILQ_EMPTY(&ic->ic_to_send), ("destroying session with non-empty send queue")); #ifdef DIAGNOSTIC KASSERT(ic->ic_outstanding_pdus == 0, ("destroying session with %d outstanding PDUs", ic->ic_outstanding_pdus)); #endif ICL_CONN_UNLOCK(ic); } +void +icl_soft_conn_close(struct icl_conn *ic) +{ + + icl_conn_close(ic); +} + bool -icl_conn_connected(struct icl_conn *ic) +icl_soft_conn_connected(struct icl_conn *ic) { ICL_CONN_LOCK_ASSERT_NOT(ic); ICL_CONN_LOCK(ic); if (ic->ic_socket == NULL) { ICL_CONN_UNLOCK(ic); return (false); } if (ic->ic_socket->so_error != 0) { ICL_CONN_UNLOCK(ic); return (false); } ICL_CONN_UNLOCK(ic); return (true); } +static int +icl_soft_limits(size_t *limitp) +{ + + *limitp = 128 * 1024; + + return (0); +} + #ifdef ICL_KERNEL_PROXY int icl_conn_handoff_sock(struct icl_conn *ic, struct socket *so) { int error; ICL_CONN_LOCK_ASSERT_NOT(ic); if (so->so_type != SOCK_STREAM) return (EINVAL); ICL_CONN_LOCK(ic); if (ic->ic_socket != NULL) { ICL_CONN_UNLOCK(ic); return (EBUSY); } ic->ic_socket = so; ICL_CONN_UNLOCK(ic); error = icl_conn_start(ic); return (error); } #endif /* ICL_KERNEL_PROXY */ static int -icl_unload(void) +icl_soft_load(void) { + int error; - if (icl_ncons != 0) - return (EBUSY); + icl_pdu_zone = uma_zcreate("icl_pdu", + sizeof(struct icl_pdu), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + refcount_init(&icl_ncons, 0); - uma_zdestroy(icl_conn_zone); - uma_zdestroy(icl_pdu_zone); + /* + * The reason we call this "none" is that to the user, + * it's known as "offload driver"; "offload driver: soft" + * doesn't make much sense. + */ + error = icl_register("none", 0, icl_soft_limits, icl_soft_new_conn); + KASSERT(error == 0, ("failed to register")); - return (0); + return (error); } -static void -icl_load(void) +static int +icl_soft_unload(void) { - icl_conn_zone = uma_zcreate("icl_conn", - sizeof(struct icl_conn), NULL, NULL, NULL, NULL, - UMA_ALIGN_PTR, 0); - icl_pdu_zone = uma_zcreate("icl_pdu", - sizeof(struct icl_pdu), NULL, NULL, NULL, NULL, - UMA_ALIGN_PTR, 0); + if (icl_ncons != 0) + return (EBUSY); - refcount_init(&icl_ncons, 0); + icl_unregister("none"); + + uma_zdestroy(icl_pdu_zone); + + return (0); } static int -icl_modevent(module_t mod, int what, void *arg) +icl_soft_modevent(module_t mod, int what, void *arg) { switch (what) { case MOD_LOAD: - icl_load(); - return (0); + return (icl_soft_load()); case MOD_UNLOAD: - return (icl_unload()); + return (icl_soft_unload()); default: return (EINVAL); } } -moduledata_t icl_data = { - "icl", - icl_modevent, +moduledata_t icl_soft_data = { + "icl_soft", + icl_soft_modevent, 0 }; -DECLARE_MODULE(icl, icl_data, SI_SUB_DRIVERS, SI_ORDER_FIRST); +DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); +MODULE_DEPEND(icl_soft, icl, 1, 1, 1); MODULE_VERSION(icl, 1); diff --git a/sys/dev/iscsi/icl_wrappers.h b/sys/dev/iscsi/icl_wrappers.h new file mode 100644 index 000000000000..374213970462 --- /dev/null +++ b/sys/dev/iscsi/icl_wrappers.h @@ -0,0 +1,116 @@ +/*- + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Edward Tomasz Napierala under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* + * This file is used to provide the initiator and target with a prettier + * interface. It must not be included by ICL modules, such as icl_soft.c. + */ + +#ifndef ICL_WRAPPERS_H +#define ICL_WRAPPERS_H + +#include +#include + +#include +#include + +static inline struct icl_pdu * +icl_pdu_new(struct icl_conn *ic, int flags) +{ + + return (ICL_CONN_NEW_PDU(ic, flags)); +} + +static inline size_t +icl_pdu_data_segment_length(const struct icl_pdu *ip) +{ + + return (ICL_CONN_PDU_DATA_SEGMENT_LENGTH(ip->ip_conn, ip)); +} + +static inline int +icl_pdu_append_data(struct icl_pdu *ip, const void *addr, size_t len, int flags) +{ + + return (ICL_CONN_PDU_APPEND_DATA(ip->ip_conn, ip, addr, len, flags)); +} + +static inline void +icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len) +{ + + ICL_CONN_PDU_GET_DATA(ip->ip_conn, ip, off, addr, len); +} + +static inline void +icl_pdu_queue(struct icl_pdu *ip) +{ + + ICL_CONN_PDU_QUEUE(ip->ip_conn, ip); +} + +static inline void +icl_pdu_free(struct icl_pdu *ip) +{ + + ICL_CONN_PDU_FREE(ip->ip_conn, ip); +} + +static inline void +icl_conn_free(struct icl_conn *ic) +{ + + ICL_CONN_FREE(ic); +} + +static inline int +icl_conn_handoff(struct icl_conn *ic, int fd) +{ + + return (ICL_CONN_HANDOFF(ic, fd)); +} + +static inline void +icl_conn_close(struct icl_conn *ic) +{ + + ICL_CONN_CLOSE(ic); +} + +static inline bool +icl_conn_connected(struct icl_conn *ic) +{ + + return (ICL_CONN_CONNECTED(ic)); +} + +#endif /* !ICL_WRAPPERS_H */ diff --git a/sys/dev/iscsi/iscsi.c b/sys/dev/iscsi/iscsi.c index 0d762aadb837..8f07620a697d 100644 --- a/sys/dev/iscsi/iscsi.c +++ b/sys/dev/iscsi/iscsi.c @@ -1,2401 +1,2402 @@ /*- * Copyright (c) 2012 The FreeBSD Foundation * All rights reserved. * * This software was developed by Edward Tomasz Napierala under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #ifdef ICL_KERNEL_PROXY #include #endif #ifdef ICL_KERNEL_PROXY FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY"); #endif /* * XXX: This is global so the iscsi_unload() can access it. * Think about how to do this properly. */ static struct iscsi_softc *sc; SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator"); static int debug = 1; SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN, &debug, 0, "Enable debug messages"); static int ping_timeout = 5; SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout, 0, "Timeout for ping (NOP-Out) requests, in seconds"); static int iscsid_timeout = 60; SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout, 0, "Time to wait for iscsid(8) to handle reconnection, in seconds"); static int login_timeout = 60; SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout, 0, "Time to wait for iscsid(8) to finish Login Phase, in seconds"); static int maxtags = 255; SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags, 0, "Max number of IO requests queued"); static int fail_on_disconnection = 0; SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN, &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure"); static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator"); static uma_zone_t iscsi_outstanding_zone; #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0) #define PDU_SESSION(X) (CONN_SESSION(X->ip_conn)) #define ISCSI_DEBUG(X, ...) \ do { \ if (debug > 1) \ printf("%s: " X "\n", __func__, ## __VA_ARGS__);\ } while (0) #define ISCSI_WARN(X, ...) \ do { \ if (debug > 0) { \ printf("WARNING: %s: " X "\n", \ __func__, ## __VA_ARGS__); \ } \ } while (0) #define ISCSI_SESSION_DEBUG(S, X, ...) \ do { \ if (debug > 1) { \ printf("%s: %s (%s): " X "\n", \ __func__, S->is_conf.isc_target_addr, \ S->is_conf.isc_target, ## __VA_ARGS__); \ } \ } while (0) #define ISCSI_SESSION_WARN(S, X, ...) \ do { \ if (debug > 0) { \ printf("WARNING: %s (%s): " X "\n", \ S->is_conf.isc_target_addr, \ S->is_conf.isc_target, ## __VA_ARGS__); \ } \ } while (0) #define ISCSI_SESSION_LOCK(X) mtx_lock(&X->is_lock) #define ISCSI_SESSION_UNLOCK(X) mtx_unlock(&X->is_lock) #define ISCSI_SESSION_LOCK_ASSERT(X) mtx_assert(&X->is_lock, MA_OWNED) #define ISCSI_SESSION_LOCK_ASSERT_NOT(X) mtx_assert(&X->is_lock, MA_NOTOWNED) static int iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode, struct thread *td); static struct cdevsw iscsi_cdevsw = { .d_version = D_VERSION, .d_ioctl = iscsi_ioctl, .d_name = "iscsi", }; static void iscsi_pdu_queue_locked(struct icl_pdu *request); static void iscsi_pdu_queue(struct icl_pdu *request); static void iscsi_pdu_update_statsn(const struct icl_pdu *response); static void iscsi_pdu_handle_nop_in(struct icl_pdu *response); static void iscsi_pdu_handle_scsi_response(struct icl_pdu *response); static void iscsi_pdu_handle_task_response(struct icl_pdu *response); static void iscsi_pdu_handle_data_in(struct icl_pdu *response); static void iscsi_pdu_handle_logout_response(struct icl_pdu *response); static void iscsi_pdu_handle_r2t(struct icl_pdu *response); static void iscsi_pdu_handle_async_message(struct icl_pdu *response); static void iscsi_pdu_handle_reject(struct icl_pdu *response); static void iscsi_session_reconnect(struct iscsi_session *is); static void iscsi_session_terminate(struct iscsi_session *is); static void iscsi_action(struct cam_sim *sim, union ccb *ccb); static void iscsi_poll(struct cam_sim *sim); static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag); static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is, uint32_t initiator_task_tag, union ccb *ccb); static void iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io); static bool iscsi_pdu_prepare(struct icl_pdu *request) { struct iscsi_session *is; struct iscsi_bhs_scsi_command *bhssc; is = PDU_SESSION(request); ISCSI_SESSION_LOCK_ASSERT(is); /* * We're only using fields common for all the request * (initiator -> target) PDUs. */ bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs; /* * Data-Out PDU does not contain CmdSN. */ if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) { if (ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn) && (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) { /* * Current MaxCmdSN prevents us from sending any more * SCSI Command PDUs to the target; postpone the PDU. * It will get resent by either iscsi_pdu_queue(), * or by maintenance thread. */ #if 0 ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %u, " "ExpCmdSN %u, MaxCmdSN %u, opcode 0x%x", is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn, bhssc->bhssc_opcode); #endif return (true); } bhssc->bhssc_cmdsn = htonl(is->is_cmdsn); if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) is->is_cmdsn++; } bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1); return (false); } static void iscsi_session_send_postponed(struct iscsi_session *is) { struct icl_pdu *request; bool postpone; ISCSI_SESSION_LOCK_ASSERT(is); while (!STAILQ_EMPTY(&is->is_postponed)) { request = STAILQ_FIRST(&is->is_postponed); postpone = iscsi_pdu_prepare(request); if (postpone) break; STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next); icl_pdu_queue(request); } } static void iscsi_pdu_queue_locked(struct icl_pdu *request) { struct iscsi_session *is; bool postpone; is = PDU_SESSION(request); ISCSI_SESSION_LOCK_ASSERT(is); iscsi_session_send_postponed(is); postpone = iscsi_pdu_prepare(request); if (postpone) { STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next); return; } icl_pdu_queue(request); } static void iscsi_pdu_queue(struct icl_pdu *request) { struct iscsi_session *is; is = PDU_SESSION(request); ISCSI_SESSION_LOCK(is); iscsi_pdu_queue_locked(request); ISCSI_SESSION_UNLOCK(is); } static void iscsi_session_logout(struct iscsi_session *is) { struct icl_pdu *request; struct iscsi_bhs_logout_request *bhslr; request = icl_pdu_new(is->is_conn, M_NOWAIT); if (request == NULL) return; bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs; bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST; bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION; iscsi_pdu_queue_locked(request); } static void iscsi_session_terminate_task(struct iscsi_session *is, struct iscsi_outstanding *io, bool requeue) { if (io->io_ccb != NULL) { io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK); if (requeue) io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ; else io->io_ccb->ccb_h.status |= CAM_REQ_ABORTED; if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN; xpt_freeze_devq(io->io_ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } xpt_done(io->io_ccb); } iscsi_outstanding_remove(is, io); } static void iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue) { struct iscsi_outstanding *io, *tmp; ISCSI_SESSION_LOCK_ASSERT(is); TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) { iscsi_session_terminate_task(is, io, requeue); } } static void iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim) { struct icl_pdu *pdu; ISCSI_SESSION_LOCK_ASSERT(is); /* * Don't queue any new PDUs. */ if (is->is_sim != NULL && is->is_simq_frozen == false) { ISCSI_SESSION_DEBUG(is, "freezing"); xpt_freeze_simq(is->is_sim, 1); is->is_simq_frozen = true; } /* * Remove postponed PDUs. */ while (!STAILQ_EMPTY(&is->is_postponed)) { pdu = STAILQ_FIRST(&is->is_postponed); STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next); icl_pdu_free(pdu); } if (destroy_sim == false) { /* * Terminate SCSI tasks, asking CAM to requeue them. */ iscsi_session_terminate_tasks(is, true); return; } iscsi_session_terminate_tasks(is, false); if (is->is_sim == NULL) return; ISCSI_SESSION_DEBUG(is, "deregistering SIM"); xpt_async(AC_LOST_DEVICE, is->is_path, NULL); if (is->is_simq_frozen) { xpt_release_simq(is->is_sim, 1); is->is_simq_frozen = false; } xpt_free_path(is->is_path); is->is_path = NULL; xpt_bus_deregister(cam_sim_path(is->is_sim)); cam_sim_free(is->is_sim, TRUE /*free_devq*/); is->is_sim = NULL; is->is_devq = NULL; } static void iscsi_maintenance_thread_reconnect(struct iscsi_session *is) { icl_conn_close(is->is_conn); ISCSI_SESSION_LOCK(is); is->is_connected = false; is->is_reconnecting = false; is->is_login_phase = false; #ifdef ICL_KERNEL_PROXY if (is->is_login_pdu != NULL) { icl_pdu_free(is->is_login_pdu); is->is_login_pdu = NULL; } cv_signal(&is->is_login_cv); #endif if (fail_on_disconnection) { ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices"); iscsi_session_cleanup(is, true); } else { iscsi_session_cleanup(is, false); } KASSERT(TAILQ_EMPTY(&is->is_outstanding), ("destroying session with active tasks")); KASSERT(STAILQ_EMPTY(&is->is_postponed), ("destroying session with postponed PDUs")); /* * Request immediate reconnection from iscsid(8). */ //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)"); is->is_waiting_for_iscsid = true; strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason)); is->is_timeout = 0; ISCSI_SESSION_UNLOCK(is); cv_signal(&is->is_softc->sc_cv); } static void iscsi_maintenance_thread_terminate(struct iscsi_session *is) { struct iscsi_softc *sc; sc = is->is_softc; sx_xlock(&sc->sc_lock); TAILQ_REMOVE(&sc->sc_sessions, is, is_next); sx_xunlock(&sc->sc_lock); icl_conn_close(is->is_conn); ISCSI_SESSION_LOCK(is); KASSERT(is->is_terminating, ("is_terminating == false")); #ifdef ICL_KERNEL_PROXY if (is->is_login_pdu != NULL) { icl_pdu_free(is->is_login_pdu); is->is_login_pdu = NULL; } cv_signal(&is->is_login_cv); #endif callout_drain(&is->is_callout); iscsi_session_cleanup(is, true); KASSERT(TAILQ_EMPTY(&is->is_outstanding), ("destroying session with active tasks")); KASSERT(STAILQ_EMPTY(&is->is_postponed), ("destroying session with postponed PDUs")); ISCSI_SESSION_UNLOCK(is); icl_conn_free(is->is_conn); mtx_destroy(&is->is_lock); cv_destroy(&is->is_maintenance_cv); #ifdef ICL_KERNEL_PROXY cv_destroy(&is->is_login_cv); #endif ISCSI_SESSION_DEBUG(is, "terminated"); free(is, M_ISCSI); /* * The iscsi_unload() routine might be waiting. */ cv_signal(&sc->sc_cv); } static void iscsi_maintenance_thread(void *arg) { struct iscsi_session *is; is = arg; for (;;) { ISCSI_SESSION_LOCK(is); if (is->is_reconnecting == false && is->is_terminating == false && STAILQ_EMPTY(&is->is_postponed)) cv_wait(&is->is_maintenance_cv, &is->is_lock); if (is->is_reconnecting) { ISCSI_SESSION_UNLOCK(is); iscsi_maintenance_thread_reconnect(is); continue; } if (is->is_terminating) { ISCSI_SESSION_UNLOCK(is); iscsi_maintenance_thread_terminate(is); kthread_exit(); return; } iscsi_session_send_postponed(is); ISCSI_SESSION_UNLOCK(is); } } static void iscsi_session_reconnect(struct iscsi_session *is) { /* * XXX: We can't use locking here, because * it's being called from various contexts. * Hope it doesn't break anything. */ if (is->is_reconnecting) return; is->is_reconnecting = true; cv_signal(&is->is_maintenance_cv); } static void iscsi_session_terminate(struct iscsi_session *is) { if (is->is_terminating) return; is->is_terminating = true; #if 0 iscsi_session_logout(is); #endif cv_signal(&is->is_maintenance_cv); } static void iscsi_callout(void *context) { struct icl_pdu *request; struct iscsi_bhs_nop_out *bhsno; struct iscsi_session *is; bool reconnect_needed = false; is = context; if (is->is_terminating) return; callout_schedule(&is->is_callout, 1 * hz); ISCSI_SESSION_LOCK(is); is->is_timeout++; if (is->is_waiting_for_iscsid) { if (iscsid_timeout > 0 && is->is_timeout > iscsid_timeout) { ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) " "for %d seconds; reconnecting", is->is_timeout); reconnect_needed = true; } goto out; } if (is->is_login_phase) { if (login_timeout > 0 && is->is_timeout > login_timeout) { ISCSI_SESSION_WARN(is, "login timed out after %d seconds; " "reconnecting", is->is_timeout); reconnect_needed = true; } goto out; } if (ping_timeout <= 0) { /* * Pings are disabled. Don't send NOP-Out in this case. * Reset the timeout, to avoid triggering reconnection, * should the user decide to reenable them. */ is->is_timeout = 0; goto out; } if (is->is_timeout >= ping_timeout) { ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; " "reconnecting", ping_timeout); reconnect_needed = true; goto out; } ISCSI_SESSION_UNLOCK(is); /* * If the ping was reset less than one second ago - which means * that we've received some PDU during the last second - assume * the traffic flows correctly and don't bother sending a NOP-Out. * * (It's 2 - one for one second, and one for incrementing is_timeout * earlier in this routine.) */ if (is->is_timeout < 2) return; request = icl_pdu_new(is->is_conn, M_NOWAIT); if (request == NULL) { ISCSI_SESSION_WARN(is, "failed to allocate PDU"); return; } bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs; bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT | ISCSI_BHS_OPCODE_IMMEDIATE; bhsno->bhsno_flags = 0x80; bhsno->bhsno_target_transfer_tag = 0xffffffff; iscsi_pdu_queue(request); return; out: ISCSI_SESSION_UNLOCK(is); if (reconnect_needed) iscsi_session_reconnect(is); } static void iscsi_pdu_update_statsn(const struct icl_pdu *response) { const struct iscsi_bhs_data_in *bhsdi; struct iscsi_session *is; uint32_t expcmdsn, maxcmdsn, statsn; is = PDU_SESSION(response); ISCSI_SESSION_LOCK_ASSERT(is); /* * We're only using fields common for all the response * (target -> initiator) PDUs. */ bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs; /* * Ok, I lied. In case of Data-In, "The fields StatSN, Status, * and Residual Count only have meaningful content if the S bit * is set to 1", so we also need to check the bit specific for * Data-In PDU. */ if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN || (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) { statsn = ntohl(bhsdi->bhsdi_statsn); if (statsn != is->is_statsn && statsn != (is->is_statsn + 1)) { /* XXX: This is normal situation for MCS */ ISCSI_SESSION_WARN(is, "PDU 0x%x StatSN %u != " "session ExpStatSN %u (or + 1); reconnecting", bhsdi->bhsdi_opcode, statsn, is->is_statsn); iscsi_session_reconnect(is); } if (ISCSI_SNGT(statsn, is->is_statsn)) is->is_statsn = statsn; } expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn); maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn); if (ISCSI_SNLT(maxcmdsn + 1, expcmdsn)) { ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %u + 1 < PDU ExpCmdSN %u; ignoring", maxcmdsn, expcmdsn); } else { if (ISCSI_SNGT(maxcmdsn, is->is_maxcmdsn)) { is->is_maxcmdsn = maxcmdsn; /* * Command window increased; kick the maintanance thread * to send out postponed commands. */ if (!STAILQ_EMPTY(&is->is_postponed)) cv_signal(&is->is_maintenance_cv); } else if (ISCSI_SNLT(maxcmdsn, is->is_maxcmdsn)) { /* XXX: This is normal situation for MCS */ ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %u < session MaxCmdSN %u; ignoring", maxcmdsn, is->is_maxcmdsn); } if (ISCSI_SNGT(expcmdsn, is->is_expcmdsn)) { is->is_expcmdsn = expcmdsn; } else if (ISCSI_SNLT(expcmdsn, is->is_expcmdsn)) { /* XXX: This is normal situation for MCS */ ISCSI_SESSION_DEBUG(is, "PDU ExpCmdSN %u < session ExpCmdSN %u; ignoring", expcmdsn, is->is_expcmdsn); } } /* * Every incoming PDU - not just NOP-In - resets the ping timer. * The purpose of the timeout is to reset the connection when it stalls; * we don't want this to happen when NOP-In or NOP-Out ends up delayed * in some queue. */ is->is_timeout = 0; } static void iscsi_receive_callback(struct icl_pdu *response) { struct iscsi_session *is; is = PDU_SESSION(response); ISCSI_SESSION_LOCK(is); #ifdef ICL_KERNEL_PROXY if (is->is_login_phase) { if (is->is_login_pdu == NULL) is->is_login_pdu = response; else icl_pdu_free(response); ISCSI_SESSION_UNLOCK(is); cv_signal(&is->is_login_cv); return; } #endif iscsi_pdu_update_statsn(response); /* * The handling routine is responsible for freeing the PDU * when it's no longer needed. */ switch (response->ip_bhs->bhs_opcode) { case ISCSI_BHS_OPCODE_NOP_IN: iscsi_pdu_handle_nop_in(response); ISCSI_SESSION_UNLOCK(is); break; case ISCSI_BHS_OPCODE_SCSI_RESPONSE: iscsi_pdu_handle_scsi_response(response); /* Session lock dropped inside. */ ISCSI_SESSION_LOCK_ASSERT_NOT(is); break; case ISCSI_BHS_OPCODE_TASK_RESPONSE: iscsi_pdu_handle_task_response(response); ISCSI_SESSION_UNLOCK(is); break; case ISCSI_BHS_OPCODE_SCSI_DATA_IN: iscsi_pdu_handle_data_in(response); /* Session lock dropped inside. */ ISCSI_SESSION_LOCK_ASSERT_NOT(is); break; case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE: iscsi_pdu_handle_logout_response(response); ISCSI_SESSION_UNLOCK(is); break; case ISCSI_BHS_OPCODE_R2T: iscsi_pdu_handle_r2t(response); ISCSI_SESSION_UNLOCK(is); break; case ISCSI_BHS_OPCODE_ASYNC_MESSAGE: iscsi_pdu_handle_async_message(response); ISCSI_SESSION_UNLOCK(is); break; case ISCSI_BHS_OPCODE_REJECT: iscsi_pdu_handle_reject(response); ISCSI_SESSION_UNLOCK(is); break; default: ISCSI_SESSION_WARN(is, "received PDU with unsupported " "opcode 0x%x; reconnecting", response->ip_bhs->bhs_opcode); iscsi_session_reconnect(is); ISCSI_SESSION_UNLOCK(is); icl_pdu_free(response); } } static void iscsi_error_callback(struct icl_conn *ic) { struct iscsi_session *is; is = CONN_SESSION(ic); ISCSI_SESSION_WARN(is, "connection error; reconnecting"); iscsi_session_reconnect(is); } static void iscsi_pdu_handle_nop_in(struct icl_pdu *response) { struct iscsi_session *is; struct iscsi_bhs_nop_out *bhsno; struct iscsi_bhs_nop_in *bhsni; struct icl_pdu *request; void *data = NULL; size_t datasize; int error; is = PDU_SESSION(response); bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs; if (bhsni->bhsni_target_transfer_tag == 0xffffffff) { /* * Nothing to do; iscsi_pdu_update_statsn() already * zeroed the timeout. */ icl_pdu_free(response); return; } datasize = icl_pdu_data_segment_length(response); if (datasize > 0) { data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO); if (data == NULL) { ISCSI_SESSION_WARN(is, "failed to allocate memory; " "reconnecting"); icl_pdu_free(response); iscsi_session_reconnect(is); return; } icl_pdu_get_data(response, 0, data, datasize); } request = icl_pdu_new(response->ip_conn, M_NOWAIT); if (request == NULL) { ISCSI_SESSION_WARN(is, "failed to allocate memory; " "reconnecting"); free(data, M_ISCSI); icl_pdu_free(response); iscsi_session_reconnect(is); return; } bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs; bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT | ISCSI_BHS_OPCODE_IMMEDIATE; bhsno->bhsno_flags = 0x80; bhsno->bhsno_initiator_task_tag = 0xffffffff; bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag; if (datasize > 0) { error = icl_pdu_append_data(request, data, datasize, M_NOWAIT); if (error != 0) { ISCSI_SESSION_WARN(is, "failed to allocate memory; " "reconnecting"); free(data, M_ISCSI); icl_pdu_free(request); icl_pdu_free(response); iscsi_session_reconnect(is); return; } free(data, M_ISCSI); } icl_pdu_free(response); iscsi_pdu_queue_locked(request); } static void iscsi_pdu_handle_scsi_response(struct icl_pdu *response) { struct iscsi_bhs_scsi_response *bhssr; struct iscsi_outstanding *io; struct iscsi_session *is; union ccb *ccb; struct ccb_scsiio *csio; size_t data_segment_len, received; uint16_t sense_len; is = PDU_SESSION(response); bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs; io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag); if (io == NULL || io->io_ccb == NULL) { ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag); icl_pdu_free(response); iscsi_session_reconnect(is); ISCSI_SESSION_UNLOCK(is); return; } ccb = io->io_ccb; received = io->io_received; iscsi_outstanding_remove(is, io); ISCSI_SESSION_UNLOCK(is); if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) { ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response); if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN; } else if (bhssr->bhssr_status == 0) { ccb->ccb_h.status = CAM_REQ_CMP; } else { if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN; ccb->csio.scsi_status = bhssr->bhssr_status; } csio = &ccb->csio; data_segment_len = icl_pdu_data_segment_length(response); if (data_segment_len > 0) { if (data_segment_len < sizeof(sense_len)) { ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)", data_segment_len); if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN; goto out; } icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len)); sense_len = ntohs(sense_len); #if 0 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd", sense_len, data_segment_len); #endif if (sizeof(sense_len) + sense_len > data_segment_len) { ISCSI_SESSION_WARN(is, "truncated data segment " "(%zd bytes, should be %zd)", data_segment_len, sizeof(sense_len) + sense_len); if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN; goto out; } else if (sizeof(sense_len) + sense_len < data_segment_len) ISCSI_SESSION_WARN(is, "oversize data segment " "(%zd bytes, should be %zd)", data_segment_len, sizeof(sense_len) + sense_len); if (sense_len > csio->sense_len) { ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d", sense_len, csio->sense_len); sense_len = csio->sense_len; } icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len); csio->sense_resid = csio->sense_len - sense_len; ccb->ccb_h.status |= CAM_AUTOSNS_VALID; } out: if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW) csio->resid = ntohl(bhssr->bhssr_residual_count); if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { KASSERT(received <= csio->dxfer_len, ("received > csio->dxfer_len")); if (received < csio->dxfer_len) { if (csio->resid != csio->dxfer_len - received) { ISCSI_SESSION_WARN(is, "underflow mismatch: " "target indicates %d, we calculated %zd", csio->resid, csio->dxfer_len - received); } csio->resid = csio->dxfer_len - received; } } xpt_done(ccb); icl_pdu_free(response); } static void iscsi_pdu_handle_task_response(struct icl_pdu *response) { struct iscsi_bhs_task_management_response *bhstmr; struct iscsi_outstanding *io, *aio; struct iscsi_session *is; is = PDU_SESSION(response); bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs; io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag); if (io == NULL || io->io_ccb != NULL) { ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhstmr->bhstmr_initiator_task_tag); icl_pdu_free(response); iscsi_session_reconnect(is); return; } if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) { ISCSI_SESSION_WARN(is, "task response 0x%x", bhstmr->bhstmr_response); } else { aio = iscsi_outstanding_find(is, io->io_datasn); if (aio != NULL && aio->io_ccb != NULL) iscsi_session_terminate_task(is, aio, false); } iscsi_outstanding_remove(is, io); icl_pdu_free(response); } static void iscsi_pdu_handle_data_in(struct icl_pdu *response) { struct iscsi_bhs_data_in *bhsdi; struct iscsi_outstanding *io; struct iscsi_session *is; union ccb *ccb; struct ccb_scsiio *csio; size_t data_segment_len, received, oreceived; is = PDU_SESSION(response); bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs; io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag); if (io == NULL || io->io_ccb == NULL) { ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag); icl_pdu_free(response); iscsi_session_reconnect(is); ISCSI_SESSION_UNLOCK(is); return; } data_segment_len = icl_pdu_data_segment_length(response); if (data_segment_len == 0) { /* * "The sending of 0 length data segments should be avoided, * but initiators and targets MUST be able to properly receive * 0 length data segments." */ ISCSI_SESSION_UNLOCK(is); icl_pdu_free(response); return; } /* * We need to track this for security reasons - without it, malicious target * could respond to SCSI READ without sending Data-In PDUs, which would result * in read operation on the initiator side returning random kernel data. */ if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) { ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd", io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset)); icl_pdu_free(response); iscsi_session_reconnect(is); ISCSI_SESSION_UNLOCK(is); return; } ccb = io->io_ccb; csio = &ccb->csio; if (io->io_received + data_segment_len > csio->dxfer_len) { ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes " "at offset %zd, buffer is %d)", data_segment_len, io->io_received, csio->dxfer_len); icl_pdu_free(response); iscsi_session_reconnect(is); ISCSI_SESSION_UNLOCK(is); return; } oreceived = io->io_received; io->io_received += data_segment_len; received = io->io_received; if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) iscsi_outstanding_remove(is, io); ISCSI_SESSION_UNLOCK(is); icl_pdu_get_data(response, 0, csio->data_ptr + oreceived, data_segment_len); /* * XXX: Check DataSN. * XXX: Check F. */ if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) { /* * Nothing more to do. */ icl_pdu_free(response); return; } //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status); if (bhsdi->bhsdi_status == 0) { ccb->ccb_h.status = CAM_REQ_CMP; } else { if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN; csio->scsi_status = bhsdi->bhsdi_status; } if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { KASSERT(received <= csio->dxfer_len, ("received > csio->dxfer_len")); if (received < csio->dxfer_len) { csio->resid = ntohl(bhsdi->bhsdi_residual_count); if (csio->resid != csio->dxfer_len - received) { ISCSI_SESSION_WARN(is, "underflow mismatch: " "target indicates %d, we calculated %zd", csio->resid, csio->dxfer_len - received); } csio->resid = csio->dxfer_len - received; } } xpt_done(ccb); icl_pdu_free(response); } static void iscsi_pdu_handle_logout_response(struct icl_pdu *response) { ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response"); icl_pdu_free(response); } static void iscsi_pdu_handle_r2t(struct icl_pdu *response) { struct icl_pdu *request; struct iscsi_session *is; struct iscsi_bhs_r2t *bhsr2t; struct iscsi_bhs_data_out *bhsdo; struct iscsi_outstanding *io; struct ccb_scsiio *csio; size_t off, len, total_len; int error; is = PDU_SESSION(response); bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs; io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag); if (io == NULL || io->io_ccb == NULL) { ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting", bhsr2t->bhsr2t_initiator_task_tag); icl_pdu_free(response); iscsi_session_reconnect(is); return; } csio = &io->io_ccb->csio; if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) { ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting"); icl_pdu_free(response); iscsi_session_reconnect(is); return; } /* * XXX: Verify R2TSN. */ io->io_datasn = 0; off = ntohl(bhsr2t->bhsr2t_buffer_offset); if (off > csio->dxfer_len) { ISCSI_SESSION_WARN(is, "target requested invalid offset " "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len); icl_pdu_free(response); iscsi_session_reconnect(is); return; } total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length); if (total_len == 0 || total_len > csio->dxfer_len) { ISCSI_SESSION_WARN(is, "target requested invalid length " "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len); icl_pdu_free(response); iscsi_session_reconnect(is); return; } //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len); for (;;) { len = total_len; if (len > is->is_max_data_segment_length) len = is->is_max_data_segment_length; if (off + len > csio->dxfer_len) { ISCSI_SESSION_WARN(is, "target requested invalid " "length/offset %zd, buffer is %d; reconnecting", off + len, csio->dxfer_len); icl_pdu_free(response); iscsi_session_reconnect(is); return; } request = icl_pdu_new(response->ip_conn, M_NOWAIT); if (request == NULL) { icl_pdu_free(response); iscsi_session_reconnect(is); return; } bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs; bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT; bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun; bhsdo->bhsdo_initiator_task_tag = bhsr2t->bhsr2t_initiator_task_tag; bhsdo->bhsdo_target_transfer_tag = bhsr2t->bhsr2t_target_transfer_tag; bhsdo->bhsdo_datasn = htonl(io->io_datasn++); bhsdo->bhsdo_buffer_offset = htonl(off); error = icl_pdu_append_data(request, csio->data_ptr + off, len, M_NOWAIT); if (error != 0) { ISCSI_SESSION_WARN(is, "failed to allocate memory; " "reconnecting"); icl_pdu_free(request); icl_pdu_free(response); iscsi_session_reconnect(is); return; } off += len; total_len -= len; if (total_len == 0) { bhsdo->bhsdo_flags |= BHSDO_FLAGS_F; //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off); } else { //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off); } iscsi_pdu_queue_locked(request); if (total_len == 0) break; } icl_pdu_free(response); } static void iscsi_pdu_handle_async_message(struct icl_pdu *response) { struct iscsi_bhs_asynchronous_message *bhsam; struct iscsi_session *is; is = PDU_SESSION(response); bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs; switch (bhsam->bhsam_async_event) { case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT: ISCSI_SESSION_WARN(is, "target requests logout; removing session"); iscsi_session_logout(is); iscsi_session_terminate(is); break; case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION: ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection"); break; case BHSAM_EVENT_TARGET_TERMINATES_SESSION: ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session"); break; default: /* * XXX: Technically, we're obligated to also handle * parameter renegotiation. */ ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event); break; } icl_pdu_free(response); } static void iscsi_pdu_handle_reject(struct icl_pdu *response) { struct iscsi_bhs_reject *bhsr; struct iscsi_session *is; is = PDU_SESSION(response); bhsr = (struct iscsi_bhs_reject *)response->ip_bhs; ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?", bhsr->bhsr_reason); icl_pdu_free(response); } static int iscsi_ioctl_daemon_wait(struct iscsi_softc *sc, struct iscsi_daemon_request *request) { struct iscsi_session *is; int error; sx_slock(&sc->sc_lock); for (;;) { TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { ISCSI_SESSION_LOCK(is); if (is->is_waiting_for_iscsid) break; ISCSI_SESSION_UNLOCK(is); } if (is == NULL) { /* * No session requires attention from iscsid(8); wait. */ error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock); if (error != 0) { sx_sunlock(&sc->sc_lock); return (error); } continue; } is->is_waiting_for_iscsid = false; is->is_login_phase = true; is->is_reason[0] = '\0'; ISCSI_SESSION_UNLOCK(is); request->idr_session_id = is->is_id; memcpy(&request->idr_isid, &is->is_isid, sizeof(request->idr_isid)); request->idr_tsih = 0; /* New or reinstated session. */ memcpy(&request->idr_conf, &is->is_conf, sizeof(request->idr_conf)); sx_sunlock(&sc->sc_lock); return (0); } } static int iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc, struct iscsi_daemon_handoff *handoff) { struct iscsi_session *is; int error; sx_slock(&sc->sc_lock); /* * Find the session to hand off socket to. */ TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { if (is->is_id == handoff->idh_session_id) break; } if (is == NULL) { sx_sunlock(&sc->sc_lock); return (ESRCH); } ISCSI_SESSION_LOCK(is); if (is->is_conf.isc_discovery || is->is_terminating) { ISCSI_SESSION_UNLOCK(is); sx_sunlock(&sc->sc_lock); return (EINVAL); } if (is->is_connected) { /* * This might have happened because another iscsid(8) * instance handed off the connection in the meantime. * Just return. */ ISCSI_SESSION_WARN(is, "handoff on already connected " "session"); ISCSI_SESSION_UNLOCK(is); sx_sunlock(&sc->sc_lock); return (EBUSY); } strlcpy(is->is_target_alias, handoff->idh_target_alias, sizeof(is->is_target_alias)); is->is_tsih = handoff->idh_tsih; is->is_statsn = handoff->idh_statsn; is->is_initial_r2t = handoff->idh_initial_r2t; is->is_immediate_data = handoff->idh_immediate_data; is->is_max_data_segment_length = handoff->idh_max_data_segment_length; is->is_max_burst_length = handoff->idh_max_burst_length; is->is_first_burst_length = handoff->idh_first_burst_length; if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C) is->is_conn->ic_header_crc32c = true; else is->is_conn->ic_header_crc32c = false; if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C) is->is_conn->ic_data_crc32c = true; else is->is_conn->ic_data_crc32c = false; is->is_cmdsn = 0; is->is_expcmdsn = 0; is->is_maxcmdsn = 0; is->is_waiting_for_iscsid = false; is->is_login_phase = false; is->is_timeout = 0; is->is_connected = true; is->is_reason[0] = '\0'; ISCSI_SESSION_UNLOCK(is); #ifdef ICL_KERNEL_PROXY if (handoff->idh_socket != 0) { #endif /* * Handoff without using ICL proxy. */ error = icl_conn_handoff(is->is_conn, handoff->idh_socket); if (error != 0) { sx_sunlock(&sc->sc_lock); iscsi_session_terminate(is); return (error); } #ifdef ICL_KERNEL_PROXY } #endif sx_sunlock(&sc->sc_lock); if (is->is_sim != NULL) { /* * When reconnecting, there already is SIM allocated for the session. */ KASSERT(is->is_simq_frozen, ("reconnect without frozen simq")); ISCSI_SESSION_LOCK(is); ISCSI_SESSION_DEBUG(is, "releasing"); xpt_release_simq(is->is_sim, 1); is->is_simq_frozen = false; ISCSI_SESSION_UNLOCK(is); } else { ISCSI_SESSION_LOCK(is); is->is_devq = cam_simq_alloc(maxtags); if (is->is_devq == NULL) { ISCSI_SESSION_WARN(is, "failed to allocate simq"); iscsi_session_terminate(is); return (ENOMEM); } is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi", is, is->is_id /* unit */, &is->is_lock, 1, maxtags, is->is_devq); if (is->is_sim == NULL) { ISCSI_SESSION_UNLOCK(is); ISCSI_SESSION_WARN(is, "failed to allocate SIM"); cam_simq_free(is->is_devq); iscsi_session_terminate(is); return (ENOMEM); } error = xpt_bus_register(is->is_sim, NULL, 0); if (error != 0) { ISCSI_SESSION_UNLOCK(is); ISCSI_SESSION_WARN(is, "failed to register bus"); iscsi_session_terminate(is); return (ENOMEM); } error = xpt_create_path(&is->is_path, /*periph*/NULL, cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); if (error != CAM_REQ_CMP) { ISCSI_SESSION_UNLOCK(is); ISCSI_SESSION_WARN(is, "failed to create path"); iscsi_session_terminate(is); return (ENOMEM); } ISCSI_SESSION_UNLOCK(is); } return (0); } static int iscsi_ioctl_daemon_fail(struct iscsi_softc *sc, struct iscsi_daemon_fail *fail) { struct iscsi_session *is; sx_slock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { if (is->is_id == fail->idf_session_id) break; } if (is == NULL) { sx_sunlock(&sc->sc_lock); return (ESRCH); } ISCSI_SESSION_LOCK(is); ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s", fail->idf_reason); strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason)); //is->is_waiting_for_iscsid = false; //is->is_login_phase = true; //iscsi_session_reconnect(is); ISCSI_SESSION_UNLOCK(is); sx_sunlock(&sc->sc_lock); return (0); } #ifdef ICL_KERNEL_PROXY static int iscsi_ioctl_daemon_connect(struct iscsi_softc *sc, struct iscsi_daemon_connect *idc) { struct iscsi_session *is; struct sockaddr *from_sa, *to_sa; int error; sx_slock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { if (is->is_id == idc->idc_session_id) break; } if (is == NULL) { sx_sunlock(&sc->sc_lock); return (ESRCH); } sx_sunlock(&sc->sc_lock); if (idc->idc_from_addrlen > 0) { error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen); if (error != 0) { ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d", error); return (error); } } else { from_sa = NULL; } error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen); if (error != 0) { ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d", error); free(from_sa, M_SONAME); return (error); } ISCSI_SESSION_LOCK(is); is->is_waiting_for_iscsid = false; is->is_login_phase = true; is->is_timeout = 0; ISCSI_SESSION_UNLOCK(is); error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain, idc->idc_socktype, idc->idc_protocol, from_sa, to_sa); free(from_sa, M_SONAME); free(to_sa, M_SONAME); /* * Digests are always disabled during login phase. */ is->is_conn->ic_header_crc32c = false; is->is_conn->ic_data_crc32c = false; return (error); } static int iscsi_ioctl_daemon_send(struct iscsi_softc *sc, struct iscsi_daemon_send *ids) { struct iscsi_session *is; struct icl_pdu *ip; size_t datalen; void *data; int error; sx_slock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { if (is->is_id == ids->ids_session_id) break; } if (is == NULL) { sx_sunlock(&sc->sc_lock); return (ESRCH); } sx_sunlock(&sc->sc_lock); if (is->is_login_phase == false) return (EBUSY); if (is->is_terminating || is->is_reconnecting) return (EIO); datalen = ids->ids_data_segment_len; if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH) return (EINVAL); if (datalen > 0) { data = malloc(datalen, M_ISCSI, M_WAITOK); error = copyin(ids->ids_data_segment, data, datalen); if (error != 0) { free(data, M_ISCSI); return (error); } } ip = icl_pdu_new(is->is_conn, M_WAITOK); memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs)); if (datalen > 0) { error = icl_pdu_append_data(ip, data, datalen, M_WAITOK); KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed")); free(data, M_ISCSI); } icl_pdu_queue(ip); return (0); } static int iscsi_ioctl_daemon_receive(struct iscsi_softc *sc, struct iscsi_daemon_receive *idr) { struct iscsi_session *is; struct icl_pdu *ip; void *data; sx_slock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { if (is->is_id == idr->idr_session_id) break; } if (is == NULL) { sx_sunlock(&sc->sc_lock); return (ESRCH); } sx_sunlock(&sc->sc_lock); if (is->is_login_phase == false) return (EBUSY); ISCSI_SESSION_LOCK(is); while (is->is_login_pdu == NULL && is->is_terminating == false && is->is_reconnecting == false) cv_wait(&is->is_login_cv, &is->is_lock); if (is->is_terminating || is->is_reconnecting) { ISCSI_SESSION_UNLOCK(is); return (EIO); } ip = is->is_login_pdu; is->is_login_pdu = NULL; ISCSI_SESSION_UNLOCK(is); if (ip->ip_data_len > idr->idr_data_segment_len) { icl_pdu_free(ip); return (EMSGSIZE); } copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs)); if (ip->ip_data_len > 0) { data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK); icl_pdu_get_data(ip, 0, data, ip->ip_data_len); copyout(data, idr->idr_data_segment, ip->ip_data_len); free(data, M_ISCSI); } icl_pdu_free(ip); return (0); } #endif /* ICL_KERNEL_PROXY */ static void iscsi_sanitize_session_conf(struct iscsi_session_conf *isc) { /* * Just make sure all the fields are null-terminated. * * XXX: This is not particularly secure. We should * create our own conf and then copy in relevant * fields. */ isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0'; isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0'; isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0'; isc->isc_target[ISCSI_NAME_LEN - 1] = '\0'; isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0'; isc->isc_user[ISCSI_NAME_LEN - 1] = '\0'; isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0'; isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0'; isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0'; } static bool iscsi_valid_session_conf(const struct iscsi_session_conf *isc) { if (isc->isc_initiator[0] == '\0') { ISCSI_DEBUG("empty isc_initiator"); return (false); } if (isc->isc_target_addr[0] == '\0') { ISCSI_DEBUG("empty isc_target_addr"); return (false); } if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) { ISCSI_DEBUG("non-empty isc_target for discovery session"); return (false); } if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) { ISCSI_DEBUG("empty isc_target for non-discovery session"); return (false); } return (true); } static int iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa) { struct iscsi_session *is; const struct iscsi_session *is2; int error; iscsi_sanitize_session_conf(&isa->isa_conf); if (iscsi_valid_session_conf(&isa->isa_conf) == false) return (EINVAL); is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK); memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf)); sx_xlock(&sc->sc_lock); /* * Prevent duplicates. */ TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) { if (!!is->is_conf.isc_discovery != !!is2->is_conf.isc_discovery) continue; if (strcmp(is->is_conf.isc_target_addr, is2->is_conf.isc_target_addr) != 0) continue; if (is->is_conf.isc_discovery == 0 && strcmp(is->is_conf.isc_target, is2->is_conf.isc_target) != 0) continue; sx_xunlock(&sc->sc_lock); free(is, M_ISCSI); return (EBUSY); } - is->is_conn = icl_conn_new("iscsi", &is->is_lock); + is->is_conn = icl_new_conn(NULL, "iscsi", &is->is_lock); is->is_conn->ic_receive = iscsi_receive_callback; is->is_conn->ic_error = iscsi_error_callback; is->is_conn->ic_prv0 = is; TAILQ_INIT(&is->is_outstanding); STAILQ_INIT(&is->is_postponed); mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF); cv_init(&is->is_maintenance_cv, "iscsi_mt"); #ifdef ICL_KERNEL_PROXY cv_init(&is->is_login_cv, "iscsi_login"); #endif is->is_softc = sc; sc->sc_last_session_id++; is->is_id = sc->sc_last_session_id; is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */ arc4rand(&is->is_isid[1], 5, 0); is->is_tsih = 0; callout_init(&is->is_callout, 1); callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is); TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next); error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt"); if (error != 0) { ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error); return (error); } /* * Trigger immediate reconnection. */ ISCSI_SESSION_LOCK(is); is->is_waiting_for_iscsid = true; strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason)); ISCSI_SESSION_UNLOCK(is); cv_signal(&sc->sc_cv); sx_xunlock(&sc->sc_lock); return (0); } static bool iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1, unsigned int id2, const struct iscsi_session_conf *c2) { if (id2 != 0 && id2 != id1) return (false); if (c2->isc_target[0] != '\0' && strcmp(c1->isc_target, c2->isc_target) != 0) return (false); if (c2->isc_target_addr[0] != '\0' && strcmp(c1->isc_target_addr, c2->isc_target_addr) != 0) return (false); return (true); } static int iscsi_ioctl_session_remove(struct iscsi_softc *sc, struct iscsi_session_remove *isr) { struct iscsi_session *is, *tmp; bool found = false; iscsi_sanitize_session_conf(&isr->isr_conf); sx_xlock(&sc->sc_lock); TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) { ISCSI_SESSION_LOCK(is); if (iscsi_session_conf_matches(is->is_id, &is->is_conf, isr->isr_session_id, &isr->isr_conf)) { found = true; iscsi_session_logout(is); iscsi_session_terminate(is); } ISCSI_SESSION_UNLOCK(is); } sx_xunlock(&sc->sc_lock); if (!found) return (ESRCH); return (0); } static int iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl) { int error; unsigned int i = 0; struct iscsi_session *is; struct iscsi_session_state iss; sx_slock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { if (i >= isl->isl_nentries) { sx_sunlock(&sc->sc_lock); return (EMSGSIZE); } memset(&iss, 0, sizeof(iss)); memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf)); iss.iss_id = is->is_id; strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias)); strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason)); if (is->is_conn->ic_header_crc32c) iss.iss_header_digest = ISCSI_DIGEST_CRC32C; else iss.iss_header_digest = ISCSI_DIGEST_NONE; if (is->is_conn->ic_data_crc32c) iss.iss_data_digest = ISCSI_DIGEST_CRC32C; else iss.iss_data_digest = ISCSI_DIGEST_NONE; iss.iss_max_data_segment_length = is->is_max_data_segment_length; iss.iss_immediate_data = is->is_immediate_data; iss.iss_connected = is->is_connected; error = copyout(&iss, isl->isl_pstates + i, sizeof(iss)); if (error != 0) { sx_sunlock(&sc->sc_lock); return (error); } i++; } sx_sunlock(&sc->sc_lock); isl->isl_nentries = i; return (0); } static int iscsi_ioctl_session_modify(struct iscsi_softc *sc, struct iscsi_session_modify *ism) { struct iscsi_session *is; iscsi_sanitize_session_conf(&ism->ism_conf); if (iscsi_valid_session_conf(&ism->ism_conf) == false) return (EINVAL); sx_xlock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { ISCSI_SESSION_LOCK(is); if (is->is_id == ism->ism_session_id) break; ISCSI_SESSION_UNLOCK(is); } if (is == NULL) { sx_xunlock(&sc->sc_lock); return (ESRCH); } sx_xunlock(&sc->sc_lock); memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf)); ISCSI_SESSION_UNLOCK(is); iscsi_session_reconnect(is); return (0); } static int iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode, struct thread *td) { struct iscsi_softc *sc; sc = dev->si_drv1; switch (cmd) { case ISCSIDWAIT: return (iscsi_ioctl_daemon_wait(sc, (struct iscsi_daemon_request *)arg)); case ISCSIDHANDOFF: return (iscsi_ioctl_daemon_handoff(sc, (struct iscsi_daemon_handoff *)arg)); case ISCSIDFAIL: return (iscsi_ioctl_daemon_fail(sc, (struct iscsi_daemon_fail *)arg)); #ifdef ICL_KERNEL_PROXY case ISCSIDCONNECT: return (iscsi_ioctl_daemon_connect(sc, (struct iscsi_daemon_connect *)arg)); case ISCSIDSEND: return (iscsi_ioctl_daemon_send(sc, (struct iscsi_daemon_send *)arg)); case ISCSIDRECEIVE: return (iscsi_ioctl_daemon_receive(sc, (struct iscsi_daemon_receive *)arg)); #endif /* ICL_KERNEL_PROXY */ case ISCSISADD: return (iscsi_ioctl_session_add(sc, (struct iscsi_session_add *)arg)); case ISCSISREMOVE: return (iscsi_ioctl_session_remove(sc, (struct iscsi_session_remove *)arg)); case ISCSISLIST: return (iscsi_ioctl_session_list(sc, (struct iscsi_session_list *)arg)); case ISCSISMODIFY: return (iscsi_ioctl_session_modify(sc, (struct iscsi_session_modify *)arg)); default: return (EINVAL); } } static struct iscsi_outstanding * iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag) { struct iscsi_outstanding *io; ISCSI_SESSION_LOCK_ASSERT(is); TAILQ_FOREACH(io, &is->is_outstanding, io_next) { if (io->io_initiator_task_tag == initiator_task_tag) return (io); } return (NULL); } static struct iscsi_outstanding * iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb) { struct iscsi_outstanding *io; ISCSI_SESSION_LOCK_ASSERT(is); TAILQ_FOREACH(io, &is->is_outstanding, io_next) { if (io->io_ccb == ccb) return (io); } return (NULL); } static struct iscsi_outstanding * iscsi_outstanding_add(struct iscsi_session *is, uint32_t initiator_task_tag, union ccb *ccb) { struct iscsi_outstanding *io; ISCSI_SESSION_LOCK_ASSERT(is); KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL, ("initiator_task_tag 0x%x already added", initiator_task_tag)); io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO); if (io == NULL) { ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io)); return (NULL); } io->io_initiator_task_tag = initiator_task_tag; io->io_ccb = ccb; TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next); return (io); } static void iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io) { ISCSI_SESSION_LOCK_ASSERT(is); TAILQ_REMOVE(&is->is_outstanding, io, io_next); uma_zfree(iscsi_outstanding_zone, io); } static void iscsi_action_abort(struct iscsi_session *is, union ccb *ccb) { struct icl_pdu *request; struct iscsi_bhs_task_management_request *bhstmr; struct ccb_abort *cab = &ccb->cab; struct iscsi_outstanding *io, *aio; ISCSI_SESSION_LOCK_ASSERT(is); #if 0 KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__)); #else if (is->is_login_phase) { ccb->ccb_h.status = CAM_REQ_ABORTED; xpt_done(ccb); return; } #endif aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb); if (aio == NULL) { ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); return; } request = icl_pdu_new(is->is_conn, M_NOWAIT); if (request == NULL) { ccb->ccb_h.status = CAM_RESRC_UNAVAIL; xpt_done(ccb); return; } bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs; bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST; bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK; bhstmr->bhstmr_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun)); bhstmr->bhstmr_initiator_task_tag = is->is_initiator_task_tag; is->is_initiator_task_tag++; bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag; io = iscsi_outstanding_add(is, bhstmr->bhstmr_initiator_task_tag, NULL); if (io == NULL) { icl_pdu_free(request); ccb->ccb_h.status = CAM_RESRC_UNAVAIL; xpt_done(ccb); return; } io->io_datasn = aio->io_initiator_task_tag; iscsi_pdu_queue_locked(request); } static void iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb) { struct icl_pdu *request; struct iscsi_bhs_scsi_command *bhssc; struct ccb_scsiio *csio; struct iscsi_outstanding *io; size_t len; int error; ISCSI_SESSION_LOCK_ASSERT(is); #if 0 KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__)); #else if (is->is_login_phase) { ISCSI_SESSION_DEBUG(is, "called during login phase"); if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN; xpt_done(ccb); return; } #endif request = icl_pdu_new(is->is_conn, M_NOWAIT); if (request == NULL) { if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN; xpt_done(ccb); return; } csio = &ccb->csio; bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs; bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND; bhssc->bhssc_flags |= BHSSC_FLAGS_F; switch (csio->ccb_h.flags & CAM_DIR_MASK) { case CAM_DIR_IN: bhssc->bhssc_flags |= BHSSC_FLAGS_R; break; case CAM_DIR_OUT: bhssc->bhssc_flags |= BHSSC_FLAGS_W; break; } if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) { switch (csio->tag_action) { case MSG_HEAD_OF_Q_TAG: bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ; break; case MSG_ORDERED_Q_TAG: bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED; break; case MSG_ACA_TASK: bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA; break; case MSG_SIMPLE_Q_TAG: default: bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE; break; } } else bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED; bhssc->bhssc_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun)); bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag; is->is_initiator_task_tag++; bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len); KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb), ("unsupported CDB size %zd", (size_t)csio->cdb_len)); if (csio->ccb_h.flags & CAM_CDB_POINTER) memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len); else memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len); io = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb); if (io == NULL) { icl_pdu_free(request); if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN; xpt_done(ccb); return; } if (is->is_immediate_data && (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) { len = csio->dxfer_len; //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len); if (len > is->is_first_burst_length) { ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length); len = is->is_first_burst_length; } if (len > is->is_max_data_segment_length) { ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_max_data_segment_length); len = is->is_max_data_segment_length; } error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT); if (error != 0) { icl_pdu_free(request); if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { xpt_freeze_devq(ccb->ccb_h.path, 1); ISCSI_SESSION_DEBUG(is, "freezing devq"); } ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN; xpt_done(ccb); return; } } iscsi_pdu_queue_locked(request); } static void iscsi_action(struct cam_sim *sim, union ccb *ccb) { struct iscsi_session *is; is = cam_sim_softc(sim); ISCSI_SESSION_LOCK_ASSERT(is); if (is->is_terminating || (is->is_connected == false && fail_on_disconnection)) { ccb->ccb_h.status = CAM_DEV_NOT_THERE; xpt_done(ccb); return; } switch (ccb->ccb_h.func_code) { case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; cpi->version_num = 1; cpi->hba_inquiry = PI_TAG_ABLE; cpi->target_sprt = 0; cpi->hba_misc = PIM_EXTLUNS; cpi->hba_eng_cnt = 0; cpi->max_target = 0; /* * Note that the variable below is only relevant for targets * that don't claim compliance with anything above SPC2, which * means they don't support REPORT_LUNS. */ cpi->max_lun = 255; cpi->initiator_id = ~0; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); cpi->base_transfer_speed = 150000; /* XXX */ cpi->transport = XPORT_ISCSI; cpi->transport_version = 0; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_SPC3; cpi->maxio = MAXPHYS; cpi->ccb_h.status = CAM_REQ_CMP; break; } case XPT_GET_TRAN_SETTINGS: { struct ccb_trans_settings *cts; struct ccb_trans_settings_scsi *scsi; cts = &ccb->cts; scsi = &cts->proto_specific.scsi; cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_SPC3; cts->transport = XPORT_ISCSI; cts->transport_version = 0; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; cts->ccb_h.status = CAM_REQ_CMP; break; } case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, /*extended*/1); ccb->ccb_h.status = CAM_REQ_CMP; break; #if 0 /* * XXX: What's the point? */ case XPT_RESET_BUS: case XPT_TERM_IO: ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io"); ccb->ccb_h.status = CAM_REQ_CMP; break; #endif case XPT_ABORT: iscsi_action_abort(is, ccb); return; case XPT_SCSI_IO: iscsi_action_scsiio(is, ccb); return; default: #if 0 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code); #endif ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; break; } xpt_done(ccb); } static void iscsi_poll(struct cam_sim *sim) { KASSERT(0, ("%s: you're not supposed to be here", __func__)); } static void iscsi_shutdown(struct iscsi_softc *sc) { struct iscsi_session *is; ISCSI_DEBUG("removing all sessions due to shutdown"); sx_slock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) iscsi_session_terminate(is); sx_sunlock(&sc->sc_lock); } static int iscsi_load(void) { int error; sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK); sx_init(&sc->sc_lock, "iscsi"); TAILQ_INIT(&sc->sc_sessions); cv_init(&sc->sc_cv, "iscsi_cv"); iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding", sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw, NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi"); if (error != 0) { ISCSI_WARN("failed to create device node, error %d", error); return (error); } sc->sc_cdev->si_drv1 = sc; /* * Note that this needs to get run before dashutdown(). Otherwise, * when rebooting with iSCSI session with outstanding requests, * but disconnected, dashutdown() will hang on cam_periph_runccb(). */ sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync, iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST); return (0); } static int iscsi_unload(void) { struct iscsi_session *is, *tmp; if (sc->sc_cdev != NULL) { ISCSI_DEBUG("removing device node"); destroy_dev(sc->sc_cdev); ISCSI_DEBUG("device node removed"); } if (sc->sc_shutdown_eh != NULL) EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh); sx_slock(&sc->sc_lock); TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) iscsi_session_terminate(is); while(!TAILQ_EMPTY(&sc->sc_sessions)) { ISCSI_DEBUG("waiting for sessions to terminate"); cv_wait(&sc->sc_cv, &sc->sc_lock); } ISCSI_DEBUG("all sessions terminated"); sx_sunlock(&sc->sc_lock); uma_zdestroy(iscsi_outstanding_zone); sx_destroy(&sc->sc_lock); cv_destroy(&sc->sc_cv); free(sc, M_ISCSI); return (0); } static int iscsi_quiesce(void) { sx_slock(&sc->sc_lock); if (!TAILQ_EMPTY(&sc->sc_sessions)) { sx_sunlock(&sc->sc_lock); return (EBUSY); } sx_sunlock(&sc->sc_lock); return (0); } static int iscsi_modevent(module_t mod, int what, void *arg) { int error; switch (what) { case MOD_LOAD: error = iscsi_load(); break; case MOD_UNLOAD: error = iscsi_unload(); break; case MOD_QUIESCE: error = iscsi_quiesce(); break; default: error = EINVAL; break; } return (error); } moduledata_t iscsi_data = { "iscsi", iscsi_modevent, 0 }; DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); MODULE_DEPEND(iscsi, cam, 1, 1, 1); MODULE_DEPEND(iscsi, icl, 1, 1, 1); diff --git a/sys/dev/usb/input/uhid.c b/sys/dev/usb/input/uhid.c index 02642a00b8b9..ece5e9573309 100644 --- a/sys/dev/usb/input/uhid.c +++ b/sys/dev/usb/input/uhid.c @@ -1,878 +1,878 @@ /* $NetBSD: uhid.c,v 1.46 2001/11/13 06:24:55 lukem Exp $ */ /* Also already merged from NetBSD: * $NetBSD: uhid.c,v 1.54 2002/09/23 05:51:21 simonb Exp $ */ #include __FBSDID("$FreeBSD$"); /*- * Copyright (c) 1998 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Lennart Augustsson (lennart@augustsson.net) at * Carlstedt Research & Technology. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * HID spec: http://www.usb.org/developers/devclass_docs/HID1_11.pdf */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "usbdevs.h" #include #include #include #include #include #define USB_DEBUG_VAR uhid_debug #include #include #include #ifdef USB_DEBUG static int uhid_debug = 0; static SYSCTL_NODE(_hw_usb, OID_AUTO, uhid, CTLFLAG_RW, 0, "USB uhid"); SYSCTL_INT(_hw_usb_uhid, OID_AUTO, debug, CTLFLAG_RWTUN, &uhid_debug, 0, "Debug level"); #endif #define UHID_BSIZE 1024 /* bytes, buffer size */ #define UHID_FRAME_NUM 50 /* bytes, frame number */ enum { UHID_INTR_DT_WR, UHID_INTR_DT_RD, UHID_CTRL_DT_WR, UHID_CTRL_DT_RD, UHID_N_TRANSFER, }; struct uhid_softc { struct usb_fifo_sc sc_fifo; struct mtx sc_mtx; struct usb_xfer *sc_xfer[UHID_N_TRANSFER]; struct usb_device *sc_udev; void *sc_repdesc_ptr; uint32_t sc_isize; uint32_t sc_osize; uint32_t sc_fsize; uint16_t sc_repdesc_size; uint8_t sc_iface_no; uint8_t sc_iface_index; uint8_t sc_iid; uint8_t sc_oid; uint8_t sc_fid; uint8_t sc_flags; #define UHID_FLAG_IMMED 0x01 /* set if read should be immediate */ #define UHID_FLAG_STATIC_DESC 0x04 /* set if report descriptors are * static */ }; static const uint8_t uhid_xb360gp_report_descr[] = {UHID_XB360GP_REPORT_DESCR()}; static const uint8_t uhid_graphire_report_descr[] = {UHID_GRAPHIRE_REPORT_DESCR()}; static const uint8_t uhid_graphire3_4x5_report_descr[] = {UHID_GRAPHIRE3_4X5_REPORT_DESCR()}; /* prototypes */ static device_probe_t uhid_probe; static device_attach_t uhid_attach; static device_detach_t uhid_detach; static usb_callback_t uhid_intr_write_callback; static usb_callback_t uhid_intr_read_callback; static usb_callback_t uhid_write_callback; static usb_callback_t uhid_read_callback; static usb_fifo_cmd_t uhid_start_read; static usb_fifo_cmd_t uhid_stop_read; static usb_fifo_cmd_t uhid_start_write; static usb_fifo_cmd_t uhid_stop_write; static usb_fifo_open_t uhid_open; static usb_fifo_close_t uhid_close; static usb_fifo_ioctl_t uhid_ioctl; static struct usb_fifo_methods uhid_fifo_methods = { .f_open = &uhid_open, .f_close = &uhid_close, .f_ioctl = &uhid_ioctl, .f_start_read = &uhid_start_read, .f_stop_read = &uhid_stop_read, .f_start_write = &uhid_start_write, .f_stop_write = &uhid_stop_write, .basename[0] = "uhid", }; static void uhid_intr_write_callback(struct usb_xfer *xfer, usb_error_t error) { struct uhid_softc *sc = usbd_xfer_softc(xfer); struct usb_page_cache *pc; int actlen; switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: case USB_ST_SETUP: tr_setup: pc = usbd_xfer_get_frame(xfer, 0); if (usb_fifo_get_data(sc->sc_fifo.fp[USB_FIFO_TX], pc, 0, usbd_xfer_max_len(xfer), &actlen, 0)) { usbd_xfer_set_frame_len(xfer, 0, actlen); usbd_transfer_submit(xfer); } return; default: /* Error */ if (error != USB_ERR_CANCELLED) { /* try to clear stall first */ usbd_xfer_set_stall(xfer); goto tr_setup; } return; } } static void uhid_intr_read_callback(struct usb_xfer *xfer, usb_error_t error) { struct uhid_softc *sc = usbd_xfer_softc(xfer); struct usb_page_cache *pc; int actlen; usbd_xfer_status(xfer, &actlen, NULL, NULL, NULL); switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: DPRINTF("transferred!\n"); pc = usbd_xfer_get_frame(xfer, 0); /* * If the ID byte is non zero we allow descriptors * having multiple sizes: */ if ((actlen >= (int)sc->sc_isize) || ((actlen > 0) && (sc->sc_iid != 0))) { /* limit report length to the maximum */ if (actlen > (int)sc->sc_isize) actlen = sc->sc_isize; usb_fifo_put_data(sc->sc_fifo.fp[USB_FIFO_RX], pc, 0, actlen, 1); } else { /* ignore it */ DPRINTF("ignored transfer, %d bytes\n", actlen); } case USB_ST_SETUP: re_submit: if (usb_fifo_put_bytes_max( sc->sc_fifo.fp[USB_FIFO_RX]) != 0) { usbd_xfer_set_frame_len(xfer, 0, sc->sc_isize); usbd_transfer_submit(xfer); } return; default: /* Error */ if (error != USB_ERR_CANCELLED) { /* try to clear stall first */ usbd_xfer_set_stall(xfer); goto re_submit; } return; } } static void uhid_fill_set_report(struct usb_device_request *req, uint8_t iface_no, uint8_t type, uint8_t id, uint16_t size) { req->bmRequestType = UT_WRITE_CLASS_INTERFACE; req->bRequest = UR_SET_REPORT; USETW2(req->wValue, type, id); req->wIndex[0] = iface_no; req->wIndex[1] = 0; USETW(req->wLength, size); } static void uhid_fill_get_report(struct usb_device_request *req, uint8_t iface_no, uint8_t type, uint8_t id, uint16_t size) { req->bmRequestType = UT_READ_CLASS_INTERFACE; req->bRequest = UR_GET_REPORT; USETW2(req->wValue, type, id); req->wIndex[0] = iface_no; req->wIndex[1] = 0; USETW(req->wLength, size); } static void uhid_write_callback(struct usb_xfer *xfer, usb_error_t error) { struct uhid_softc *sc = usbd_xfer_softc(xfer); struct usb_device_request req; struct usb_page_cache *pc; uint32_t size = sc->sc_osize; uint32_t actlen; uint8_t id; switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: case USB_ST_SETUP: /* try to extract the ID byte */ if (sc->sc_oid) { pc = usbd_xfer_get_frame(xfer, 0); if (usb_fifo_get_data(sc->sc_fifo.fp[USB_FIFO_TX], pc, 0, 1, &actlen, 0)) { if (actlen != 1) { goto tr_error; } usbd_copy_out(pc, 0, &id, 1); } else { return; } if (size) { size--; } } else { id = 0; } pc = usbd_xfer_get_frame(xfer, 1); if (usb_fifo_get_data(sc->sc_fifo.fp[USB_FIFO_TX], pc, 0, UHID_BSIZE, &actlen, 1)) { if (actlen != size) { goto tr_error; } uhid_fill_set_report (&req, sc->sc_iface_no, UHID_OUTPUT_REPORT, id, size); pc = usbd_xfer_get_frame(xfer, 0); usbd_copy_in(pc, 0, &req, sizeof(req)); usbd_xfer_set_frame_len(xfer, 0, sizeof(req)); usbd_xfer_set_frame_len(xfer, 1, size); usbd_xfer_set_frames(xfer, size ? 2 : 1); usbd_transfer_submit(xfer); } return; default: tr_error: /* bomb out */ usb_fifo_get_data_error(sc->sc_fifo.fp[USB_FIFO_TX]); return; } } static void uhid_read_callback(struct usb_xfer *xfer, usb_error_t error) { struct uhid_softc *sc = usbd_xfer_softc(xfer); struct usb_device_request req; struct usb_page_cache *pc; pc = usbd_xfer_get_frame(xfer, 0); switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: usb_fifo_put_data(sc->sc_fifo.fp[USB_FIFO_RX], pc, sizeof(req), sc->sc_isize, 1); return; case USB_ST_SETUP: if (usb_fifo_put_bytes_max(sc->sc_fifo.fp[USB_FIFO_RX]) > 0) { uhid_fill_get_report (&req, sc->sc_iface_no, UHID_INPUT_REPORT, sc->sc_iid, sc->sc_isize); usbd_copy_in(pc, 0, &req, sizeof(req)); usbd_xfer_set_frame_len(xfer, 0, sizeof(req)); usbd_xfer_set_frame_len(xfer, 1, sc->sc_isize); usbd_xfer_set_frames(xfer, sc->sc_isize ? 2 : 1); usbd_transfer_submit(xfer); } return; default: /* Error */ /* bomb out */ usb_fifo_put_data_error(sc->sc_fifo.fp[USB_FIFO_RX]); return; } } static const struct usb_config uhid_config[UHID_N_TRANSFER] = { [UHID_INTR_DT_WR] = { .type = UE_INTERRUPT, .endpoint = UE_ADDR_ANY, .direction = UE_DIR_OUT, .flags = {.pipe_bof = 1,.no_pipe_ok = 1, }, .bufsize = UHID_BSIZE, .callback = &uhid_intr_write_callback, }, [UHID_INTR_DT_RD] = { .type = UE_INTERRUPT, .endpoint = UE_ADDR_ANY, .direction = UE_DIR_IN, .flags = {.pipe_bof = 1,.short_xfer_ok = 1,}, .bufsize = UHID_BSIZE, .callback = &uhid_intr_read_callback, }, [UHID_CTRL_DT_WR] = { .type = UE_CONTROL, .endpoint = 0x00, /* Control pipe */ .direction = UE_DIR_ANY, .bufsize = sizeof(struct usb_device_request) + UHID_BSIZE, .callback = &uhid_write_callback, .timeout = 1000, /* 1 second */ }, [UHID_CTRL_DT_RD] = { .type = UE_CONTROL, .endpoint = 0x00, /* Control pipe */ .direction = UE_DIR_ANY, .bufsize = sizeof(struct usb_device_request) + UHID_BSIZE, .callback = &uhid_read_callback, .timeout = 1000, /* 1 second */ }, }; static void uhid_start_read(struct usb_fifo *fifo) { struct uhid_softc *sc = usb_fifo_softc(fifo); if (sc->sc_flags & UHID_FLAG_IMMED) { usbd_transfer_start(sc->sc_xfer[UHID_CTRL_DT_RD]); } else { usbd_transfer_start(sc->sc_xfer[UHID_INTR_DT_RD]); } } static void uhid_stop_read(struct usb_fifo *fifo) { struct uhid_softc *sc = usb_fifo_softc(fifo); usbd_transfer_stop(sc->sc_xfer[UHID_CTRL_DT_RD]); usbd_transfer_stop(sc->sc_xfer[UHID_INTR_DT_RD]); } static void uhid_start_write(struct usb_fifo *fifo) { struct uhid_softc *sc = usb_fifo_softc(fifo); if ((sc->sc_flags & UHID_FLAG_IMMED) || sc->sc_xfer[UHID_INTR_DT_WR] == NULL) { usbd_transfer_start(sc->sc_xfer[UHID_CTRL_DT_WR]); } else { usbd_transfer_start(sc->sc_xfer[UHID_INTR_DT_WR]); } } static void uhid_stop_write(struct usb_fifo *fifo) { struct uhid_softc *sc = usb_fifo_softc(fifo); usbd_transfer_stop(sc->sc_xfer[UHID_CTRL_DT_WR]); usbd_transfer_stop(sc->sc_xfer[UHID_INTR_DT_WR]); } static int uhid_get_report(struct uhid_softc *sc, uint8_t type, uint8_t id, void *kern_data, void *user_data, uint16_t len) { int err; uint8_t free_data = 0; if (kern_data == NULL) { kern_data = malloc(len, M_USBDEV, M_WAITOK); if (kern_data == NULL) { err = ENOMEM; goto done; } free_data = 1; } err = usbd_req_get_report(sc->sc_udev, NULL, kern_data, len, sc->sc_iface_index, type, id); if (err) { err = ENXIO; goto done; } if (user_data) { /* dummy buffer */ err = copyout(kern_data, user_data, len); if (err) { goto done; } } done: if (free_data) { free(kern_data, M_USBDEV); } return (err); } static int uhid_set_report(struct uhid_softc *sc, uint8_t type, uint8_t id, void *kern_data, void *user_data, uint16_t len) { int err; uint8_t free_data = 0; if (kern_data == NULL) { kern_data = malloc(len, M_USBDEV, M_WAITOK); if (kern_data == NULL) { err = ENOMEM; goto done; } free_data = 1; err = copyin(user_data, kern_data, len); if (err) { goto done; } } err = usbd_req_set_report(sc->sc_udev, NULL, kern_data, len, sc->sc_iface_index, type, id); if (err) { err = ENXIO; goto done; } done: if (free_data) { free(kern_data, M_USBDEV); } return (err); } static int uhid_open(struct usb_fifo *fifo, int fflags) { struct uhid_softc *sc = usb_fifo_softc(fifo); /* * The buffers are one byte larger than maximum so that one * can detect too large read/writes and short transfers: */ if (fflags & FREAD) { /* reset flags */ sc->sc_flags &= ~UHID_FLAG_IMMED; if (usb_fifo_alloc_buffer(fifo, sc->sc_isize + 1, UHID_FRAME_NUM)) { return (ENOMEM); } } if (fflags & FWRITE) { if (usb_fifo_alloc_buffer(fifo, sc->sc_osize + 1, UHID_FRAME_NUM)) { return (ENOMEM); } } return (0); } static void uhid_close(struct usb_fifo *fifo, int fflags) { if (fflags & (FREAD | FWRITE)) { usb_fifo_free_buffer(fifo); } } static int uhid_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags) { struct uhid_softc *sc = usb_fifo_softc(fifo); struct usb_gen_descriptor *ugd; uint32_t size; int error = 0; uint8_t id; switch (cmd) { case USB_GET_REPORT_DESC: ugd = addr; if (sc->sc_repdesc_size > ugd->ugd_maxlen) { size = ugd->ugd_maxlen; } else { size = sc->sc_repdesc_size; } ugd->ugd_actlen = size; if (ugd->ugd_data == NULL) break; /* descriptor length only */ error = copyout(sc->sc_repdesc_ptr, ugd->ugd_data, size); break; case USB_SET_IMMED: if (!(fflags & FREAD)) { error = EPERM; break; } if (*(int *)addr) { /* do a test read */ error = uhid_get_report(sc, UHID_INPUT_REPORT, sc->sc_iid, NULL, NULL, sc->sc_isize); if (error) { break; } mtx_lock(&sc->sc_mtx); sc->sc_flags |= UHID_FLAG_IMMED; mtx_unlock(&sc->sc_mtx); } else { mtx_lock(&sc->sc_mtx); sc->sc_flags &= ~UHID_FLAG_IMMED; mtx_unlock(&sc->sc_mtx); } break; case USB_GET_REPORT: if (!(fflags & FREAD)) { error = EPERM; break; } ugd = addr; switch (ugd->ugd_report_type) { case UHID_INPUT_REPORT: size = sc->sc_isize; id = sc->sc_iid; break; case UHID_OUTPUT_REPORT: size = sc->sc_osize; id = sc->sc_oid; break; case UHID_FEATURE_REPORT: size = sc->sc_fsize; id = sc->sc_fid; break; default: return (EINVAL); } if (id != 0) copyin(ugd->ugd_data, &id, 1); error = uhid_get_report(sc, ugd->ugd_report_type, id, NULL, ugd->ugd_data, imin(ugd->ugd_maxlen, size)); break; case USB_SET_REPORT: if (!(fflags & FWRITE)) { error = EPERM; break; } ugd = addr; switch (ugd->ugd_report_type) { case UHID_INPUT_REPORT: size = sc->sc_isize; id = sc->sc_iid; break; case UHID_OUTPUT_REPORT: size = sc->sc_osize; id = sc->sc_oid; break; case UHID_FEATURE_REPORT: size = sc->sc_fsize; id = sc->sc_fid; break; default: return (EINVAL); } if (id != 0) copyin(ugd->ugd_data, &id, 1); error = uhid_set_report(sc, ugd->ugd_report_type, id, NULL, ugd->ugd_data, imin(ugd->ugd_maxlen, size)); break; case USB_GET_REPORT_ID: *(int *)addr = 0; /* XXX: we only support reportid 0? */ break; default: error = EINVAL; break; } return (error); } static const STRUCT_USB_HOST_ID uhid_devs[] = { /* generic HID class */ {USB_IFACE_CLASS(UICLASS_HID),}, /* the Xbox 360 gamepad doesn't use the HID class */ {USB_IFACE_CLASS(UICLASS_VENDOR), USB_IFACE_SUBCLASS(UISUBCLASS_XBOX360_CONTROLLER), USB_IFACE_PROTOCOL(UIPROTO_XBOX360_GAMEPAD),}, }; static int uhid_probe(device_t dev) { struct usb_attach_arg *uaa = device_get_ivars(dev); int error; DPRINTFN(11, "\n"); if (uaa->usb_mode != USB_MODE_HOST) return (ENXIO); error = usbd_lookup_id_by_uaa(uhid_devs, sizeof(uhid_devs), uaa); if (error) return (error); if (usb_test_quirk(uaa, UQ_HID_IGNORE)) return (ENXIO); /* * Don't attach to mouse and keyboard devices, hence then no * "nomatch" event is generated and then ums and ukbd won't * attach properly when loaded. */ if ((uaa->info.bInterfaceClass == UICLASS_HID) && (uaa->info.bInterfaceSubClass == UISUBCLASS_BOOT) && (((uaa->info.bInterfaceProtocol == UIPROTO_BOOT_KEYBOARD) && !usb_test_quirk(uaa, UQ_KBD_IGNORE)) || ((uaa->info.bInterfaceProtocol == UIPROTO_MOUSE) && !usb_test_quirk(uaa, UQ_UMS_IGNORE)))) return (ENXIO); return (BUS_PROBE_GENERIC); } static int uhid_attach(device_t dev) { struct usb_attach_arg *uaa = device_get_ivars(dev); struct uhid_softc *sc = device_get_softc(dev); int unit = device_get_unit(dev); int error = 0; DPRINTFN(10, "sc=%p\n", sc); device_set_usb_desc(dev); mtx_init(&sc->sc_mtx, "uhid lock", NULL, MTX_DEF | MTX_RECURSE); sc->sc_udev = uaa->device; sc->sc_iface_no = uaa->info.bIfaceNum; sc->sc_iface_index = uaa->info.bIfaceIndex; error = usbd_transfer_setup(uaa->device, &uaa->info.bIfaceIndex, sc->sc_xfer, uhid_config, UHID_N_TRANSFER, sc, &sc->sc_mtx); if (error) { DPRINTF("error=%s\n", usbd_errstr(error)); goto detach; } if (uaa->info.idVendor == USB_VENDOR_WACOM) { /* the report descriptor for the Wacom Graphire is broken */ if (uaa->info.idProduct == USB_PRODUCT_WACOM_GRAPHIRE) { sc->sc_repdesc_size = sizeof(uhid_graphire_report_descr); - sc->sc_repdesc_ptr = (void *)&uhid_graphire_report_descr; + sc->sc_repdesc_ptr = __DECONST(void *, &uhid_graphire_report_descr); sc->sc_flags |= UHID_FLAG_STATIC_DESC; } else if (uaa->info.idProduct == USB_PRODUCT_WACOM_GRAPHIRE3_4X5) { static uint8_t reportbuf[] = {2, 2, 2}; /* * The Graphire3 needs 0x0202 to be written to * feature report ID 2 before it'll start * returning digitizer data. */ error = usbd_req_set_report(uaa->device, NULL, reportbuf, sizeof(reportbuf), uaa->info.bIfaceIndex, UHID_FEATURE_REPORT, 2); if (error) { DPRINTF("set report failed, error=%s (ignored)\n", usbd_errstr(error)); } sc->sc_repdesc_size = sizeof(uhid_graphire3_4x5_report_descr); - sc->sc_repdesc_ptr = (void *)&uhid_graphire3_4x5_report_descr; + sc->sc_repdesc_ptr = __DECONST(void *, &uhid_graphire3_4x5_report_descr); sc->sc_flags |= UHID_FLAG_STATIC_DESC; } } else if ((uaa->info.bInterfaceClass == UICLASS_VENDOR) && (uaa->info.bInterfaceSubClass == UISUBCLASS_XBOX360_CONTROLLER) && (uaa->info.bInterfaceProtocol == UIPROTO_XBOX360_GAMEPAD)) { static const uint8_t reportbuf[3] = {1, 3, 0}; /* * Turn off the four LEDs on the gamepad which * are blinking by default: */ error = usbd_req_set_report(uaa->device, NULL, __DECONST(void *, reportbuf), sizeof(reportbuf), uaa->info.bIfaceIndex, UHID_OUTPUT_REPORT, 0); if (error) { DPRINTF("set output report failed, error=%s (ignored)\n", usbd_errstr(error)); } /* the Xbox 360 gamepad has no report descriptor */ sc->sc_repdesc_size = sizeof(uhid_xb360gp_report_descr); - sc->sc_repdesc_ptr = (void *)&uhid_xb360gp_report_descr; + sc->sc_repdesc_ptr = __DECONST(void *, &uhid_xb360gp_report_descr); sc->sc_flags |= UHID_FLAG_STATIC_DESC; } if (sc->sc_repdesc_ptr == NULL) { error = usbd_req_get_hid_desc(uaa->device, NULL, &sc->sc_repdesc_ptr, &sc->sc_repdesc_size, M_USBDEV, uaa->info.bIfaceIndex); if (error) { device_printf(dev, "no report descriptor\n"); goto detach; } } error = usbd_req_set_idle(uaa->device, NULL, uaa->info.bIfaceIndex, 0, 0); if (error) { DPRINTF("set idle failed, error=%s (ignored)\n", usbd_errstr(error)); } sc->sc_isize = hid_report_size (sc->sc_repdesc_ptr, sc->sc_repdesc_size, hid_input, &sc->sc_iid); sc->sc_osize = hid_report_size (sc->sc_repdesc_ptr, sc->sc_repdesc_size, hid_output, &sc->sc_oid); sc->sc_fsize = hid_report_size (sc->sc_repdesc_ptr, sc->sc_repdesc_size, hid_feature, &sc->sc_fid); if (sc->sc_isize > UHID_BSIZE) { DPRINTF("input size is too large, " "%d bytes (truncating)\n", sc->sc_isize); sc->sc_isize = UHID_BSIZE; } if (sc->sc_osize > UHID_BSIZE) { DPRINTF("output size is too large, " "%d bytes (truncating)\n", sc->sc_osize); sc->sc_osize = UHID_BSIZE; } if (sc->sc_fsize > UHID_BSIZE) { DPRINTF("feature size is too large, " "%d bytes (truncating)\n", sc->sc_fsize); sc->sc_fsize = UHID_BSIZE; } error = usb_fifo_attach(uaa->device, sc, &sc->sc_mtx, &uhid_fifo_methods, &sc->sc_fifo, unit, -1, uaa->info.bIfaceIndex, UID_ROOT, GID_OPERATOR, 0644); if (error) { goto detach; } return (0); /* success */ detach: uhid_detach(dev); return (ENOMEM); } static int uhid_detach(device_t dev) { struct uhid_softc *sc = device_get_softc(dev); usb_fifo_detach(&sc->sc_fifo); usbd_transfer_unsetup(sc->sc_xfer, UHID_N_TRANSFER); if (sc->sc_repdesc_ptr) { if (!(sc->sc_flags & UHID_FLAG_STATIC_DESC)) { free(sc->sc_repdesc_ptr, M_USBDEV); } } mtx_destroy(&sc->sc_mtx); return (0); } static devclass_t uhid_devclass; static device_method_t uhid_methods[] = { DEVMETHOD(device_probe, uhid_probe), DEVMETHOD(device_attach, uhid_attach), DEVMETHOD(device_detach, uhid_detach), DEVMETHOD_END }; static driver_t uhid_driver = { .name = "uhid", .methods = uhid_methods, .size = sizeof(struct uhid_softc), }; DRIVER_MODULE(uhid, uhub, uhid_driver, uhid_devclass, NULL, 0); MODULE_DEPEND(uhid, usb, 1, 1, 1); MODULE_VERSION(uhid, 1); diff --git a/sys/fs/tmpfs/tmpfs_subr.c b/sys/fs/tmpfs/tmpfs_subr.c index c1930f159ca4..af0a8af29050 100644 --- a/sys/fs/tmpfs/tmpfs_subr.c +++ b/sys/fs/tmpfs/tmpfs_subr.c @@ -1,1811 +1,1812 @@ /* $NetBSD: tmpfs_subr.c,v 1.35 2007/07/09 21:10:50 ad Exp $ */ /*- * Copyright (c) 2005 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Julio M. Merino Vidal, developed as part of Google's Summer of Code * 2005 program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Efficient memory file system supporting functions. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct tmpfs_dir_cursor { struct tmpfs_dirent *tdc_current; struct tmpfs_dirent *tdc_tree; }; SYSCTL_NODE(_vfs, OID_AUTO, tmpfs, CTLFLAG_RW, 0, "tmpfs file system"); static long tmpfs_pages_reserved = TMPFS_PAGES_MINRESERVED; static int sysctl_mem_reserved(SYSCTL_HANDLER_ARGS) { int error; long pages, bytes; pages = *(long *)arg1; bytes = pages * PAGE_SIZE; error = sysctl_handle_long(oidp, &bytes, 0, req); if (error || !req->newptr) return (error); pages = bytes / PAGE_SIZE; if (pages < TMPFS_PAGES_MINRESERVED) return (EINVAL); *(long *)arg1 = pages; return (0); } SYSCTL_PROC(_vfs_tmpfs, OID_AUTO, memory_reserved, CTLTYPE_LONG|CTLFLAG_RW, &tmpfs_pages_reserved, 0, sysctl_mem_reserved, "L", "Amount of available memory and swap below which tmpfs growth stops"); static __inline int tmpfs_dirtree_cmp(struct tmpfs_dirent *a, struct tmpfs_dirent *b); RB_PROTOTYPE_STATIC(tmpfs_dir, tmpfs_dirent, uh.td_entries, tmpfs_dirtree_cmp); size_t tmpfs_mem_avail(void) { vm_ooffset_t avail; avail = swap_pager_avail + vm_cnt.v_free_count + vm_cnt.v_cache_count - tmpfs_pages_reserved; if (__predict_false(avail < 0)) avail = 0; return (avail); } size_t tmpfs_pages_used(struct tmpfs_mount *tmp) { const size_t node_size = sizeof(struct tmpfs_node) + sizeof(struct tmpfs_dirent); size_t meta_pages; meta_pages = howmany((uintmax_t)tmp->tm_nodes_inuse * node_size, PAGE_SIZE); return (meta_pages + tmp->tm_pages_used); } static size_t tmpfs_pages_check_avail(struct tmpfs_mount *tmp, size_t req_pages) { if (tmpfs_mem_avail() < req_pages) return (0); if (tmp->tm_pages_max != SIZE_MAX && tmp->tm_pages_max < req_pages + tmpfs_pages_used(tmp)) return (0); return (1); } /* * Allocates a new node of type 'type' inside the 'tmp' mount point, with * its owner set to 'uid', its group to 'gid' and its mode set to 'mode', * using the credentials of the process 'p'. * * If the node type is set to 'VDIR', then the parent parameter must point * to the parent directory of the node being created. It may only be NULL * while allocating the root node. * * If the node type is set to 'VBLK' or 'VCHR', then the rdev parameter * specifies the device the node represents. * * If the node type is set to 'VLNK', then the parameter target specifies * the file name of the target file for the symbolic link that is being * created. * * Note that new nodes are retrieved from the available list if it has * items or, if it is empty, from the node pool as long as there is enough * space to create them. * * Returns zero on success or an appropriate error code on failure. */ int tmpfs_alloc_node(struct mount *mp, struct tmpfs_mount *tmp, enum vtype type, uid_t uid, gid_t gid, mode_t mode, struct tmpfs_node *parent, char *target, dev_t rdev, struct tmpfs_node **node) { struct tmpfs_node *nnode; vm_object_t obj; /* If the root directory of the 'tmp' file system is not yet * allocated, this must be the request to do it. */ MPASS(IMPLIES(tmp->tm_root == NULL, parent == NULL && type == VDIR)); KASSERT(tmp->tm_root == NULL || mp->mnt_writeopcount > 0, ("creating node not under vn_start_write")); MPASS(IFF(type == VLNK, target != NULL)); MPASS(IFF(type == VBLK || type == VCHR, rdev != VNOVAL)); if (tmp->tm_nodes_inuse >= tmp->tm_nodes_max) return (ENOSPC); if (tmpfs_pages_check_avail(tmp, 1) == 0) return (ENOSPC); if ((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) { /* * When a new tmpfs node is created for fully * constructed mount point, there must be a parent * node, which vnode is locked exclusively. As * consequence, if the unmount is executing in * parallel, vflush() cannot reclaim the parent vnode. * Due to this, the check for MNTK_UNMOUNT flag is not * racy: if we did not see MNTK_UNMOUNT flag, then tmp * cannot be destroyed until node construction is * finished and the parent vnode unlocked. * * Tmpfs does not need to instantiate new nodes during * unmount. */ return (EBUSY); } nnode = (struct tmpfs_node *)uma_zalloc_arg( tmp->tm_node_pool, tmp, M_WAITOK); /* Generic initialization. */ nnode->tn_type = type; vfs_timestamp(&nnode->tn_atime); nnode->tn_birthtime = nnode->tn_ctime = nnode->tn_mtime = nnode->tn_atime; nnode->tn_uid = uid; nnode->tn_gid = gid; nnode->tn_mode = mode; nnode->tn_id = alloc_unr(tmp->tm_ino_unr); /* Type-specific initialization. */ switch (nnode->tn_type) { case VBLK: case VCHR: nnode->tn_rdev = rdev; break; case VDIR: RB_INIT(&nnode->tn_dir.tn_dirhead); LIST_INIT(&nnode->tn_dir.tn_dupindex); MPASS(parent != nnode); MPASS(IMPLIES(parent == NULL, tmp->tm_root == NULL)); nnode->tn_dir.tn_parent = (parent == NULL) ? nnode : parent; nnode->tn_dir.tn_readdir_lastn = 0; nnode->tn_dir.tn_readdir_lastp = NULL; nnode->tn_links++; TMPFS_NODE_LOCK(nnode->tn_dir.tn_parent); nnode->tn_dir.tn_parent->tn_links++; TMPFS_NODE_UNLOCK(nnode->tn_dir.tn_parent); break; case VFIFO: /* FALLTHROUGH */ case VSOCK: break; case VLNK: MPASS(strlen(target) < MAXPATHLEN); nnode->tn_size = strlen(target); nnode->tn_link = malloc(nnode->tn_size, M_TMPFSNAME, M_WAITOK); memcpy(nnode->tn_link, target, nnode->tn_size); break; case VREG: obj = nnode->tn_reg.tn_aobj = vm_pager_allocate(OBJT_SWAP, NULL, 0, VM_PROT_DEFAULT, 0, NULL /* XXXKIB - tmpfs needs swap reservation */); VM_OBJECT_WLOCK(obj); /* OBJ_TMPFS is set together with the setting of vp->v_object */ vm_object_set_flag(obj, OBJ_NOSPLIT | OBJ_TMPFS_NODE); vm_object_clear_flag(obj, OBJ_ONEMAPPING); VM_OBJECT_WUNLOCK(obj); break; default: panic("tmpfs_alloc_node: type %p %d", nnode, (int)nnode->tn_type); } TMPFS_LOCK(tmp); LIST_INSERT_HEAD(&tmp->tm_nodes_used, nnode, tn_entries); tmp->tm_nodes_inuse++; TMPFS_UNLOCK(tmp); *node = nnode; return 0; } /* * Destroys the node pointed to by node from the file system 'tmp'. * If the node does not belong to the given mount point, the results are * unpredicted. * * If the node references a directory; no entries are allowed because * their removal could need a recursive algorithm, something forbidden in * kernel space. Furthermore, there is not need to provide such * functionality (recursive removal) because the only primitives offered * to the user are the removal of empty directories and the deletion of * individual files. * * Note that nodes are not really deleted; in fact, when a node has been * allocated, it cannot be deleted during the whole life of the file * system. Instead, they are moved to the available list and remain there * until reused. */ void tmpfs_free_node(struct tmpfs_mount *tmp, struct tmpfs_node *node) { vm_object_t uobj; #ifdef INVARIANTS TMPFS_NODE_LOCK(node); MPASS(node->tn_vnode == NULL); MPASS((node->tn_vpstate & TMPFS_VNODE_ALLOCATING) == 0); TMPFS_NODE_UNLOCK(node); #endif TMPFS_LOCK(tmp); LIST_REMOVE(node, tn_entries); tmp->tm_nodes_inuse--; TMPFS_UNLOCK(tmp); switch (node->tn_type) { case VNON: /* Do not do anything. VNON is provided to let the * allocation routine clean itself easily by avoiding * duplicating code in it. */ /* FALLTHROUGH */ case VBLK: /* FALLTHROUGH */ case VCHR: /* FALLTHROUGH */ case VDIR: /* FALLTHROUGH */ case VFIFO: /* FALLTHROUGH */ case VSOCK: break; case VLNK: free(node->tn_link, M_TMPFSNAME); break; case VREG: uobj = node->tn_reg.tn_aobj; if (uobj != NULL) { TMPFS_LOCK(tmp); tmp->tm_pages_used -= uobj->size; TMPFS_UNLOCK(tmp); KASSERT((uobj->flags & OBJ_TMPFS) == 0, ("leaked OBJ_TMPFS node %p vm_obj %p", node, uobj)); vm_object_deallocate(uobj); } break; default: panic("tmpfs_free_node: type %p %d", node, (int)node->tn_type); } free_unr(tmp->tm_ino_unr, node->tn_id); uma_zfree(tmp->tm_node_pool, node); } static __inline uint32_t tmpfs_dirent_hash(const char *name, u_int len) { uint32_t hash; hash = fnv_32_buf(name, len, FNV1_32_INIT + len) & TMPFS_DIRCOOKIE_MASK; #ifdef TMPFS_DEBUG_DIRCOOKIE_DUP hash &= 0xf; #endif if (hash < TMPFS_DIRCOOKIE_MIN) hash += TMPFS_DIRCOOKIE_MIN; return (hash); } static __inline off_t tmpfs_dirent_cookie(struct tmpfs_dirent *de) { if (de == NULL) return (TMPFS_DIRCOOKIE_EOF); MPASS(de->td_cookie >= TMPFS_DIRCOOKIE_MIN); return (de->td_cookie); } static __inline boolean_t tmpfs_dirent_dup(struct tmpfs_dirent *de) { return ((de->td_cookie & TMPFS_DIRCOOKIE_DUP) != 0); } static __inline boolean_t tmpfs_dirent_duphead(struct tmpfs_dirent *de) { return ((de->td_cookie & TMPFS_DIRCOOKIE_DUPHEAD) != 0); } void tmpfs_dirent_init(struct tmpfs_dirent *de, const char *name, u_int namelen) { de->td_hash = de->td_cookie = tmpfs_dirent_hash(name, namelen); memcpy(de->ud.td_name, name, namelen); de->td_namelen = namelen; } /* * Allocates a new directory entry for the node node with a name of name. * The new directory entry is returned in *de. * * The link count of node is increased by one to reflect the new object * referencing it. * * Returns zero on success or an appropriate error code on failure. */ int tmpfs_alloc_dirent(struct tmpfs_mount *tmp, struct tmpfs_node *node, const char *name, u_int len, struct tmpfs_dirent **de) { struct tmpfs_dirent *nde; nde = uma_zalloc(tmp->tm_dirent_pool, M_WAITOK); nde->td_node = node; if (name != NULL) { nde->ud.td_name = malloc(len, M_TMPFSNAME, M_WAITOK); tmpfs_dirent_init(nde, name, len); } else nde->td_namelen = 0; if (node != NULL) node->tn_links++; *de = nde; return 0; } /* * Frees a directory entry. It is the caller's responsibility to destroy * the node referenced by it if needed. * * The link count of node is decreased by one to reflect the removal of an * object that referenced it. This only happens if 'node_exists' is true; * otherwise the function will not access the node referred to by the * directory entry, as it may already have been released from the outside. */ void tmpfs_free_dirent(struct tmpfs_mount *tmp, struct tmpfs_dirent *de) { struct tmpfs_node *node; node = de->td_node; if (node != NULL) { MPASS(node->tn_links > 0); node->tn_links--; } if (!tmpfs_dirent_duphead(de) && de->ud.td_name != NULL) free(de->ud.td_name, M_TMPFSNAME); uma_zfree(tmp->tm_dirent_pool, de); } void tmpfs_destroy_vobject(struct vnode *vp, vm_object_t obj) { ASSERT_VOP_ELOCKED(vp, "tmpfs_destroy_vobject"); if (vp->v_type != VREG || obj == NULL) return; VM_OBJECT_WLOCK(obj); VI_LOCK(vp); vm_object_clear_flag(obj, OBJ_TMPFS); obj->un_pager.swp.swp_tmpfs = NULL; VI_UNLOCK(vp); VM_OBJECT_WUNLOCK(obj); } /* * Need to clear v_object for insmntque failure. */ static void tmpfs_insmntque_dtr(struct vnode *vp, void *dtr_arg) { tmpfs_destroy_vobject(vp, vp->v_object); vp->v_object = NULL; vp->v_data = NULL; vp->v_op = &dead_vnodeops; vgone(vp); vput(vp); } /* * Allocates a new vnode for the node node or returns a new reference to * an existing one if the node had already a vnode referencing it. The * resulting locked vnode is returned in *vpp. * * Returns zero on success or an appropriate error code on failure. */ int tmpfs_alloc_vp(struct mount *mp, struct tmpfs_node *node, int lkflag, struct vnode **vpp) { struct vnode *vp; vm_object_t object; int error; error = 0; loop: TMPFS_NODE_LOCK(node); loop1: if ((vp = node->tn_vnode) != NULL) { MPASS((node->tn_vpstate & TMPFS_VNODE_DOOMED) == 0); VI_LOCK(vp); if ((node->tn_type == VDIR && node->tn_dir.tn_parent == NULL) || ((vp->v_iflag & VI_DOOMED) != 0 && (lkflag & LK_NOWAIT) != 0)) { VI_UNLOCK(vp); TMPFS_NODE_UNLOCK(node); error = ENOENT; vp = NULL; goto out; } if ((vp->v_iflag & VI_DOOMED) != 0) { VI_UNLOCK(vp); node->tn_vpstate |= TMPFS_VNODE_WRECLAIM; while ((node->tn_vpstate & TMPFS_VNODE_WRECLAIM) != 0) { msleep(&node->tn_vnode, TMPFS_NODE_MTX(node), 0, "tmpfsE", 0); } goto loop1; } TMPFS_NODE_UNLOCK(node); error = vget(vp, lkflag | LK_INTERLOCK, curthread); if (error == ENOENT) goto loop; if (error != 0) { vp = NULL; goto out; } /* * Make sure the vnode is still there after * getting the interlock to avoid racing a free. */ if (node->tn_vnode == NULL || node->tn_vnode != vp) { vput(vp); goto loop; } goto out; } if ((node->tn_vpstate & TMPFS_VNODE_DOOMED) || (node->tn_type == VDIR && node->tn_dir.tn_parent == NULL)) { TMPFS_NODE_UNLOCK(node); error = ENOENT; vp = NULL; goto out; } /* * otherwise lock the vp list while we call getnewvnode * since that can block. */ if (node->tn_vpstate & TMPFS_VNODE_ALLOCATING) { node->tn_vpstate |= TMPFS_VNODE_WANT; error = msleep((caddr_t) &node->tn_vpstate, TMPFS_NODE_MTX(node), PDROP | PCATCH, "tmpfs_alloc_vp", 0); if (error) return error; goto loop; } else node->tn_vpstate |= TMPFS_VNODE_ALLOCATING; TMPFS_NODE_UNLOCK(node); /* Get a new vnode and associate it with our node. */ error = getnewvnode("tmpfs", mp, &tmpfs_vnodeop_entries, &vp); if (error != 0) goto unlock; MPASS(vp != NULL); /* lkflag is ignored, the lock is exclusive */ (void) vn_lock(vp, lkflag | LK_RETRY); vp->v_data = node; vp->v_type = node->tn_type; /* Type-specific initialization. */ switch (node->tn_type) { case VBLK: /* FALLTHROUGH */ case VCHR: /* FALLTHROUGH */ case VLNK: /* FALLTHROUGH */ case VSOCK: break; case VFIFO: vp->v_op = &tmpfs_fifoop_entries; break; case VREG: object = node->tn_reg.tn_aobj; VM_OBJECT_WLOCK(object); VI_LOCK(vp); KASSERT(vp->v_object == NULL, ("Not NULL v_object in tmpfs")); vp->v_object = object; object->un_pager.swp.swp_tmpfs = vp; vm_object_set_flag(object, OBJ_TMPFS); VI_UNLOCK(vp); VM_OBJECT_WUNLOCK(object); break; case VDIR: MPASS(node->tn_dir.tn_parent != NULL); if (node->tn_dir.tn_parent == node) vp->v_vflag |= VV_ROOT; break; default: panic("tmpfs_alloc_vp: type %p %d", node, (int)node->tn_type); } if (vp->v_type != VFIFO) VN_LOCK_ASHARE(vp); error = insmntque1(vp, mp, tmpfs_insmntque_dtr, NULL); if (error) vp = NULL; unlock: TMPFS_NODE_LOCK(node); MPASS(node->tn_vpstate & TMPFS_VNODE_ALLOCATING); node->tn_vpstate &= ~TMPFS_VNODE_ALLOCATING; node->tn_vnode = vp; if (node->tn_vpstate & TMPFS_VNODE_WANT) { node->tn_vpstate &= ~TMPFS_VNODE_WANT; TMPFS_NODE_UNLOCK(node); wakeup((caddr_t) &node->tn_vpstate); } else TMPFS_NODE_UNLOCK(node); out: *vpp = vp; #ifdef INVARIANTS if (error == 0) { MPASS(*vpp != NULL && VOP_ISLOCKED(*vpp)); TMPFS_NODE_LOCK(node); MPASS(*vpp == node->tn_vnode); TMPFS_NODE_UNLOCK(node); } #endif return error; } /* * Destroys the association between the vnode vp and the node it * references. */ void tmpfs_free_vp(struct vnode *vp) { struct tmpfs_node *node; node = VP_TO_TMPFS_NODE(vp); TMPFS_NODE_ASSERT_LOCKED(node); node->tn_vnode = NULL; if ((node->tn_vpstate & TMPFS_VNODE_WRECLAIM) != 0) wakeup(&node->tn_vnode); node->tn_vpstate &= ~TMPFS_VNODE_WRECLAIM; vp->v_data = NULL; } /* * Allocates a new file of type 'type' and adds it to the parent directory * 'dvp'; this addition is done using the component name given in 'cnp'. * The ownership of the new file is automatically assigned based on the * credentials of the caller (through 'cnp'), the group is set based on * the parent directory and the mode is determined from the 'vap' argument. * If successful, *vpp holds a vnode to the newly created file and zero * is returned. Otherwise *vpp is NULL and the function returns an * appropriate error code. */ int tmpfs_alloc_file(struct vnode *dvp, struct vnode **vpp, struct vattr *vap, struct componentname *cnp, char *target) { int error; struct tmpfs_dirent *de; struct tmpfs_mount *tmp; struct tmpfs_node *dnode; struct tmpfs_node *node; struct tmpfs_node *parent; MPASS(VOP_ISLOCKED(dvp)); MPASS(cnp->cn_flags & HASBUF); tmp = VFS_TO_TMPFS(dvp->v_mount); dnode = VP_TO_TMPFS_DIR(dvp); *vpp = NULL; /* If the entry we are creating is a directory, we cannot overflow * the number of links of its parent, because it will get a new * link. */ if (vap->va_type == VDIR) { /* Ensure that we do not overflow the maximum number of links * imposed by the system. */ MPASS(dnode->tn_links <= LINK_MAX); if (dnode->tn_links == LINK_MAX) { return (EMLINK); } parent = dnode; MPASS(parent != NULL); } else parent = NULL; /* Allocate a node that represents the new file. */ error = tmpfs_alloc_node(dvp->v_mount, tmp, vap->va_type, cnp->cn_cred->cr_uid, dnode->tn_gid, vap->va_mode, parent, target, vap->va_rdev, &node); if (error != 0) return (error); /* Allocate a directory entry that points to the new file. */ error = tmpfs_alloc_dirent(tmp, node, cnp->cn_nameptr, cnp->cn_namelen, &de); if (error != 0) { tmpfs_free_node(tmp, node); return (error); } /* Allocate a vnode for the new file. */ error = tmpfs_alloc_vp(dvp->v_mount, node, LK_EXCLUSIVE, vpp); if (error != 0) { tmpfs_free_dirent(tmp, de); tmpfs_free_node(tmp, node); return (error); } /* Now that all required items are allocated, we can proceed to * insert the new node into the directory, an operation that * cannot fail. */ if (cnp->cn_flags & ISWHITEOUT) tmpfs_dir_whiteout_remove(dvp, cnp); tmpfs_dir_attach(dvp, de); return (0); } static struct tmpfs_dirent * tmpfs_dir_first(struct tmpfs_node *dnode, struct tmpfs_dir_cursor *dc) { struct tmpfs_dirent *de; de = RB_MIN(tmpfs_dir, &dnode->tn_dir.tn_dirhead); dc->tdc_tree = de; if (de != NULL && tmpfs_dirent_duphead(de)) de = LIST_FIRST(&de->ud.td_duphead); dc->tdc_current = de; return (dc->tdc_current); } static struct tmpfs_dirent * tmpfs_dir_next(struct tmpfs_node *dnode, struct tmpfs_dir_cursor *dc) { struct tmpfs_dirent *de; MPASS(dc->tdc_tree != NULL); if (tmpfs_dirent_dup(dc->tdc_current)) { dc->tdc_current = LIST_NEXT(dc->tdc_current, uh.td_dup.entries); if (dc->tdc_current != NULL) return (dc->tdc_current); } dc->tdc_tree = dc->tdc_current = RB_NEXT(tmpfs_dir, &dnode->tn_dir.tn_dirhead, dc->tdc_tree); if ((de = dc->tdc_current) != NULL && tmpfs_dirent_duphead(de)) { dc->tdc_current = LIST_FIRST(&de->ud.td_duphead); MPASS(dc->tdc_current != NULL); } return (dc->tdc_current); } /* Lookup directory entry in RB-Tree. Function may return duphead entry. */ static struct tmpfs_dirent * tmpfs_dir_xlookup_hash(struct tmpfs_node *dnode, uint32_t hash) { struct tmpfs_dirent *de, dekey; dekey.td_hash = hash; de = RB_FIND(tmpfs_dir, &dnode->tn_dir.tn_dirhead, &dekey); return (de); } /* Lookup directory entry by cookie, initialize directory cursor accordingly. */ static struct tmpfs_dirent * tmpfs_dir_lookup_cookie(struct tmpfs_node *node, off_t cookie, struct tmpfs_dir_cursor *dc) { struct tmpfs_dir *dirhead = &node->tn_dir.tn_dirhead; struct tmpfs_dirent *de, dekey; MPASS(cookie >= TMPFS_DIRCOOKIE_MIN); if (cookie == node->tn_dir.tn_readdir_lastn && (de = node->tn_dir.tn_readdir_lastp) != NULL) { /* Protect against possible race, tn_readdir_last[pn] * may be updated with only shared vnode lock held. */ if (cookie == tmpfs_dirent_cookie(de)) goto out; } if ((cookie & TMPFS_DIRCOOKIE_DUP) != 0) { LIST_FOREACH(de, &node->tn_dir.tn_dupindex, uh.td_dup.index_entries) { MPASS(tmpfs_dirent_dup(de)); if (de->td_cookie == cookie) goto out; /* dupindex list is sorted. */ if (de->td_cookie < cookie) { de = NULL; goto out; } } MPASS(de == NULL); goto out; } MPASS((cookie & TMPFS_DIRCOOKIE_MASK) == cookie); dekey.td_hash = cookie; /* Recover if direntry for cookie was removed */ de = RB_NFIND(tmpfs_dir, dirhead, &dekey); dc->tdc_tree = de; dc->tdc_current = de; if (de != NULL && tmpfs_dirent_duphead(de)) { dc->tdc_current = LIST_FIRST(&de->ud.td_duphead); MPASS(dc->tdc_current != NULL); } return (dc->tdc_current); out: dc->tdc_tree = de; dc->tdc_current = de; if (de != NULL && tmpfs_dirent_dup(de)) dc->tdc_tree = tmpfs_dir_xlookup_hash(node, de->td_hash); return (dc->tdc_current); } /* * Looks for a directory entry in the directory represented by node. * 'cnp' describes the name of the entry to look for. Note that the . * and .. components are not allowed as they do not physically exist * within directories. * * Returns a pointer to the entry when found, otherwise NULL. */ struct tmpfs_dirent * tmpfs_dir_lookup(struct tmpfs_node *node, struct tmpfs_node *f, struct componentname *cnp) { struct tmpfs_dir_duphead *duphead; struct tmpfs_dirent *de; uint32_t hash; MPASS(IMPLIES(cnp->cn_namelen == 1, cnp->cn_nameptr[0] != '.')); MPASS(IMPLIES(cnp->cn_namelen == 2, !(cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.'))); TMPFS_VALIDATE_DIR(node); hash = tmpfs_dirent_hash(cnp->cn_nameptr, cnp->cn_namelen); de = tmpfs_dir_xlookup_hash(node, hash); if (de != NULL && tmpfs_dirent_duphead(de)) { duphead = &de->ud.td_duphead; LIST_FOREACH(de, duphead, uh.td_dup.entries) { if (TMPFS_DIRENT_MATCHES(de, cnp->cn_nameptr, cnp->cn_namelen)) break; } } else if (de != NULL) { if (!TMPFS_DIRENT_MATCHES(de, cnp->cn_nameptr, cnp->cn_namelen)) de = NULL; } if (de != NULL && f != NULL && de->td_node != f) de = NULL; return (de); } /* * Attach duplicate-cookie directory entry nde to dnode and insert to dupindex * list, allocate new cookie value. */ static void tmpfs_dir_attach_dup(struct tmpfs_node *dnode, struct tmpfs_dir_duphead *duphead, struct tmpfs_dirent *nde) { struct tmpfs_dir_duphead *dupindex; struct tmpfs_dirent *de, *pde; dupindex = &dnode->tn_dir.tn_dupindex; de = LIST_FIRST(dupindex); if (de == NULL || de->td_cookie < TMPFS_DIRCOOKIE_DUP_MAX) { if (de == NULL) nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MIN; else nde->td_cookie = de->td_cookie + 1; MPASS(tmpfs_dirent_dup(nde)); LIST_INSERT_HEAD(dupindex, nde, uh.td_dup.index_entries); LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); return; } /* * Cookie numbers are near exhaustion. Scan dupindex list for unused * numbers. dupindex list is sorted in descending order. Keep it so * after inserting nde. */ while (1) { pde = de; de = LIST_NEXT(de, uh.td_dup.index_entries); if (de == NULL && pde->td_cookie != TMPFS_DIRCOOKIE_DUP_MIN) { /* * Last element of the index doesn't have minimal cookie * value, use it. */ nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MIN; LIST_INSERT_AFTER(pde, nde, uh.td_dup.index_entries); LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); return; } else if (de == NULL) { /* * We are so lucky have 2^30 hash duplicates in single * directory :) Return largest possible cookie value. * It should be fine except possible issues with * VOP_READDIR restart. */ nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MAX; LIST_INSERT_HEAD(dupindex, nde, uh.td_dup.index_entries); LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); return; } if (de->td_cookie + 1 == pde->td_cookie || de->td_cookie >= TMPFS_DIRCOOKIE_DUP_MAX) continue; /* No hole or invalid cookie. */ nde->td_cookie = de->td_cookie + 1; MPASS(tmpfs_dirent_dup(nde)); MPASS(pde->td_cookie > nde->td_cookie); MPASS(nde->td_cookie > de->td_cookie); LIST_INSERT_BEFORE(de, nde, uh.td_dup.index_entries); LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries); return; }; } /* * Attaches the directory entry de to the directory represented by vp. * Note that this does not change the link count of the node pointed by * the directory entry, as this is done by tmpfs_alloc_dirent. */ void tmpfs_dir_attach(struct vnode *vp, struct tmpfs_dirent *de) { struct tmpfs_node *dnode; struct tmpfs_dirent *xde, *nde; ASSERT_VOP_ELOCKED(vp, __func__); MPASS(de->td_namelen > 0); MPASS(de->td_hash >= TMPFS_DIRCOOKIE_MIN); MPASS(de->td_cookie == de->td_hash); dnode = VP_TO_TMPFS_DIR(vp); dnode->tn_dir.tn_readdir_lastn = 0; dnode->tn_dir.tn_readdir_lastp = NULL; MPASS(!tmpfs_dirent_dup(de)); xde = RB_INSERT(tmpfs_dir, &dnode->tn_dir.tn_dirhead, de); if (xde != NULL && tmpfs_dirent_duphead(xde)) tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, de); else if (xde != NULL) { /* * Allocate new duphead. Swap xde with duphead to avoid * adding/removing elements with the same hash. */ MPASS(!tmpfs_dirent_dup(xde)); tmpfs_alloc_dirent(VFS_TO_TMPFS(vp->v_mount), NULL, NULL, 0, &nde); /* *nde = *xde; XXX gcc 4.2.1 may generate invalid code. */ memcpy(nde, xde, sizeof(*xde)); xde->td_cookie |= TMPFS_DIRCOOKIE_DUPHEAD; LIST_INIT(&xde->ud.td_duphead); xde->td_namelen = 0; xde->td_node = NULL; tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, nde); tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, de); } dnode->tn_size += sizeof(struct tmpfs_dirent); dnode->tn_status |= TMPFS_NODE_ACCESSED | TMPFS_NODE_CHANGED | \ TMPFS_NODE_MODIFIED; } /* * Detaches the directory entry de from the directory represented by vp. * Note that this does not change the link count of the node pointed by * the directory entry, as this is done by tmpfs_free_dirent. */ void tmpfs_dir_detach(struct vnode *vp, struct tmpfs_dirent *de) { struct tmpfs_mount *tmp; struct tmpfs_dir *head; struct tmpfs_node *dnode; struct tmpfs_dirent *xde; ASSERT_VOP_ELOCKED(vp, __func__); dnode = VP_TO_TMPFS_DIR(vp); head = &dnode->tn_dir.tn_dirhead; dnode->tn_dir.tn_readdir_lastn = 0; dnode->tn_dir.tn_readdir_lastp = NULL; if (tmpfs_dirent_dup(de)) { /* Remove duphead if de was last entry. */ if (LIST_NEXT(de, uh.td_dup.entries) == NULL) { xde = tmpfs_dir_xlookup_hash(dnode, de->td_hash); MPASS(tmpfs_dirent_duphead(xde)); } else xde = NULL; LIST_REMOVE(de, uh.td_dup.entries); LIST_REMOVE(de, uh.td_dup.index_entries); if (xde != NULL) { if (LIST_EMPTY(&xde->ud.td_duphead)) { RB_REMOVE(tmpfs_dir, head, xde); tmp = VFS_TO_TMPFS(vp->v_mount); MPASS(xde->td_node == NULL); tmpfs_free_dirent(tmp, xde); } } } else RB_REMOVE(tmpfs_dir, head, de); dnode->tn_size -= sizeof(struct tmpfs_dirent); dnode->tn_status |= TMPFS_NODE_ACCESSED | TMPFS_NODE_CHANGED | \ TMPFS_NODE_MODIFIED; } void tmpfs_dir_destroy(struct tmpfs_mount *tmp, struct tmpfs_node *dnode) { struct tmpfs_dirent *de, *dde, *nde; RB_FOREACH_SAFE(de, tmpfs_dir, &dnode->tn_dir.tn_dirhead, nde) { RB_REMOVE(tmpfs_dir, &dnode->tn_dir.tn_dirhead, de); /* Node may already be destroyed. */ de->td_node = NULL; if (tmpfs_dirent_duphead(de)) { while ((dde = LIST_FIRST(&de->ud.td_duphead)) != NULL) { LIST_REMOVE(dde, uh.td_dup.entries); dde->td_node = NULL; tmpfs_free_dirent(tmp, dde); } } tmpfs_free_dirent(tmp, de); } } /* * Helper function for tmpfs_readdir. Creates a '.' entry for the given * directory and returns it in the uio space. The function returns 0 * on success, -1 if there was not enough space in the uio structure to * hold the directory entry or an appropriate error code if another * error happens. */ static int tmpfs_dir_getdotdent(struct tmpfs_node *node, struct uio *uio) { int error; struct dirent dent; TMPFS_VALIDATE_DIR(node); MPASS(uio->uio_offset == TMPFS_DIRCOOKIE_DOT); dent.d_fileno = node->tn_id; dent.d_type = DT_DIR; dent.d_namlen = 1; dent.d_name[0] = '.'; dent.d_name[1] = '\0'; dent.d_reclen = GENERIC_DIRSIZ(&dent); if (dent.d_reclen > uio->uio_resid) error = EJUSTRETURN; else error = uiomove(&dent, dent.d_reclen, uio); node->tn_status |= TMPFS_NODE_ACCESSED; return error; } /* * Helper function for tmpfs_readdir. Creates a '..' entry for the given * directory and returns it in the uio space. The function returns 0 * on success, -1 if there was not enough space in the uio structure to * hold the directory entry or an appropriate error code if another * error happens. */ static int tmpfs_dir_getdotdotdent(struct tmpfs_node *node, struct uio *uio) { int error; struct dirent dent; TMPFS_VALIDATE_DIR(node); MPASS(uio->uio_offset == TMPFS_DIRCOOKIE_DOTDOT); /* * Return ENOENT if the current node is already removed. */ TMPFS_ASSERT_LOCKED(node); if (node->tn_dir.tn_parent == NULL) { return (ENOENT); } TMPFS_NODE_LOCK(node->tn_dir.tn_parent); dent.d_fileno = node->tn_dir.tn_parent->tn_id; TMPFS_NODE_UNLOCK(node->tn_dir.tn_parent); dent.d_type = DT_DIR; dent.d_namlen = 2; dent.d_name[0] = '.'; dent.d_name[1] = '.'; dent.d_name[2] = '\0'; dent.d_reclen = GENERIC_DIRSIZ(&dent); if (dent.d_reclen > uio->uio_resid) error = EJUSTRETURN; else error = uiomove(&dent, dent.d_reclen, uio); node->tn_status |= TMPFS_NODE_ACCESSED; return error; } /* * Helper function for tmpfs_readdir. Returns as much directory entries * as can fit in the uio space. The read starts at uio->uio_offset. * The function returns 0 on success, -1 if there was not enough space * in the uio structure to hold the directory entry or an appropriate * error code if another error happens. */ int tmpfs_dir_getdents(struct tmpfs_node *node, struct uio *uio, int maxcookies, u_long *cookies, int *ncookies) { struct tmpfs_dir_cursor dc; struct tmpfs_dirent *de; off_t off; int error; TMPFS_VALIDATE_DIR(node); off = 0; /* * Lookup the node from the current offset. The starting offset of * 0 will lookup both '.' and '..', and then the first real entry, * or EOF if there are none. Then find all entries for the dir that * fit into the buffer. Once no more entries are found (de == NULL), * the offset is set to TMPFS_DIRCOOKIE_EOF, which will cause the next * call to return 0. */ switch (uio->uio_offset) { case TMPFS_DIRCOOKIE_DOT: error = tmpfs_dir_getdotdent(node, uio); if (error != 0) return (error); uio->uio_offset = TMPFS_DIRCOOKIE_DOTDOT; if (cookies != NULL) cookies[(*ncookies)++] = off = uio->uio_offset; /* FALLTHROUGH */ case TMPFS_DIRCOOKIE_DOTDOT: error = tmpfs_dir_getdotdotdent(node, uio); if (error != 0) return (error); de = tmpfs_dir_first(node, &dc); uio->uio_offset = tmpfs_dirent_cookie(de); if (cookies != NULL) cookies[(*ncookies)++] = off = uio->uio_offset; /* EOF. */ if (de == NULL) return (0); break; case TMPFS_DIRCOOKIE_EOF: return (0); default: de = tmpfs_dir_lookup_cookie(node, uio->uio_offset, &dc); if (de == NULL) return (EINVAL); if (cookies != NULL) off = tmpfs_dirent_cookie(de); } /* Read as much entries as possible; i.e., until we reach the end of * the directory or we exhaust uio space. */ do { struct dirent d; /* Create a dirent structure representing the current * tmpfs_node and fill it. */ if (de->td_node == NULL) { d.d_fileno = 1; d.d_type = DT_WHT; } else { d.d_fileno = de->td_node->tn_id; switch (de->td_node->tn_type) { case VBLK: d.d_type = DT_BLK; break; case VCHR: d.d_type = DT_CHR; break; case VDIR: d.d_type = DT_DIR; break; case VFIFO: d.d_type = DT_FIFO; break; case VLNK: d.d_type = DT_LNK; break; case VREG: d.d_type = DT_REG; break; case VSOCK: d.d_type = DT_SOCK; break; default: panic("tmpfs_dir_getdents: type %p %d", de->td_node, (int)de->td_node->tn_type); } } d.d_namlen = de->td_namelen; MPASS(de->td_namelen < sizeof(d.d_name)); (void)memcpy(d.d_name, de->ud.td_name, de->td_namelen); d.d_name[de->td_namelen] = '\0'; d.d_reclen = GENERIC_DIRSIZ(&d); /* Stop reading if the directory entry we are treating is * bigger than the amount of data that can be returned. */ if (d.d_reclen > uio->uio_resid) { error = EJUSTRETURN; break; } /* Copy the new dirent structure into the output buffer and * advance pointers. */ error = uiomove(&d, d.d_reclen, uio); if (error == 0) { de = tmpfs_dir_next(node, &dc); if (cookies != NULL) { off = tmpfs_dirent_cookie(de); MPASS(*ncookies < maxcookies); cookies[(*ncookies)++] = off; } } } while (error == 0 && uio->uio_resid > 0 && de != NULL); /* Skip setting off when using cookies as it is already done above. */ if (cookies == NULL) off = tmpfs_dirent_cookie(de); /* Update the offset and cache. */ uio->uio_offset = off; node->tn_dir.tn_readdir_lastn = off; node->tn_dir.tn_readdir_lastp = de; node->tn_status |= TMPFS_NODE_ACCESSED; return error; } int tmpfs_dir_whiteout_add(struct vnode *dvp, struct componentname *cnp) { struct tmpfs_dirent *de; int error; error = tmpfs_alloc_dirent(VFS_TO_TMPFS(dvp->v_mount), NULL, cnp->cn_nameptr, cnp->cn_namelen, &de); if (error != 0) return (error); tmpfs_dir_attach(dvp, de); return (0); } void tmpfs_dir_whiteout_remove(struct vnode *dvp, struct componentname *cnp) { struct tmpfs_dirent *de; de = tmpfs_dir_lookup(VP_TO_TMPFS_DIR(dvp), NULL, cnp); MPASS(de != NULL && de->td_node == NULL); tmpfs_dir_detach(dvp, de); tmpfs_free_dirent(VFS_TO_TMPFS(dvp->v_mount), de); } /* * Resizes the aobj associated with the regular file pointed to by 'vp' to the * size 'newsize'. 'vp' must point to a vnode that represents a regular file. * 'newsize' must be positive. * * Returns zero on success or an appropriate error code on failure. */ int tmpfs_reg_resize(struct vnode *vp, off_t newsize, boolean_t ignerr) { struct tmpfs_mount *tmp; struct tmpfs_node *node; vm_object_t uobj; vm_page_t m, ma[1]; vm_pindex_t idx, newpages, oldpages; off_t oldsize; int base, rv; MPASS(vp->v_type == VREG); MPASS(newsize >= 0); node = VP_TO_TMPFS_NODE(vp); uobj = node->tn_reg.tn_aobj; tmp = VFS_TO_TMPFS(vp->v_mount); /* * Convert the old and new sizes to the number of pages needed to * store them. It may happen that we do not need to do anything * because the last allocated page can accommodate the change on * its own. */ oldsize = node->tn_size; oldpages = OFF_TO_IDX(oldsize + PAGE_MASK); MPASS(oldpages == uobj->size); newpages = OFF_TO_IDX(newsize + PAGE_MASK); if (newpages > oldpages && tmpfs_pages_check_avail(tmp, newpages - oldpages) == 0) return (ENOSPC); VM_OBJECT_WLOCK(uobj); if (newsize < oldsize) { /* * Zero the truncated part of the last page. */ base = newsize & PAGE_MASK; if (base != 0) { idx = OFF_TO_IDX(newsize); retry: m = vm_page_lookup(uobj, idx); if (m != NULL) { if (vm_page_sleep_if_busy(m, "tmfssz")) goto retry; MPASS(m->valid == VM_PAGE_BITS_ALL); } else if (vm_pager_has_page(uobj, idx, NULL, NULL)) { m = vm_page_alloc(uobj, idx, VM_ALLOC_NORMAL); if (m == NULL) { VM_OBJECT_WUNLOCK(uobj); VM_WAIT; VM_OBJECT_WLOCK(uobj); goto retry; } else if (m->valid != VM_PAGE_BITS_ALL) { ma[0] = m; rv = vm_pager_get_pages(uobj, ma, 1, 0); m = vm_page_lookup(uobj, idx); } else /* A cached page was reactivated. */ rv = VM_PAGER_OK; vm_page_lock(m); if (rv == VM_PAGER_OK) { vm_page_deactivate(m); vm_page_unlock(m); vm_page_xunbusy(m); } else { vm_page_free(m); vm_page_unlock(m); if (ignerr) m = NULL; else { VM_OBJECT_WUNLOCK(uobj); return (EIO); } } } if (m != NULL) { pmap_zero_page_area(m, base, PAGE_SIZE - base); vm_page_dirty(m); vm_pager_page_unswapped(m); } } /* * Release any swap space and free any whole pages. */ if (newpages < oldpages) { swap_pager_freespace(uobj, newpages, oldpages - newpages); vm_object_page_remove(uobj, newpages, 0, 0); } } uobj->size = newpages; VM_OBJECT_WUNLOCK(uobj); TMPFS_LOCK(tmp); tmp->tm_pages_used += (newpages - oldpages); TMPFS_UNLOCK(tmp); node->tn_size = newsize; return (0); } void tmpfs_check_mtime(struct vnode *vp) { struct tmpfs_node *node; struct vm_object *obj; ASSERT_VOP_ELOCKED(vp, "check_mtime"); if (vp->v_type != VREG) return; node = VP_TO_TMPFS_NODE(vp); obj = vp->v_object; KASSERT((obj->flags & (OBJ_TMPFS_NODE | OBJ_TMPFS)) == (OBJ_TMPFS_NODE | OBJ_TMPFS), ("non-tmpfs obj")); /* unlocked read */ if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) { VM_OBJECT_WLOCK(obj); if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) { obj->flags &= ~OBJ_TMPFS_DIRTY; node = VP_TO_TMPFS_NODE(vp); - node->tn_status |= TMPFS_NODE_MODIFIED; + node->tn_status |= TMPFS_NODE_MODIFIED | + TMPFS_NODE_CHANGED; } VM_OBJECT_WUNLOCK(obj); } } /* * Change flags of the given vnode. * Caller should execute tmpfs_update on vp after a successful execution. * The vnode must be locked on entry and remain locked on exit. */ int tmpfs_chflags(struct vnode *vp, u_long flags, struct ucred *cred, struct thread *p) { int error; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(vp)); node = VP_TO_TMPFS_NODE(vp); if ((flags & ~(SF_APPEND | SF_ARCHIVED | SF_IMMUTABLE | SF_NOUNLINK | UF_APPEND | UF_ARCHIVE | UF_HIDDEN | UF_IMMUTABLE | UF_NODUMP | UF_NOUNLINK | UF_OFFLINE | UF_OPAQUE | UF_READONLY | UF_REPARSE | UF_SPARSE | UF_SYSTEM)) != 0) return (EOPNOTSUPP); /* Disallow this operation if the file system is mounted read-only. */ if (vp->v_mount->mnt_flag & MNT_RDONLY) return EROFS; /* * Callers may only modify the file flags on objects they * have VADMIN rights for. */ if ((error = VOP_ACCESS(vp, VADMIN, cred, p))) return (error); /* * Unprivileged processes are not permitted to unset system * flags, or modify flags if any system flags are set. */ if (!priv_check_cred(cred, PRIV_VFS_SYSFLAGS, 0)) { if (node->tn_flags & (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND)) { error = securelevel_gt(cred, 0); if (error) return (error); } } else { if (node->tn_flags & (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND) || ((flags ^ node->tn_flags) & SF_SETTABLE)) return (EPERM); } node->tn_flags = flags; node->tn_status |= TMPFS_NODE_CHANGED; MPASS(VOP_ISLOCKED(vp)); return 0; } /* * Change access mode on the given vnode. * Caller should execute tmpfs_update on vp after a successful execution. * The vnode must be locked on entry and remain locked on exit. */ int tmpfs_chmod(struct vnode *vp, mode_t mode, struct ucred *cred, struct thread *p) { int error; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(vp)); node = VP_TO_TMPFS_NODE(vp); /* Disallow this operation if the file system is mounted read-only. */ if (vp->v_mount->mnt_flag & MNT_RDONLY) return EROFS; /* Immutable or append-only files cannot be modified, either. */ if (node->tn_flags & (IMMUTABLE | APPEND)) return EPERM; /* * To modify the permissions on a file, must possess VADMIN * for that file. */ if ((error = VOP_ACCESS(vp, VADMIN, cred, p))) return (error); /* * Privileged processes may set the sticky bit on non-directories, * as well as set the setgid bit on a file with a group that the * process is not a member of. */ if (vp->v_type != VDIR && (mode & S_ISTXT)) { if (priv_check_cred(cred, PRIV_VFS_STICKYFILE, 0)) return (EFTYPE); } if (!groupmember(node->tn_gid, cred) && (mode & S_ISGID)) { error = priv_check_cred(cred, PRIV_VFS_SETGID, 0); if (error) return (error); } node->tn_mode &= ~ALLPERMS; node->tn_mode |= mode & ALLPERMS; node->tn_status |= TMPFS_NODE_CHANGED; MPASS(VOP_ISLOCKED(vp)); return 0; } /* * Change ownership of the given vnode. At least one of uid or gid must * be different than VNOVAL. If one is set to that value, the attribute * is unchanged. * Caller should execute tmpfs_update on vp after a successful execution. * The vnode must be locked on entry and remain locked on exit. */ int tmpfs_chown(struct vnode *vp, uid_t uid, gid_t gid, struct ucred *cred, struct thread *p) { int error; struct tmpfs_node *node; uid_t ouid; gid_t ogid; MPASS(VOP_ISLOCKED(vp)); node = VP_TO_TMPFS_NODE(vp); /* Assign default values if they are unknown. */ MPASS(uid != VNOVAL || gid != VNOVAL); if (uid == VNOVAL) uid = node->tn_uid; if (gid == VNOVAL) gid = node->tn_gid; MPASS(uid != VNOVAL && gid != VNOVAL); /* Disallow this operation if the file system is mounted read-only. */ if (vp->v_mount->mnt_flag & MNT_RDONLY) return EROFS; /* Immutable or append-only files cannot be modified, either. */ if (node->tn_flags & (IMMUTABLE | APPEND)) return EPERM; /* * To modify the ownership of a file, must possess VADMIN for that * file. */ if ((error = VOP_ACCESS(vp, VADMIN, cred, p))) return (error); /* * To change the owner of a file, or change the group of a file to a * group of which we are not a member, the caller must have * privilege. */ if ((uid != node->tn_uid || (gid != node->tn_gid && !groupmember(gid, cred))) && (error = priv_check_cred(cred, PRIV_VFS_CHOWN, 0))) return (error); ogid = node->tn_gid; ouid = node->tn_uid; node->tn_uid = uid; node->tn_gid = gid; node->tn_status |= TMPFS_NODE_CHANGED; if ((node->tn_mode & (S_ISUID | S_ISGID)) && (ouid != uid || ogid != gid)) { if (priv_check_cred(cred, PRIV_VFS_RETAINSUGID, 0)) node->tn_mode &= ~(S_ISUID | S_ISGID); } MPASS(VOP_ISLOCKED(vp)); return 0; } /* * Change size of the given vnode. * Caller should execute tmpfs_update on vp after a successful execution. * The vnode must be locked on entry and remain locked on exit. */ int tmpfs_chsize(struct vnode *vp, u_quad_t size, struct ucred *cred, struct thread *p) { int error; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(vp)); node = VP_TO_TMPFS_NODE(vp); /* Decide whether this is a valid operation based on the file type. */ error = 0; switch (vp->v_type) { case VDIR: return EISDIR; case VREG: if (vp->v_mount->mnt_flag & MNT_RDONLY) return EROFS; break; case VBLK: /* FALLTHROUGH */ case VCHR: /* FALLTHROUGH */ case VFIFO: /* Allow modifications of special files even if in the file * system is mounted read-only (we are not modifying the * files themselves, but the objects they represent). */ return 0; default: /* Anything else is unsupported. */ return EOPNOTSUPP; } /* Immutable or append-only files cannot be modified, either. */ if (node->tn_flags & (IMMUTABLE | APPEND)) return EPERM; error = tmpfs_truncate(vp, size); /* tmpfs_truncate will raise the NOTE_EXTEND and NOTE_ATTRIB kevents * for us, as will update tn_status; no need to do that here. */ MPASS(VOP_ISLOCKED(vp)); return error; } /* * Change access and modification times of the given vnode. * Caller should execute tmpfs_update on vp after a successful execution. * The vnode must be locked on entry and remain locked on exit. */ int tmpfs_chtimes(struct vnode *vp, struct vattr *vap, struct ucred *cred, struct thread *l) { int error; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(vp)); node = VP_TO_TMPFS_NODE(vp); /* Disallow this operation if the file system is mounted read-only. */ if (vp->v_mount->mnt_flag & MNT_RDONLY) return EROFS; /* Immutable or append-only files cannot be modified, either. */ if (node->tn_flags & (IMMUTABLE | APPEND)) return EPERM; error = vn_utimes_perm(vp, vap, cred, l); if (error != 0) return (error); if (vap->va_atime.tv_sec != VNOVAL && vap->va_atime.tv_nsec != VNOVAL) node->tn_status |= TMPFS_NODE_ACCESSED; if (vap->va_mtime.tv_sec != VNOVAL && vap->va_mtime.tv_nsec != VNOVAL) node->tn_status |= TMPFS_NODE_MODIFIED; if (vap->va_birthtime.tv_nsec != VNOVAL && vap->va_birthtime.tv_nsec != VNOVAL) node->tn_status |= TMPFS_NODE_MODIFIED; tmpfs_itimes(vp, &vap->va_atime, &vap->va_mtime); if (vap->va_birthtime.tv_nsec != VNOVAL && vap->va_birthtime.tv_nsec != VNOVAL) node->tn_birthtime = vap->va_birthtime; MPASS(VOP_ISLOCKED(vp)); return 0; } /* Sync timestamps */ void tmpfs_itimes(struct vnode *vp, const struct timespec *acc, const struct timespec *mod) { struct tmpfs_node *node; struct timespec now; node = VP_TO_TMPFS_NODE(vp); if ((node->tn_status & (TMPFS_NODE_ACCESSED | TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED)) == 0) return; vfs_timestamp(&now); if (node->tn_status & TMPFS_NODE_ACCESSED) { if (acc == NULL) acc = &now; node->tn_atime = *acc; } if (node->tn_status & TMPFS_NODE_MODIFIED) { if (mod == NULL) mod = &now; node->tn_mtime = *mod; } if (node->tn_status & TMPFS_NODE_CHANGED) { node->tn_ctime = now; } node->tn_status &= ~(TMPFS_NODE_ACCESSED | TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED); } void tmpfs_update(struct vnode *vp) { tmpfs_itimes(vp, NULL, NULL); } int tmpfs_truncate(struct vnode *vp, off_t length) { int error; struct tmpfs_node *node; node = VP_TO_TMPFS_NODE(vp); if (length < 0) { error = EINVAL; goto out; } if (node->tn_size == length) { error = 0; goto out; } if (length > VFS_TO_TMPFS(vp->v_mount)->tm_maxfilesize) return (EFBIG); error = tmpfs_reg_resize(vp, length, FALSE); if (error == 0) { node->tn_status |= TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED; } out: tmpfs_update(vp); return error; } static __inline int tmpfs_dirtree_cmp(struct tmpfs_dirent *a, struct tmpfs_dirent *b) { if (a->td_hash > b->td_hash) return (1); else if (a->td_hash < b->td_hash) return (-1); return (0); } RB_GENERATE_STATIC(tmpfs_dir, tmpfs_dirent, uh.td_entries, tmpfs_dirtree_cmp); diff --git a/sys/fs/tmpfs/tmpfs_vnops.c b/sys/fs/tmpfs/tmpfs_vnops.c index 65c5f82d5de5..885f84cdf2b5 100644 --- a/sys/fs/tmpfs/tmpfs_vnops.c +++ b/sys/fs/tmpfs/tmpfs_vnops.c @@ -1,1420 +1,1418 @@ /* $NetBSD: tmpfs_vnops.c,v 1.39 2007/07/23 15:41:01 jmmv Exp $ */ /*- * Copyright (c) 2005, 2006 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Julio M. Merino Vidal, developed as part of Google's Summer of Code * 2005 program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * tmpfs vnode interface. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include SYSCTL_DECL(_vfs_tmpfs); static volatile int tmpfs_rename_restarts; SYSCTL_INT(_vfs_tmpfs, OID_AUTO, rename_restarts, CTLFLAG_RD, __DEVOLATILE(int *, &tmpfs_rename_restarts), 0, "Times rename had to restart due to lock contention"); static int tmpfs_vn_get_ino_alloc(struct mount *mp, void *arg, int lkflags, struct vnode **rvp) { return (tmpfs_alloc_vp(mp, arg, lkflags, rvp)); } static int tmpfs_lookup(struct vop_cachedlookup_args *v) { struct vnode *dvp = v->a_dvp; struct vnode **vpp = v->a_vpp; struct componentname *cnp = v->a_cnp; struct tmpfs_dirent *de; struct tmpfs_node *dnode; int error; dnode = VP_TO_TMPFS_DIR(dvp); *vpp = NULLVP; /* Check accessibility of requested node as a first step. */ error = VOP_ACCESS(dvp, VEXEC, cnp->cn_cred, cnp->cn_thread); if (error != 0) goto out; /* We cannot be requesting the parent directory of the root node. */ MPASS(IMPLIES(dnode->tn_type == VDIR && dnode->tn_dir.tn_parent == dnode, !(cnp->cn_flags & ISDOTDOT))); TMPFS_ASSERT_LOCKED(dnode); if (dnode->tn_dir.tn_parent == NULL) { error = ENOENT; goto out; } if (cnp->cn_flags & ISDOTDOT) { error = vn_vget_ino_gen(dvp, tmpfs_vn_get_ino_alloc, dnode->tn_dir.tn_parent, cnp->cn_lkflags, vpp); if (error != 0) goto out; } else if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') { VREF(dvp); *vpp = dvp; error = 0; } else { de = tmpfs_dir_lookup(dnode, NULL, cnp); if (de != NULL && de->td_node == NULL) cnp->cn_flags |= ISWHITEOUT; if (de == NULL || de->td_node == NULL) { /* The entry was not found in the directory. * This is OK if we are creating or renaming an * entry and are working on the last component of * the path name. */ if ((cnp->cn_flags & ISLASTCN) && (cnp->cn_nameiop == CREATE || \ cnp->cn_nameiop == RENAME || (cnp->cn_nameiop == DELETE && cnp->cn_flags & DOWHITEOUT && cnp->cn_flags & ISWHITEOUT))) { error = VOP_ACCESS(dvp, VWRITE, cnp->cn_cred, cnp->cn_thread); if (error != 0) goto out; /* Keep the component name in the buffer for * future uses. */ cnp->cn_flags |= SAVENAME; error = EJUSTRETURN; } else error = ENOENT; } else { struct tmpfs_node *tnode; /* The entry was found, so get its associated * tmpfs_node. */ tnode = de->td_node; /* If we are not at the last path component and * found a non-directory or non-link entry (which * may itself be pointing to a directory), raise * an error. */ if ((tnode->tn_type != VDIR && tnode->tn_type != VLNK) && !(cnp->cn_flags & ISLASTCN)) { error = ENOTDIR; goto out; } /* If we are deleting or renaming the entry, keep * track of its tmpfs_dirent so that it can be * easily deleted later. */ if ((cnp->cn_flags & ISLASTCN) && (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) { error = VOP_ACCESS(dvp, VWRITE, cnp->cn_cred, cnp->cn_thread); if (error != 0) goto out; /* Allocate a new vnode on the matching entry. */ error = tmpfs_alloc_vp(dvp->v_mount, tnode, cnp->cn_lkflags, vpp); if (error != 0) goto out; if ((dnode->tn_mode & S_ISTXT) && VOP_ACCESS(dvp, VADMIN, cnp->cn_cred, cnp->cn_thread) && VOP_ACCESS(*vpp, VADMIN, cnp->cn_cred, cnp->cn_thread)) { error = EPERM; vput(*vpp); *vpp = NULL; goto out; } cnp->cn_flags |= SAVENAME; } else { error = tmpfs_alloc_vp(dvp->v_mount, tnode, cnp->cn_lkflags, vpp); if (error != 0) goto out; } } } /* Store the result of this lookup in the cache. Avoid this if the * request was for creation, as it does not improve timings on * emprical tests. */ if ((cnp->cn_flags & MAKEENTRY) != 0) cache_enter(dvp, *vpp, cnp); out: /* If there were no errors, *vpp cannot be null and it must be * locked. */ MPASS(IFF(error == 0, *vpp != NULLVP && VOP_ISLOCKED(*vpp))); return error; } static int tmpfs_create(struct vop_create_args *v) { struct vnode *dvp = v->a_dvp; struct vnode **vpp = v->a_vpp; struct componentname *cnp = v->a_cnp; struct vattr *vap = v->a_vap; int error; MPASS(vap->va_type == VREG || vap->va_type == VSOCK); error = tmpfs_alloc_file(dvp, vpp, vap, cnp, NULL); if (error == 0 && (cnp->cn_flags & MAKEENTRY) != 0) cache_enter(dvp, *vpp, cnp); return (error); } static int tmpfs_mknod(struct vop_mknod_args *v) { struct vnode *dvp = v->a_dvp; struct vnode **vpp = v->a_vpp; struct componentname *cnp = v->a_cnp; struct vattr *vap = v->a_vap; if (vap->va_type != VBLK && vap->va_type != VCHR && vap->va_type != VFIFO) return EINVAL; return tmpfs_alloc_file(dvp, vpp, vap, cnp, NULL); } static int tmpfs_open(struct vop_open_args *v) { struct vnode *vp = v->a_vp; int mode = v->a_mode; int error; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(vp)); node = VP_TO_TMPFS_NODE(vp); /* The file is still active but all its names have been removed * (e.g. by a "rmdir $(pwd)"). It cannot be opened any more as * it is about to die. */ if (node->tn_links < 1) return (ENOENT); /* If the file is marked append-only, deny write requests. */ if (node->tn_flags & APPEND && (mode & (FWRITE | O_APPEND)) == FWRITE) error = EPERM; else { error = 0; /* For regular files, the call below is nop. */ KASSERT(vp->v_type != VREG || (node->tn_reg.tn_aobj->flags & OBJ_DEAD) == 0, ("dead object")); vnode_create_vobject(vp, node->tn_size, v->a_td); } MPASS(VOP_ISLOCKED(vp)); return error; } static int tmpfs_close(struct vop_close_args *v) { struct vnode *vp = v->a_vp; /* Update node times. */ tmpfs_update(vp); return (0); } int tmpfs_access(struct vop_access_args *v) { struct vnode *vp = v->a_vp; accmode_t accmode = v->a_accmode; struct ucred *cred = v->a_cred; int error; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(vp)); node = VP_TO_TMPFS_NODE(vp); switch (vp->v_type) { case VDIR: /* FALLTHROUGH */ case VLNK: /* FALLTHROUGH */ case VREG: if (accmode & VWRITE && vp->v_mount->mnt_flag & MNT_RDONLY) { error = EROFS; goto out; } break; case VBLK: /* FALLTHROUGH */ case VCHR: /* FALLTHROUGH */ case VSOCK: /* FALLTHROUGH */ case VFIFO: break; default: error = EINVAL; goto out; } if (accmode & VWRITE && node->tn_flags & IMMUTABLE) { error = EPERM; goto out; } error = vaccess(vp->v_type, node->tn_mode, node->tn_uid, node->tn_gid, accmode, cred, NULL); out: MPASS(VOP_ISLOCKED(vp)); return error; } int tmpfs_getattr(struct vop_getattr_args *v) { struct vnode *vp = v->a_vp; struct vattr *vap = v->a_vap; struct tmpfs_node *node; node = VP_TO_TMPFS_NODE(vp); tmpfs_update(vp); vap->va_type = vp->v_type; vap->va_mode = node->tn_mode; vap->va_nlink = node->tn_links; vap->va_uid = node->tn_uid; vap->va_gid = node->tn_gid; vap->va_fsid = vp->v_mount->mnt_stat.f_fsid.val[0]; vap->va_fileid = node->tn_id; vap->va_size = node->tn_size; vap->va_blocksize = PAGE_SIZE; vap->va_atime = node->tn_atime; vap->va_mtime = node->tn_mtime; vap->va_ctime = node->tn_ctime; vap->va_birthtime = node->tn_birthtime; vap->va_gen = node->tn_gen; vap->va_flags = node->tn_flags; vap->va_rdev = (vp->v_type == VBLK || vp->v_type == VCHR) ? node->tn_rdev : NODEV; vap->va_bytes = round_page(node->tn_size); vap->va_filerev = 0; return 0; } int tmpfs_setattr(struct vop_setattr_args *v) { struct vnode *vp = v->a_vp; struct vattr *vap = v->a_vap; struct ucred *cred = v->a_cred; struct thread *td = curthread; int error; MPASS(VOP_ISLOCKED(vp)); error = 0; /* Abort if any unsettable attribute is given. */ if (vap->va_type != VNON || vap->va_nlink != VNOVAL || vap->va_fsid != VNOVAL || vap->va_fileid != VNOVAL || vap->va_blocksize != VNOVAL || vap->va_gen != VNOVAL || vap->va_rdev != VNOVAL || vap->va_bytes != VNOVAL) error = EINVAL; if (error == 0 && (vap->va_flags != VNOVAL)) error = tmpfs_chflags(vp, vap->va_flags, cred, td); if (error == 0 && (vap->va_size != VNOVAL)) error = tmpfs_chsize(vp, vap->va_size, cred, td); if (error == 0 && (vap->va_uid != VNOVAL || vap->va_gid != VNOVAL)) error = tmpfs_chown(vp, vap->va_uid, vap->va_gid, cred, td); if (error == 0 && (vap->va_mode != (mode_t)VNOVAL)) error = tmpfs_chmod(vp, vap->va_mode, cred, td); if (error == 0 && ((vap->va_atime.tv_sec != VNOVAL && vap->va_atime.tv_nsec != VNOVAL) || (vap->va_mtime.tv_sec != VNOVAL && vap->va_mtime.tv_nsec != VNOVAL) || (vap->va_birthtime.tv_sec != VNOVAL && vap->va_birthtime.tv_nsec != VNOVAL))) error = tmpfs_chtimes(vp, vap, cred, td); /* Update the node times. We give preference to the error codes * generated by this function rather than the ones that may arise * from tmpfs_update. */ tmpfs_update(vp); MPASS(VOP_ISLOCKED(vp)); return error; } static int tmpfs_read(struct vop_read_args *v) { struct vnode *vp; struct uio *uio; struct tmpfs_node *node; vp = v->a_vp; if (vp->v_type != VREG) return (EISDIR); uio = v->a_uio; if (uio->uio_offset < 0) return (EINVAL); node = VP_TO_TMPFS_NODE(vp); node->tn_status |= TMPFS_NODE_ACCESSED; return (uiomove_object(node->tn_reg.tn_aobj, node->tn_size, uio)); } static int tmpfs_write(struct vop_write_args *v) { struct vnode *vp; struct uio *uio; struct tmpfs_node *node; off_t oldsize; int error, ioflag; - boolean_t extended; vp = v->a_vp; uio = v->a_uio; ioflag = v->a_ioflag; error = 0; node = VP_TO_TMPFS_NODE(vp); oldsize = node->tn_size; if (uio->uio_offset < 0 || vp->v_type != VREG) return (EINVAL); if (uio->uio_resid == 0) return (0); if (ioflag & IO_APPEND) uio->uio_offset = node->tn_size; if (uio->uio_offset + uio->uio_resid > VFS_TO_TMPFS(vp->v_mount)->tm_maxfilesize) return (EFBIG); if (vn_rlimit_fsize(vp, uio, uio->uio_td)) return (EFBIG); - extended = uio->uio_offset + uio->uio_resid > node->tn_size; - if (extended) { + if (uio->uio_offset + uio->uio_resid > node->tn_size) { error = tmpfs_reg_resize(vp, uio->uio_offset + uio->uio_resid, FALSE); if (error != 0) goto out; } error = uiomove_object(node->tn_reg.tn_aobj, node->tn_size, uio); node->tn_status |= TMPFS_NODE_ACCESSED | TMPFS_NODE_MODIFIED | - (extended ? TMPFS_NODE_CHANGED : 0); + TMPFS_NODE_CHANGED; if (node->tn_mode & (S_ISUID | S_ISGID)) { if (priv_check_cred(v->a_cred, PRIV_VFS_RETAINSUGID, 0)) node->tn_mode &= ~(S_ISUID | S_ISGID); } if (error != 0) (void)tmpfs_reg_resize(vp, oldsize, TRUE); out: MPASS(IMPLIES(error == 0, uio->uio_resid == 0)); MPASS(IMPLIES(error != 0, oldsize == node->tn_size)); return (error); } static int tmpfs_fsync(struct vop_fsync_args *v) { struct vnode *vp = v->a_vp; MPASS(VOP_ISLOCKED(vp)); tmpfs_check_mtime(vp); tmpfs_update(vp); return 0; } static int tmpfs_remove(struct vop_remove_args *v) { struct vnode *dvp = v->a_dvp; struct vnode *vp = v->a_vp; int error; struct tmpfs_dirent *de; struct tmpfs_mount *tmp; struct tmpfs_node *dnode; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(dvp)); MPASS(VOP_ISLOCKED(vp)); if (vp->v_type == VDIR) { error = EISDIR; goto out; } dnode = VP_TO_TMPFS_DIR(dvp); node = VP_TO_TMPFS_NODE(vp); tmp = VFS_TO_TMPFS(vp->v_mount); de = tmpfs_dir_lookup(dnode, node, v->a_cnp); MPASS(de != NULL); /* Files marked as immutable or append-only cannot be deleted. */ if ((node->tn_flags & (IMMUTABLE | APPEND | NOUNLINK)) || (dnode->tn_flags & APPEND)) { error = EPERM; goto out; } /* Remove the entry from the directory; as it is a file, we do not * have to change the number of hard links of the directory. */ tmpfs_dir_detach(dvp, de); if (v->a_cnp->cn_flags & DOWHITEOUT) tmpfs_dir_whiteout_add(dvp, v->a_cnp); /* Free the directory entry we just deleted. Note that the node * referred by it will not be removed until the vnode is really * reclaimed. */ tmpfs_free_dirent(tmp, de); node->tn_status |= TMPFS_NODE_ACCESSED | TMPFS_NODE_CHANGED; error = 0; out: return error; } static int tmpfs_link(struct vop_link_args *v) { struct vnode *dvp = v->a_tdvp; struct vnode *vp = v->a_vp; struct componentname *cnp = v->a_cnp; int error; struct tmpfs_dirent *de; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(dvp)); MPASS(cnp->cn_flags & HASBUF); MPASS(dvp != vp); /* XXX When can this be false? */ node = VP_TO_TMPFS_NODE(vp); /* Ensure that we do not overflow the maximum number of links imposed * by the system. */ MPASS(node->tn_links <= LINK_MAX); if (node->tn_links == LINK_MAX) { error = EMLINK; goto out; } /* We cannot create links of files marked immutable or append-only. */ if (node->tn_flags & (IMMUTABLE | APPEND)) { error = EPERM; goto out; } /* Allocate a new directory entry to represent the node. */ error = tmpfs_alloc_dirent(VFS_TO_TMPFS(vp->v_mount), node, cnp->cn_nameptr, cnp->cn_namelen, &de); if (error != 0) goto out; /* Insert the new directory entry into the appropriate directory. */ if (cnp->cn_flags & ISWHITEOUT) tmpfs_dir_whiteout_remove(dvp, cnp); tmpfs_dir_attach(dvp, de); /* vp link count has changed, so update node times. */ node->tn_status |= TMPFS_NODE_CHANGED; tmpfs_update(vp); error = 0; out: return error; } /* * We acquire all but fdvp locks using non-blocking acquisitions. If we * fail to acquire any lock in the path we will drop all held locks, * acquire the new lock in a blocking fashion, and then release it and * restart the rename. This acquire/release step ensures that we do not * spin on a lock waiting for release. On error release all vnode locks * and decrement references the way tmpfs_rename() would do. */ static int tmpfs_rename_relock(struct vnode *fdvp, struct vnode **fvpp, struct vnode *tdvp, struct vnode **tvpp, struct componentname *fcnp, struct componentname *tcnp) { struct vnode *nvp; struct mount *mp; struct tmpfs_dirent *de; int error, restarts = 0; VOP_UNLOCK(tdvp, 0); if (*tvpp != NULL && *tvpp != tdvp) VOP_UNLOCK(*tvpp, 0); mp = fdvp->v_mount; relock: restarts += 1; error = vn_lock(fdvp, LK_EXCLUSIVE); if (error) goto releout; if (vn_lock(tdvp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { VOP_UNLOCK(fdvp, 0); error = vn_lock(tdvp, LK_EXCLUSIVE); if (error) goto releout; VOP_UNLOCK(tdvp, 0); goto relock; } /* * Re-resolve fvp to be certain it still exists and fetch the * correct vnode. */ de = tmpfs_dir_lookup(VP_TO_TMPFS_DIR(fdvp), NULL, fcnp); if (de == NULL) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(tdvp, 0); if ((fcnp->cn_flags & ISDOTDOT) != 0 || (fcnp->cn_namelen == 1 && fcnp->cn_nameptr[0] == '.')) error = EINVAL; else error = ENOENT; goto releout; } error = tmpfs_alloc_vp(mp, de->td_node, LK_EXCLUSIVE | LK_NOWAIT, &nvp); if (error != 0) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(tdvp, 0); if (error != EBUSY) goto releout; error = tmpfs_alloc_vp(mp, de->td_node, LK_EXCLUSIVE, &nvp); if (error != 0) goto releout; VOP_UNLOCK(nvp, 0); /* * Concurrent rename race. */ if (nvp == tdvp) { vrele(nvp); error = EINVAL; goto releout; } vrele(*fvpp); *fvpp = nvp; goto relock; } vrele(*fvpp); *fvpp = nvp; VOP_UNLOCK(*fvpp, 0); /* * Re-resolve tvp and acquire the vnode lock if present. */ de = tmpfs_dir_lookup(VP_TO_TMPFS_DIR(tdvp), NULL, tcnp); /* * If tvp disappeared we just carry on. */ if (de == NULL && *tvpp != NULL) { vrele(*tvpp); *tvpp = NULL; } /* * Get the tvp ino if the lookup succeeded. We may have to restart * if the non-blocking acquire fails. */ if (de != NULL) { nvp = NULL; error = tmpfs_alloc_vp(mp, de->td_node, LK_EXCLUSIVE | LK_NOWAIT, &nvp); if (*tvpp != NULL) vrele(*tvpp); *tvpp = nvp; if (error != 0) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(tdvp, 0); if (error != EBUSY) goto releout; error = tmpfs_alloc_vp(mp, de->td_node, LK_EXCLUSIVE, &nvp); if (error != 0) goto releout; VOP_UNLOCK(nvp, 0); /* * fdvp contains fvp, thus tvp (=fdvp) is not empty. */ if (nvp == fdvp) { error = ENOTEMPTY; goto releout; } goto relock; } } tmpfs_rename_restarts += restarts; return (0); releout: vrele(fdvp); vrele(*fvpp); vrele(tdvp); if (*tvpp != NULL) vrele(*tvpp); tmpfs_rename_restarts += restarts; return (error); } static int tmpfs_rename(struct vop_rename_args *v) { struct vnode *fdvp = v->a_fdvp; struct vnode *fvp = v->a_fvp; struct componentname *fcnp = v->a_fcnp; struct vnode *tdvp = v->a_tdvp; struct vnode *tvp = v->a_tvp; struct componentname *tcnp = v->a_tcnp; struct mount *mp = NULL; char *newname; int error; struct tmpfs_dirent *de; struct tmpfs_mount *tmp; struct tmpfs_node *fdnode; struct tmpfs_node *fnode; struct tmpfs_node *tnode; struct tmpfs_node *tdnode; MPASS(VOP_ISLOCKED(tdvp)); MPASS(IMPLIES(tvp != NULL, VOP_ISLOCKED(tvp))); MPASS(fcnp->cn_flags & HASBUF); MPASS(tcnp->cn_flags & HASBUF); /* Disallow cross-device renames. * XXX Why isn't this done by the caller? */ if (fvp->v_mount != tdvp->v_mount || (tvp != NULL && fvp->v_mount != tvp->v_mount)) { error = EXDEV; goto out; } /* If source and target are the same file, there is nothing to do. */ if (fvp == tvp) { error = 0; goto out; } /* If we need to move the directory between entries, lock the * source so that we can safely operate on it. */ if (fdvp != tdvp && fdvp != tvp) { if (vn_lock(fdvp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { mp = tdvp->v_mount; error = vfs_busy(mp, 0); if (error != 0) { mp = NULL; goto out; } error = tmpfs_rename_relock(fdvp, &fvp, tdvp, &tvp, fcnp, tcnp); if (error != 0) { vfs_unbusy(mp); return (error); } ASSERT_VOP_ELOCKED(fdvp, "tmpfs_rename: fdvp not locked"); ASSERT_VOP_ELOCKED(tdvp, "tmpfs_rename: tdvp not locked"); if (tvp != NULL) ASSERT_VOP_ELOCKED(tvp, "tmpfs_rename: tvp not locked"); if (fvp == tvp) { error = 0; goto out_locked; } } } tmp = VFS_TO_TMPFS(tdvp->v_mount); tdnode = VP_TO_TMPFS_DIR(tdvp); tnode = (tvp == NULL) ? NULL : VP_TO_TMPFS_NODE(tvp); fdnode = VP_TO_TMPFS_DIR(fdvp); fnode = VP_TO_TMPFS_NODE(fvp); de = tmpfs_dir_lookup(fdnode, fnode, fcnp); /* Entry can disappear before we lock fdvp, * also avoid manipulating '.' and '..' entries. */ if (de == NULL) { if ((fcnp->cn_flags & ISDOTDOT) != 0 || (fcnp->cn_namelen == 1 && fcnp->cn_nameptr[0] == '.')) error = EINVAL; else error = ENOENT; goto out_locked; } MPASS(de->td_node == fnode); /* If re-naming a directory to another preexisting directory * ensure that the target directory is empty so that its * removal causes no side effects. * Kern_rename gurantees the destination to be a directory * if the source is one. */ if (tvp != NULL) { MPASS(tnode != NULL); if ((tnode->tn_flags & (NOUNLINK | IMMUTABLE | APPEND)) || (tdnode->tn_flags & (APPEND | IMMUTABLE))) { error = EPERM; goto out_locked; } if (fnode->tn_type == VDIR && tnode->tn_type == VDIR) { if (tnode->tn_size > 0) { error = ENOTEMPTY; goto out_locked; } } else if (fnode->tn_type == VDIR && tnode->tn_type != VDIR) { error = ENOTDIR; goto out_locked; } else if (fnode->tn_type != VDIR && tnode->tn_type == VDIR) { error = EISDIR; goto out_locked; } else { MPASS(fnode->tn_type != VDIR && tnode->tn_type != VDIR); } } if ((fnode->tn_flags & (NOUNLINK | IMMUTABLE | APPEND)) || (fdnode->tn_flags & (APPEND | IMMUTABLE))) { error = EPERM; goto out_locked; } /* Ensure that we have enough memory to hold the new name, if it * has to be changed. */ if (fcnp->cn_namelen != tcnp->cn_namelen || bcmp(fcnp->cn_nameptr, tcnp->cn_nameptr, fcnp->cn_namelen) != 0) { newname = malloc(tcnp->cn_namelen, M_TMPFSNAME, M_WAITOK); } else newname = NULL; /* If the node is being moved to another directory, we have to do * the move. */ if (fdnode != tdnode) { /* In case we are moving a directory, we have to adjust its * parent to point to the new parent. */ if (de->td_node->tn_type == VDIR) { struct tmpfs_node *n; /* Ensure the target directory is not a child of the * directory being moved. Otherwise, we'd end up * with stale nodes. */ n = tdnode; /* TMPFS_LOCK garanties that no nodes are freed while * traversing the list. Nodes can only be marked as * removed: tn_parent == NULL. */ TMPFS_LOCK(tmp); TMPFS_NODE_LOCK(n); while (n != n->tn_dir.tn_parent) { struct tmpfs_node *parent; if (n == fnode) { TMPFS_NODE_UNLOCK(n); TMPFS_UNLOCK(tmp); error = EINVAL; if (newname != NULL) free(newname, M_TMPFSNAME); goto out_locked; } parent = n->tn_dir.tn_parent; TMPFS_NODE_UNLOCK(n); if (parent == NULL) { n = NULL; break; } TMPFS_NODE_LOCK(parent); if (parent->tn_dir.tn_parent == NULL) { TMPFS_NODE_UNLOCK(parent); n = NULL; break; } n = parent; } TMPFS_UNLOCK(tmp); if (n == NULL) { error = EINVAL; if (newname != NULL) free(newname, M_TMPFSNAME); goto out_locked; } TMPFS_NODE_UNLOCK(n); /* Adjust the parent pointer. */ TMPFS_VALIDATE_DIR(fnode); TMPFS_NODE_LOCK(de->td_node); de->td_node->tn_dir.tn_parent = tdnode; TMPFS_NODE_UNLOCK(de->td_node); /* As a result of changing the target of the '..' * entry, the link count of the source and target * directories has to be adjusted. */ TMPFS_NODE_LOCK(tdnode); TMPFS_ASSERT_LOCKED(tdnode); tdnode->tn_links++; TMPFS_NODE_UNLOCK(tdnode); TMPFS_NODE_LOCK(fdnode); TMPFS_ASSERT_LOCKED(fdnode); fdnode->tn_links--; TMPFS_NODE_UNLOCK(fdnode); } } /* Do the move: just remove the entry from the source directory * and insert it into the target one. */ tmpfs_dir_detach(fdvp, de); if (fcnp->cn_flags & DOWHITEOUT) tmpfs_dir_whiteout_add(fdvp, fcnp); if (tcnp->cn_flags & ISWHITEOUT) tmpfs_dir_whiteout_remove(tdvp, tcnp); /* If the name has changed, we need to make it effective by changing * it in the directory entry. */ if (newname != NULL) { MPASS(tcnp->cn_namelen <= MAXNAMLEN); free(de->ud.td_name, M_TMPFSNAME); de->ud.td_name = newname; tmpfs_dirent_init(de, tcnp->cn_nameptr, tcnp->cn_namelen); fnode->tn_status |= TMPFS_NODE_CHANGED; tdnode->tn_status |= TMPFS_NODE_MODIFIED; } /* If we are overwriting an entry, we have to remove the old one * from the target directory. */ if (tvp != NULL) { struct tmpfs_dirent *tde; /* Remove the old entry from the target directory. */ tde = tmpfs_dir_lookup(tdnode, tnode, tcnp); tmpfs_dir_detach(tdvp, tde); /* Free the directory entry we just deleted. Note that the * node referred by it will not be removed until the vnode is * really reclaimed. */ tmpfs_free_dirent(VFS_TO_TMPFS(tvp->v_mount), tde); } tmpfs_dir_attach(tdvp, de); cache_purge(fvp); if (tvp != NULL) cache_purge(tvp); cache_purge_negative(tdvp); error = 0; out_locked: if (fdvp != tdvp && fdvp != tvp) VOP_UNLOCK(fdvp, 0); out: /* Release target nodes. */ /* XXX: I don't understand when tdvp can be the same as tvp, but * other code takes care of this... */ if (tdvp == tvp) vrele(tdvp); else vput(tdvp); if (tvp != NULL) vput(tvp); /* Release source nodes. */ vrele(fdvp); vrele(fvp); if (mp != NULL) vfs_unbusy(mp); return error; } static int tmpfs_mkdir(struct vop_mkdir_args *v) { struct vnode *dvp = v->a_dvp; struct vnode **vpp = v->a_vpp; struct componentname *cnp = v->a_cnp; struct vattr *vap = v->a_vap; MPASS(vap->va_type == VDIR); return tmpfs_alloc_file(dvp, vpp, vap, cnp, NULL); } static int tmpfs_rmdir(struct vop_rmdir_args *v) { struct vnode *dvp = v->a_dvp; struct vnode *vp = v->a_vp; int error; struct tmpfs_dirent *de; struct tmpfs_mount *tmp; struct tmpfs_node *dnode; struct tmpfs_node *node; MPASS(VOP_ISLOCKED(dvp)); MPASS(VOP_ISLOCKED(vp)); tmp = VFS_TO_TMPFS(dvp->v_mount); dnode = VP_TO_TMPFS_DIR(dvp); node = VP_TO_TMPFS_DIR(vp); /* Directories with more than two entries ('.' and '..') cannot be * removed. */ if (node->tn_size > 0) { error = ENOTEMPTY; goto out; } if ((dnode->tn_flags & APPEND) || (node->tn_flags & (NOUNLINK | IMMUTABLE | APPEND))) { error = EPERM; goto out; } /* This invariant holds only if we are not trying to remove "..". * We checked for that above so this is safe now. */ MPASS(node->tn_dir.tn_parent == dnode); /* Get the directory entry associated with node (vp). This was * filled by tmpfs_lookup while looking up the entry. */ de = tmpfs_dir_lookup(dnode, node, v->a_cnp); MPASS(TMPFS_DIRENT_MATCHES(de, v->a_cnp->cn_nameptr, v->a_cnp->cn_namelen)); /* Check flags to see if we are allowed to remove the directory. */ if (dnode->tn_flags & APPEND || node->tn_flags & (NOUNLINK | IMMUTABLE | APPEND)) { error = EPERM; goto out; } /* Detach the directory entry from the directory (dnode). */ tmpfs_dir_detach(dvp, de); if (v->a_cnp->cn_flags & DOWHITEOUT) tmpfs_dir_whiteout_add(dvp, v->a_cnp); /* No vnode should be allocated for this entry from this point */ TMPFS_NODE_LOCK(node); TMPFS_ASSERT_ELOCKED(node); node->tn_links--; node->tn_dir.tn_parent = NULL; node->tn_status |= TMPFS_NODE_ACCESSED | TMPFS_NODE_CHANGED | \ TMPFS_NODE_MODIFIED; TMPFS_NODE_UNLOCK(node); TMPFS_NODE_LOCK(dnode); TMPFS_ASSERT_ELOCKED(dnode); dnode->tn_links--; dnode->tn_status |= TMPFS_NODE_ACCESSED | \ TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED; TMPFS_NODE_UNLOCK(dnode); cache_purge(dvp); cache_purge(vp); /* Free the directory entry we just deleted. Note that the node * referred by it will not be removed until the vnode is really * reclaimed. */ tmpfs_free_dirent(tmp, de); /* Release the deleted vnode (will destroy the node, notify * interested parties and clean it from the cache). */ dnode->tn_status |= TMPFS_NODE_CHANGED; tmpfs_update(dvp); error = 0; out: return error; } static int tmpfs_symlink(struct vop_symlink_args *v) { struct vnode *dvp = v->a_dvp; struct vnode **vpp = v->a_vpp; struct componentname *cnp = v->a_cnp; struct vattr *vap = v->a_vap; char *target = v->a_target; #ifdef notyet /* XXX FreeBSD BUG: kern_symlink is not setting VLNK */ MPASS(vap->va_type == VLNK); #else vap->va_type = VLNK; #endif return tmpfs_alloc_file(dvp, vpp, vap, cnp, target); } static int tmpfs_readdir(struct vop_readdir_args *v) { struct vnode *vp = v->a_vp; struct uio *uio = v->a_uio; int *eofflag = v->a_eofflag; u_long **cookies = v->a_cookies; int *ncookies = v->a_ncookies; int error; ssize_t startresid; int maxcookies; struct tmpfs_node *node; /* This operation only makes sense on directory nodes. */ if (vp->v_type != VDIR) return ENOTDIR; maxcookies = 0; node = VP_TO_TMPFS_DIR(vp); startresid = uio->uio_resid; /* Allocate cookies for NFS and compat modules. */ if (cookies != NULL && ncookies != NULL) { maxcookies = howmany(node->tn_size, sizeof(struct tmpfs_dirent)) + 2; *cookies = malloc(maxcookies * sizeof(**cookies), M_TEMP, M_WAITOK); *ncookies = 0; } if (cookies == NULL) error = tmpfs_dir_getdents(node, uio, 0, NULL, NULL); else error = tmpfs_dir_getdents(node, uio, maxcookies, *cookies, ncookies); /* Buffer was filled without hitting EOF. */ if (error == EJUSTRETURN) error = (uio->uio_resid != startresid) ? 0 : EINVAL; if (error != 0 && cookies != NULL) free(*cookies, M_TEMP); if (eofflag != NULL) *eofflag = (error == 0 && uio->uio_offset == TMPFS_DIRCOOKIE_EOF); return error; } static int tmpfs_readlink(struct vop_readlink_args *v) { struct vnode *vp = v->a_vp; struct uio *uio = v->a_uio; int error; struct tmpfs_node *node; MPASS(uio->uio_offset == 0); MPASS(vp->v_type == VLNK); node = VP_TO_TMPFS_NODE(vp); error = uiomove(node->tn_link, MIN(node->tn_size, uio->uio_resid), uio); node->tn_status |= TMPFS_NODE_ACCESSED; return error; } static int tmpfs_inactive(struct vop_inactive_args *v) { struct vnode *vp; struct tmpfs_node *node; vp = v->a_vp; node = VP_TO_TMPFS_NODE(vp); if (node->tn_links == 0) vrecycle(vp); else tmpfs_check_mtime(vp); return (0); } int tmpfs_reclaim(struct vop_reclaim_args *v) { struct vnode *vp = v->a_vp; struct tmpfs_mount *tmp; struct tmpfs_node *node; node = VP_TO_TMPFS_NODE(vp); tmp = VFS_TO_TMPFS(vp->v_mount); if (vp->v_type == VREG) tmpfs_destroy_vobject(vp, node->tn_reg.tn_aobj); else vnode_destroy_vobject(vp); vp->v_object = NULL; cache_purge(vp); TMPFS_NODE_LOCK(node); TMPFS_ASSERT_ELOCKED(node); tmpfs_free_vp(vp); /* If the node referenced by this vnode was deleted by the user, * we must free its associated data structures (now that the vnode * is being reclaimed). */ if (node->tn_links == 0 && (node->tn_vpstate & TMPFS_VNODE_ALLOCATING) == 0) { node->tn_vpstate = TMPFS_VNODE_DOOMED; TMPFS_NODE_UNLOCK(node); tmpfs_free_node(tmp, node); } else TMPFS_NODE_UNLOCK(node); MPASS(vp->v_data == NULL); return 0; } static int tmpfs_print(struct vop_print_args *v) { struct vnode *vp = v->a_vp; struct tmpfs_node *node; node = VP_TO_TMPFS_NODE(vp); printf("tag VT_TMPFS, tmpfs_node %p, flags 0x%lx, links %d\n", node, node->tn_flags, node->tn_links); printf("\tmode 0%o, owner %d, group %d, size %jd, status 0x%x\n", node->tn_mode, node->tn_uid, node->tn_gid, (intmax_t)node->tn_size, node->tn_status); if (vp->v_type == VFIFO) fifo_printinfo(vp); printf("\n"); return 0; } static int tmpfs_pathconf(struct vop_pathconf_args *v) { int name = v->a_name; register_t *retval = v->a_retval; int error; error = 0; switch (name) { case _PC_LINK_MAX: *retval = LINK_MAX; break; case _PC_NAME_MAX: *retval = NAME_MAX; break; case _PC_PATH_MAX: *retval = PATH_MAX; break; case _PC_PIPE_BUF: *retval = PIPE_BUF; break; case _PC_CHOWN_RESTRICTED: *retval = 1; break; case _PC_NO_TRUNC: *retval = 1; break; case _PC_SYNC_IO: *retval = 1; break; case _PC_FILESIZEBITS: *retval = 0; /* XXX Don't know which value should I return. */ break; default: error = EINVAL; } return error; } static int tmpfs_vptofh(struct vop_vptofh_args *ap) { struct tmpfs_fid *tfhp; struct tmpfs_node *node; tfhp = (struct tmpfs_fid *)ap->a_fhp; node = VP_TO_TMPFS_NODE(ap->a_vp); tfhp->tf_len = sizeof(struct tmpfs_fid); tfhp->tf_id = node->tn_id; tfhp->tf_gen = node->tn_gen; return (0); } static int tmpfs_whiteout(struct vop_whiteout_args *ap) { struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; struct tmpfs_dirent *de; switch (ap->a_flags) { case LOOKUP: return (0); case CREATE: de = tmpfs_dir_lookup(VP_TO_TMPFS_DIR(dvp), NULL, cnp); if (de != NULL) return (de->td_node == NULL ? 0 : EEXIST); return (tmpfs_dir_whiteout_add(dvp, cnp)); case DELETE: tmpfs_dir_whiteout_remove(dvp, cnp); return (0); default: panic("tmpfs_whiteout: unknown op"); } } /* * vnode operations vector used for files stored in a tmpfs file system. */ struct vop_vector tmpfs_vnodeop_entries = { .vop_default = &default_vnodeops, .vop_lookup = vfs_cache_lookup, .vop_cachedlookup = tmpfs_lookup, .vop_create = tmpfs_create, .vop_mknod = tmpfs_mknod, .vop_open = tmpfs_open, .vop_close = tmpfs_close, .vop_access = tmpfs_access, .vop_getattr = tmpfs_getattr, .vop_setattr = tmpfs_setattr, .vop_read = tmpfs_read, .vop_write = tmpfs_write, .vop_fsync = tmpfs_fsync, .vop_remove = tmpfs_remove, .vop_link = tmpfs_link, .vop_rename = tmpfs_rename, .vop_mkdir = tmpfs_mkdir, .vop_rmdir = tmpfs_rmdir, .vop_symlink = tmpfs_symlink, .vop_readdir = tmpfs_readdir, .vop_readlink = tmpfs_readlink, .vop_inactive = tmpfs_inactive, .vop_reclaim = tmpfs_reclaim, .vop_print = tmpfs_print, .vop_pathconf = tmpfs_pathconf, .vop_vptofh = tmpfs_vptofh, .vop_whiteout = tmpfs_whiteout, .vop_bmap = VOP_EOPNOTSUPP, }; diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index 5b42c6fd118a..317e05bfb57a 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -1,3762 +1,3812 @@ /*- * Copyright (c) 2004, David Xu * Copyright (c) 2002, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include "opt_umtx_profiling.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef COMPAT_FREEBSD32 #include #endif #define _UMUTEX_TRY 1 #define _UMUTEX_WAIT 2 #ifdef UMTX_PROFILING #define UPROF_PERC_BIGGER(w, f, sw, sf) \ (((w) > (sw)) || ((w) == (sw) && (f) > (sf))) #endif /* Priority inheritance mutex info. */ struct umtx_pi { /* Owner thread */ struct thread *pi_owner; /* Reference count */ int pi_refcount; /* List entry to link umtx holding by thread */ TAILQ_ENTRY(umtx_pi) pi_link; /* List entry in hash */ TAILQ_ENTRY(umtx_pi) pi_hashlink; /* List for waiters */ TAILQ_HEAD(,umtx_q) pi_blocked; /* Identify a userland lock object */ struct umtx_key pi_key; }; /* A userland synchronous object user. */ struct umtx_q { /* Linked list for the hash. */ TAILQ_ENTRY(umtx_q) uq_link; /* Umtx key. */ struct umtx_key uq_key; /* Umtx flags. */ int uq_flags; #define UQF_UMTXQ 0x0001 /* The thread waits on. */ struct thread *uq_thread; /* * Blocked on PI mutex. read can use chain lock * or umtx_lock, write must have both chain lock and * umtx_lock being hold. */ struct umtx_pi *uq_pi_blocked; /* On blocked list */ TAILQ_ENTRY(umtx_q) uq_lockq; /* Thread contending with us */ TAILQ_HEAD(,umtx_pi) uq_pi_contested; /* Inherited priority from PP mutex */ u_char uq_inherited_pri; /* Spare queue ready to be reused */ struct umtxq_queue *uq_spare_queue; /* The queue we on */ struct umtxq_queue *uq_cur_queue; }; TAILQ_HEAD(umtxq_head, umtx_q); /* Per-key wait-queue */ struct umtxq_queue { struct umtxq_head head; struct umtx_key key; LIST_ENTRY(umtxq_queue) link; int length; }; LIST_HEAD(umtxq_list, umtxq_queue); /* Userland lock object's wait-queue chain */ struct umtxq_chain { /* Lock for this chain. */ struct mtx uc_lock; /* List of sleep queues. */ struct umtxq_list uc_queue[2]; #define UMTX_SHARED_QUEUE 0 #define UMTX_EXCLUSIVE_QUEUE 1 LIST_HEAD(, umtxq_queue) uc_spare_queue; /* Busy flag */ char uc_busy; /* Chain lock waiters */ int uc_waiters; /* All PI in the list */ TAILQ_HEAD(,umtx_pi) uc_pi_list; #ifdef UMTX_PROFILING u_int length; u_int max_length; #endif }; #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) /* * Don't propagate time-sharing priority, there is a security reason, * a user can simply introduce PI-mutex, let thread A lock the mutex, * and let another thread B block on the mutex, because B is * sleeping, its priority will be boosted, this causes A's priority to * be boosted via priority propagating too and will never be lowered even * if it is using 100%CPU, this is unfair to other processes. */ #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ PRI_MAX_TIMESHARE : (td)->td_user_pri) #define GOLDEN_RATIO_PRIME 2654404609U #define UMTX_CHAINS 512 #define UMTX_SHIFTS (__WORD_BIT - 9) #define GET_SHARE(flags) \ (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) #define BUSY_SPINS 200 struct abs_timeout { int clockid; struct timespec cur; struct timespec end; }; static uma_zone_t umtx_pi_zone; static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); static int umtx_pi_allocated; static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, &umtx_pi_allocated, 0, "Allocated umtx_pi"); #ifdef UMTX_PROFILING static long max_length; SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length"); static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD, 0, "umtx chain stats"); #endif static void umtxq_sysinit(void *); static void umtxq_hash(struct umtx_key *key); static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); static void umtxq_lock(struct umtx_key *key); static void umtxq_unlock(struct umtx_key *key); static void umtxq_busy(struct umtx_key *key); static void umtxq_unbusy(struct umtx_key *key); static void umtxq_insert_queue(struct umtx_q *uq, int q); static void umtxq_remove_queue(struct umtx_q *uq, int q); static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *); static int umtxq_count(struct umtx_key *key); static struct umtx_pi *umtx_pi_alloc(int); static void umtx_pi_free(struct umtx_pi *pi); static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags); static void umtx_thread_cleanup(struct thread *td); static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, struct image_params *imgp __unused); SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE) #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE) #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE) static struct mtx umtx_lock; #ifdef UMTX_PROFILING static void umtx_init_profiling(void) { struct sysctl_oid *chain_oid; char chain_name[10]; int i; for (i = 0; i < UMTX_CHAINS; ++i) { snprintf(chain_name, sizeof(chain_name), "%d", i); chain_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO, chain_name, CTLFLAG_RD, NULL, "umtx hash stats"); SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO, "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL); SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO, "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL); } } static int sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS) { char buf[512]; struct sbuf sb; struct umtxq_chain *uc; u_int fract, i, j, tot, whole; u_int sf0, sf1, sf2, sf3, sf4; u_int si0, si1, si2, si3, si4; u_int sw0, sw1, sw2, sw3, sw4; sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN); for (i = 0; i < 2; i++) { tot = 0; for (j = 0; j < UMTX_CHAINS; ++j) { uc = &umtxq_chains[i][j]; mtx_lock(&uc->uc_lock); tot += uc->max_length; mtx_unlock(&uc->uc_lock); } if (tot == 0) sbuf_printf(&sb, "%u) Empty ", i); else { sf0 = sf1 = sf2 = sf3 = sf4 = 0; si0 = si1 = si2 = si3 = si4 = 0; sw0 = sw1 = sw2 = sw3 = sw4 = 0; for (j = 0; j < UMTX_CHAINS; j++) { uc = &umtxq_chains[i][j]; mtx_lock(&uc->uc_lock); whole = uc->max_length * 100; mtx_unlock(&uc->uc_lock); fract = (whole % tot) * 100; if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) { sf0 = fract; si0 = j; sw0 = whole; } else if (UPROF_PERC_BIGGER(whole, fract, sw1, sf1)) { sf1 = fract; si1 = j; sw1 = whole; } else if (UPROF_PERC_BIGGER(whole, fract, sw2, sf2)) { sf2 = fract; si2 = j; sw2 = whole; } else if (UPROF_PERC_BIGGER(whole, fract, sw3, sf3)) { sf3 = fract; si3 = j; sw3 = whole; } else if (UPROF_PERC_BIGGER(whole, fract, sw4, sf4)) { sf4 = fract; si4 = j; sw4 = whole; } } sbuf_printf(&sb, "queue %u:\n", i); sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot, sf0 / tot, si0); sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot, sf1 / tot, si1); sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot, sf2 / tot, si2); sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot, sf3 / tot, si3); sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot, sf4 / tot, si4); } } sbuf_trim(&sb); sbuf_finish(&sb); sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); sbuf_delete(&sb); return (0); } static int sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS) { struct umtxq_chain *uc; u_int i, j; int clear, error; clear = 0; error = sysctl_handle_int(oidp, &clear, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (clear != 0) { for (i = 0; i < 2; ++i) { for (j = 0; j < UMTX_CHAINS; ++j) { uc = &umtxq_chains[i][j]; mtx_lock(&uc->uc_lock); uc->length = 0; uc->max_length = 0; mtx_unlock(&uc->uc_lock); } } } return (0); } SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, sysctl_debug_umtx_chains_clear, "I", "Clear umtx chains statistics"); SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0, sysctl_debug_umtx_chains_peaks, "A", "Highest peaks in chains max length"); #endif static void umtxq_sysinit(void *arg __unused) { int i, j; umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); for (i = 0; i < 2; ++i) { for (j = 0; j < UMTX_CHAINS; ++j) { mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL, MTX_DEF | MTX_DUPOK); LIST_INIT(&umtxq_chains[i][j].uc_queue[0]); LIST_INIT(&umtxq_chains[i][j].uc_queue[1]); LIST_INIT(&umtxq_chains[i][j].uc_spare_queue); TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list); umtxq_chains[i][j].uc_busy = 0; umtxq_chains[i][j].uc_waiters = 0; #ifdef UMTX_PROFILING umtxq_chains[i][j].length = 0; umtxq_chains[i][j].max_length = 0; #endif } } #ifdef UMTX_PROFILING umtx_init_profiling(); #endif mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN); EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, EVENTHANDLER_PRI_ANY); } struct umtx_q * umtxq_alloc(void) { struct umtx_q *uq; uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, M_WAITOK | M_ZERO); TAILQ_INIT(&uq->uq_spare_queue->head); TAILQ_INIT(&uq->uq_pi_contested); uq->uq_inherited_pri = PRI_MAX; return (uq); } void umtxq_free(struct umtx_q *uq) { MPASS(uq->uq_spare_queue != NULL); free(uq->uq_spare_queue, M_UMTX); free(uq, M_UMTX); } static inline void umtxq_hash(struct umtx_key *key) { unsigned n = (uintptr_t)key->info.both.a + key->info.both.b; key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; } static inline struct umtxq_chain * umtxq_getchain(struct umtx_key *key) { if (key->type <= TYPE_SEM) return (&umtxq_chains[1][key->hash]); return (&umtxq_chains[0][key->hash]); } /* * Lock a chain. */ static inline void umtxq_lock(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_lock(&uc->uc_lock); } /* * Unlock a chain. */ static inline void umtxq_unlock(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_unlock(&uc->uc_lock); } /* * Set chain to busy state when following operation * may be blocked (kernel mutex can not be used). */ static inline void umtxq_busy(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_assert(&uc->uc_lock, MA_OWNED); if (uc->uc_busy) { #ifdef SMP if (smp_cpus > 1) { int count = BUSY_SPINS; if (count > 0) { umtxq_unlock(key); while (uc->uc_busy && --count > 0) cpu_spinwait(); umtxq_lock(key); } } #endif while (uc->uc_busy) { uc->uc_waiters++; msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); uc->uc_waiters--; } } uc->uc_busy = 1; } /* * Unbusy a chain. */ static inline void umtxq_unbusy(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_assert(&uc->uc_lock, MA_OWNED); KASSERT(uc->uc_busy != 0, ("not busy")); uc->uc_busy = 0; if (uc->uc_waiters) wakeup_one(uc); } static inline void umtxq_unbusy_unlocked(struct umtx_key *key) { umtxq_lock(key); umtxq_unbusy(key); umtxq_unlock(key); } static struct umtxq_queue * umtxq_queue_lookup(struct umtx_key *key, int q) { struct umtxq_queue *uh; struct umtxq_chain *uc; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); LIST_FOREACH(uh, &uc->uc_queue[q], link) { if (umtx_key_match(&uh->key, key)) return (uh); } return (NULL); } static inline void umtxq_insert_queue(struct umtx_q *uq, int q) { struct umtxq_queue *uh; struct umtxq_chain *uc; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue")); uh = umtxq_queue_lookup(&uq->uq_key, q); if (uh != NULL) { LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link); } else { uh = uq->uq_spare_queue; uh->key = uq->uq_key; LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link); #ifdef UMTX_PROFILING uc->length++; if (uc->length > uc->max_length) { uc->max_length = uc->length; if (uc->max_length > max_length) max_length = uc->max_length; } #endif } uq->uq_spare_queue = NULL; TAILQ_INSERT_TAIL(&uh->head, uq, uq_link); uh->length++; uq->uq_flags |= UQF_UMTXQ; uq->uq_cur_queue = uh; return; } static inline void umtxq_remove_queue(struct umtx_q *uq, int q) { struct umtxq_chain *uc; struct umtxq_queue *uh; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); if (uq->uq_flags & UQF_UMTXQ) { uh = uq->uq_cur_queue; TAILQ_REMOVE(&uh->head, uq, uq_link); uh->length--; uq->uq_flags &= ~UQF_UMTXQ; if (TAILQ_EMPTY(&uh->head)) { KASSERT(uh->length == 0, ("inconsistent umtxq_queue length")); #ifdef UMTX_PROFILING uc->length--; #endif LIST_REMOVE(uh, link); } else { uh = LIST_FIRST(&uc->uc_spare_queue); KASSERT(uh != NULL, ("uc_spare_queue is empty")); LIST_REMOVE(uh, link); } uq->uq_spare_queue = uh; uq->uq_cur_queue = NULL; } } /* * Check if there are multiple waiters */ static int umtxq_count(struct umtx_key *key) { struct umtxq_chain *uc; struct umtxq_queue *uh; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); if (uh != NULL) return (uh->length); return (0); } /* * Check if there are multiple PI waiters and returns first * waiter. */ static int umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) { struct umtxq_chain *uc; struct umtxq_queue *uh; *first = NULL; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); if (uh != NULL) { *first = TAILQ_FIRST(&uh->head); return (uh->length); } return (0); } static int umtxq_check_susp(struct thread *td) { struct proc *p; int error; /* * The check for TDF_NEEDSUSPCHK is racy, but it is enough to * eventually break the lockstep loop. */ if ((td->td_flags & TDF_NEEDSUSPCHK) == 0) return (0); error = 0; p = td->td_proc; PROC_LOCK(p); if (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) { if (p->p_flag & P_SINGLE_EXIT) error = EINTR; else error = ERESTART; } PROC_UNLOCK(p); return (error); } /* * Wake up threads waiting on an userland object. */ static int umtxq_signal_queue(struct umtx_key *key, int n_wake, int q) { struct umtxq_chain *uc; struct umtxq_queue *uh; struct umtx_q *uq; int ret; ret = 0; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); uh = umtxq_queue_lookup(key, q); if (uh != NULL) { while ((uq = TAILQ_FIRST(&uh->head)) != NULL) { umtxq_remove_queue(uq, q); wakeup(uq); if (++ret >= n_wake) return (ret); } } return (ret); } /* * Wake up specified thread. */ static inline void umtxq_signal_thread(struct umtx_q *uq) { struct umtxq_chain *uc; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); umtxq_remove(uq); wakeup(uq); } static inline int tstohz(const struct timespec *tsp) { struct timeval tv; TIMESPEC_TO_TIMEVAL(&tv, tsp); return tvtohz(&tv); } static void abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute, const struct timespec *timeout) { timo->clockid = clockid; if (!absolute) { kern_clock_gettime(curthread, clockid, &timo->end); timo->cur = timo->end; timespecadd(&timo->end, timeout); } else { timo->end = *timeout; kern_clock_gettime(curthread, clockid, &timo->cur); } } static void abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime) { abs_timeout_init(timo, umtxtime->_clockid, (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); } static inline void abs_timeout_update(struct abs_timeout *timo) { kern_clock_gettime(curthread, timo->clockid, &timo->cur); } static int abs_timeout_gethz(struct abs_timeout *timo) { struct timespec tts; if (timespeccmp(&timo->end, &timo->cur, <=)) return (-1); tts = timo->end; timespecsub(&tts, &timo->cur); return (tstohz(&tts)); } /* * Put thread into sleep state, before sleeping, check if * thread was removed from umtx queue. */ static inline int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime) { struct umtxq_chain *uc; int error, timo; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); for (;;) { if (!(uq->uq_flags & UQF_UMTXQ)) return (0); if (abstime != NULL) { timo = abs_timeout_gethz(abstime); if (timo < 0) return (ETIMEDOUT); } else timo = 0; error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo); if (error != EWOULDBLOCK) { umtxq_lock(&uq->uq_key); break; } if (abstime != NULL) abs_timeout_update(abstime); umtxq_lock(&uq->uq_key); } return (error); } /* * Convert userspace address into unique logical address. */ int umtx_key_get(void *addr, int type, int share, struct umtx_key *key) { struct thread *td = curthread; vm_map_t map; vm_map_entry_t entry; vm_pindex_t pindex; vm_prot_t prot; boolean_t wired; key->type = type; if (share == THREAD_SHARE) { key->shared = 0; key->info.private.vs = td->td_proc->p_vmspace; key->info.private.addr = (uintptr_t)addr; } else { MPASS(share == PROCESS_SHARE || share == AUTO_SHARE); map = &td->td_proc->p_vmspace->vm_map; if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE, &entry, &key->info.shared.object, &pindex, &prot, &wired) != KERN_SUCCESS) { return EFAULT; } if ((share == PROCESS_SHARE) || (share == AUTO_SHARE && VM_INHERIT_SHARE == entry->inheritance)) { key->shared = 1; key->info.shared.offset = entry->offset + entry->start - (vm_offset_t)addr; vm_object_reference(key->info.shared.object); } else { key->shared = 0; key->info.private.vs = td->td_proc->p_vmspace; key->info.private.addr = (uintptr_t)addr; } vm_map_lookup_done(map, entry); } umtxq_hash(key); return (0); } /* * Release key. */ void umtx_key_release(struct umtx_key *key) { if (key->shared) vm_object_deallocate(key->info.shared.object); } /* * Fetch and compare value, sleep on the address if value is not changed. */ static int do_wait(struct thread *td, void *addr, u_long id, struct _umtx_time *timeout, int compat32, int is_private) { struct abs_timeout timo; struct umtx_q *uq; u_long tmp; uint32_t tmp32; int error = 0; uq = td->td_umtxq; if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) return (error); if (timeout != NULL) abs_timeout_init2(&timo, timeout); umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); if (compat32 == 0) { error = fueword(addr, &tmp); if (error != 0) error = EFAULT; } else { error = fueword32(addr, &tmp32); if (error == 0) tmp = tmp32; else error = EFAULT; } umtxq_lock(&uq->uq_key); if (error == 0) { if (tmp == id) error = umtxq_sleep(uq, "uwait", timeout == NULL ? NULL : &timo); if ((uq->uq_flags & UQF_UMTXQ) == 0) error = 0; else umtxq_remove(uq); } else if ((uq->uq_flags & UQF_UMTXQ) != 0) { umtxq_remove(uq); } umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); if (error == ERESTART) error = EINTR; return (error); } /* * Wake up threads sleeping on the specified address. */ int kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) { struct umtx_key key; int ret; if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) return (ret); umtxq_lock(&key); ret = umtxq_signal(&key, n_wake); umtxq_unlock(&key); umtx_key_release(&key); return (0); } /* * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. */ static int do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, struct _umtx_time *timeout, int mode) { struct abs_timeout timo; struct umtx_q *uq; uint32_t owner, old, id; int error, rv; id = td->td_tid; uq = td->td_umtxq; error = 0; if (timeout != NULL) abs_timeout_init2(&timo, timeout); /* * Care must be exercised when dealing with umtx structure. It * can fault on any access. */ for (;;) { rv = fueword32(&m->m_owner, &owner); if (rv == -1) return (EFAULT); if (mode == _UMUTEX_WAIT) { if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED) return (0); } else { /* * Try the uncontested case. This should be done in userland. */ rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); /* The address was invalid. */ if (rv == -1) return (EFAULT); /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) return (0); /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) return (EFAULT); if (owner == UMUTEX_CONTESTED) return (0); rv = umtxq_check_susp(td); if (rv != 0) return (rv); /* If this failed the lock has changed, restart. */ continue; } } if (mode == _UMUTEX_TRY) return (EBUSY); /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) return (error); if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); /* * Set the contested bit so that a release in user space * knows to use the system call for unlock. If this fails * either some one else has acquired the lock or it has been * released. */ rv = casueword32(&m->m_owner, owner, &old, owner | UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (EFAULT); } /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread * unlocking the umtx. */ umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); if (old == owner) error = umtxq_sleep(uq, "umtxn", timeout == NULL ? NULL : &timo); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); if (error == 0) error = umtxq_check_susp(td); } return (0); } /* * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. */ static int do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) { struct umtx_key key; uint32_t owner, old, id; int error; int count; id = td->td_tid; /* * Make sure we own this mtx. */ error = fueword32(&m->m_owner, &owner); if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); if ((owner & UMUTEX_CONTESTED) == 0) { error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); if (error == -1) return (EFAULT); if (old == owner) return (0); owner = old; } /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); /* * When unlocking the umtx, it must be marked as unowned if * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ error = casueword32(&m->m_owner, owner, &old, count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); umtxq_lock(&key); umtxq_signal(&key,1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); if (error == -1) return (EFAULT); if (old != owner) return (EINVAL); return (0); } /* * Check if the mutex is available and wake up a waiter, * only for simple mutex. */ static int do_wake_umutex(struct thread *td, struct umutex *m) { struct umtx_key key; uint32_t owner; uint32_t flags; int error; int count; error = fueword32(&m->m_owner, &owner); if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != 0) return (0); error = fueword32(&m->m_flags, &flags); if (error == -1) return (EFAULT); /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); if (count <= 1) { error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, UMUTEX_UNOWNED); if (error == -1) error = EFAULT; } umtxq_lock(&key); if (error == 0 && count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); return (error); } /* * Check if the mutex has waiters and tries to fix contention bit. */ static int do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) { struct umtx_key key; uint32_t owner, old; int type; int error; int count; switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: type = TYPE_NORMAL_UMUTEX; break; case UMUTEX_PRIO_INHERIT: type = TYPE_PI_UMUTEX; break; case UMUTEX_PRIO_PROTECT: type = TYPE_PP_UMUTEX; break; default: return (EINVAL); } if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0) return (error); owner = 0; umtxq_lock(&key); umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); /* * Only repair contention bit if there is a waiter, this means the mutex * is still being referenced by userland code, otherwise don't update * any memory. */ if (count > 1) { error = fueword32(&m->m_owner, &owner); if (error == -1) error = EFAULT; while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) { error = casueword32(&m->m_owner, owner, &old, owner | UMUTEX_CONTESTED); if (error == -1) { error = EFAULT; break; } if (old == owner) break; owner = old; error = umtxq_check_susp(td); if (error != 0) break; } } else if (count == 1) { error = fueword32(&m->m_owner, &owner); if (error == -1) error = EFAULT; while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 && (owner & UMUTEX_CONTESTED) == 0) { error = casueword32(&m->m_owner, owner, &old, owner | UMUTEX_CONTESTED); if (error == -1) { error = EFAULT; break; } if (old == owner) break; owner = old; error = umtxq_check_susp(td); if (error != 0) break; } } umtxq_lock(&key); if (error == EFAULT) { umtxq_signal(&key, INT_MAX); } else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); return (error); } static inline struct umtx_pi * umtx_pi_alloc(int flags) { struct umtx_pi *pi; pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags); TAILQ_INIT(&pi->pi_blocked); atomic_add_int(&umtx_pi_allocated, 1); return (pi); } static inline void umtx_pi_free(struct umtx_pi *pi) { uma_zfree(umtx_pi_zone, pi); atomic_add_int(&umtx_pi_allocated, -1); } /* * Adjust the thread's position on a pi_state after its priority has been * changed. */ static int umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) { struct umtx_q *uq, *uq1, *uq2; struct thread *td1; mtx_assert(&umtx_lock, MA_OWNED); if (pi == NULL) return (0); uq = td->td_umtxq; /* * Check if the thread needs to be moved on the blocked chain. * It needs to be moved if either its priority is lower than * the previous thread or higher than the next thread. */ uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq); uq2 = TAILQ_NEXT(uq, uq_lockq); if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { /* * Remove thread from blocked chain and determine where * it should be moved to. */ TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { td1 = uq1->uq_thread; MPASS(td1->td_proc->p_magic == P_MAGIC); if (UPRI(td1) > UPRI(td)) break; } if (uq1 == NULL) TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); else TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); } return (1); } +static struct umtx_pi * +umtx_pi_next(struct umtx_pi *pi) +{ + struct umtx_q *uq_owner; + + if (pi->pi_owner == NULL) + return (NULL); + uq_owner = pi->pi_owner->td_umtxq; + if (uq_owner == NULL) + return (NULL); + return (uq_owner->uq_pi_blocked); +} + +/* + * Floyd's Cycle-Finding Algorithm. + */ +static bool +umtx_pi_check_loop(struct umtx_pi *pi) +{ + struct umtx_pi *pi1; /* fast iterator */ + + mtx_assert(&umtx_lock, MA_OWNED); + if (pi == NULL) + return (false); + pi1 = pi; + for (;;) { + pi = umtx_pi_next(pi); + if (pi == NULL) + break; + pi1 = umtx_pi_next(pi1); + if (pi1 == NULL) + break; + pi1 = umtx_pi_next(pi1); + if (pi1 == NULL) + break; + if (pi == pi1) + return (true); + } + return (false); +} + /* * Propagate priority when a thread is blocked on POSIX * PI mutex. */ static void umtx_propagate_priority(struct thread *td) { struct umtx_q *uq; struct umtx_pi *pi; int pri; mtx_assert(&umtx_lock, MA_OWNED); pri = UPRI(td); uq = td->td_umtxq; pi = uq->uq_pi_blocked; if (pi == NULL) return; + if (umtx_pi_check_loop(pi)) + return; for (;;) { td = pi->pi_owner; if (td == NULL || td == curthread) return; MPASS(td->td_proc != NULL); MPASS(td->td_proc->p_magic == P_MAGIC); thread_lock(td); if (td->td_lend_user_pri > pri) sched_lend_user_prio(td, pri); else { thread_unlock(td); break; } thread_unlock(td); /* * Pick up the lock that td is blocked on. */ uq = td->td_umtxq; pi = uq->uq_pi_blocked; if (pi == NULL) break; /* Resort td on the list if needed. */ umtx_pi_adjust_thread(pi, td); } } /* * Unpropagate priority for a PI mutex when a thread blocked on * it is interrupted by signal or resumed by others. */ static void umtx_repropagate_priority(struct umtx_pi *pi) { struct umtx_q *uq, *uq_owner; struct umtx_pi *pi2; int pri; mtx_assert(&umtx_lock, MA_OWNED); + if (umtx_pi_check_loop(pi)) + return; while (pi != NULL && pi->pi_owner != NULL) { pri = PRI_MAX; uq_owner = pi->pi_owner->td_umtxq; TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { uq = TAILQ_FIRST(&pi2->pi_blocked); if (uq != NULL) { if (pri > UPRI(uq->uq_thread)) pri = UPRI(uq->uq_thread); } } if (pri > uq_owner->uq_inherited_pri) pri = uq_owner->uq_inherited_pri; thread_lock(pi->pi_owner); sched_lend_user_prio(pi->pi_owner, pri); thread_unlock(pi->pi_owner); if ((pi = uq_owner->uq_pi_blocked) != NULL) umtx_pi_adjust_thread(pi, uq_owner->uq_thread); } } /* * Insert a PI mutex into owned list. */ static void umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) { struct umtx_q *uq_owner; uq_owner = owner->td_umtxq; mtx_assert(&umtx_lock, MA_OWNED); if (pi->pi_owner != NULL) panic("pi_ower != NULL"); pi->pi_owner = owner; TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); } /* * Claim ownership of a PI mutex. */ static int umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) { struct umtx_q *uq, *uq_owner; uq_owner = owner->td_umtxq; mtx_lock_spin(&umtx_lock); if (pi->pi_owner == owner) { mtx_unlock_spin(&umtx_lock); return (0); } if (pi->pi_owner != NULL) { /* * userland may have already messed the mutex, sigh. */ mtx_unlock_spin(&umtx_lock); return (EPERM); } umtx_pi_setowner(pi, owner); uq = TAILQ_FIRST(&pi->pi_blocked); if (uq != NULL) { int pri; pri = UPRI(uq->uq_thread); thread_lock(owner); if (pri < UPRI(owner)) sched_lend_user_prio(owner, pri); thread_unlock(owner); } mtx_unlock_spin(&umtx_lock); return (0); } /* * Adjust a thread's order position in its blocked PI mutex, * this may result new priority propagating process. */ void umtx_pi_adjust(struct thread *td, u_char oldpri) { struct umtx_q *uq; struct umtx_pi *pi; uq = td->td_umtxq; mtx_lock_spin(&umtx_lock); /* * Pick up the lock that td is blocked on. */ pi = uq->uq_pi_blocked; if (pi != NULL) { umtx_pi_adjust_thread(pi, td); umtx_repropagate_priority(pi); } mtx_unlock_spin(&umtx_lock); } /* * Sleep on a PI mutex. */ static int umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner, const char *wmesg, struct abs_timeout *timo) { struct umtxq_chain *uc; struct thread *td, *td1; struct umtx_q *uq1; int pri; int error = 0; td = uq->uq_thread; KASSERT(td == curthread, ("inconsistent uq_thread")); uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); umtxq_insert(uq); mtx_lock_spin(&umtx_lock); if (pi->pi_owner == NULL) { mtx_unlock_spin(&umtx_lock); /* XXX Only look up thread in current process. */ td1 = tdfind(owner, curproc->p_pid); mtx_lock_spin(&umtx_lock); if (td1 != NULL) { if (pi->pi_owner == NULL) umtx_pi_setowner(pi, td1); PROC_UNLOCK(td1->td_proc); } } TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { pri = UPRI(uq1->uq_thread); if (pri > UPRI(td)) break; } if (uq1 != NULL) TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); else TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); uq->uq_pi_blocked = pi; thread_lock(td); td->td_flags |= TDF_UPIBLOCKED; thread_unlock(td); umtx_propagate_priority(td); mtx_unlock_spin(&umtx_lock); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, wmesg, timo); umtxq_remove(uq); mtx_lock_spin(&umtx_lock); uq->uq_pi_blocked = NULL; thread_lock(td); td->td_flags &= ~TDF_UPIBLOCKED; thread_unlock(td); TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); umtx_repropagate_priority(pi); mtx_unlock_spin(&umtx_lock); umtxq_unlock(&uq->uq_key); return (error); } /* * Add reference count for a PI mutex. */ static void umtx_pi_ref(struct umtx_pi *pi) { struct umtxq_chain *uc; uc = umtxq_getchain(&pi->pi_key); UMTXQ_LOCKED_ASSERT(uc); pi->pi_refcount++; } /* * Decrease reference count for a PI mutex, if the counter * is decreased to zero, its memory space is freed. */ static void umtx_pi_unref(struct umtx_pi *pi) { struct umtxq_chain *uc; uc = umtxq_getchain(&pi->pi_key); UMTXQ_LOCKED_ASSERT(uc); KASSERT(pi->pi_refcount > 0, ("invalid reference count")); if (--pi->pi_refcount == 0) { mtx_lock_spin(&umtx_lock); if (pi->pi_owner != NULL) { TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link); pi->pi_owner = NULL; } KASSERT(TAILQ_EMPTY(&pi->pi_blocked), ("blocked queue not empty")); mtx_unlock_spin(&umtx_lock); TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); umtx_pi_free(pi); } } /* * Find a PI mutex in hash table. */ static struct umtx_pi * umtx_pi_lookup(struct umtx_key *key) { struct umtxq_chain *uc; struct umtx_pi *pi; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { if (umtx_key_match(&pi->pi_key, key)) { return (pi); } } return (NULL); } /* * Insert a PI mutex into hash table. */ static inline void umtx_pi_insert(struct umtx_pi *pi) { struct umtxq_chain *uc; uc = umtxq_getchain(&pi->pi_key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); } /* * Lock a PI mutex. */ static int do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, struct _umtx_time *timeout, int try) { struct abs_timeout timo; struct umtx_q *uq; struct umtx_pi *pi, *new_pi; uint32_t id, owner, old; int error, rv; id = td->td_tid; uq = td->td_umtxq; if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); if (timeout != NULL) abs_timeout_init2(&timo, timeout); umtxq_lock(&uq->uq_key); pi = umtx_pi_lookup(&uq->uq_key); if (pi == NULL) { new_pi = umtx_pi_alloc(M_NOWAIT); if (new_pi == NULL) { umtxq_unlock(&uq->uq_key); new_pi = umtx_pi_alloc(M_WAITOK); umtxq_lock(&uq->uq_key); pi = umtx_pi_lookup(&uq->uq_key); if (pi != NULL) { umtx_pi_free(new_pi); new_pi = NULL; } } if (new_pi != NULL) { new_pi->pi_key = uq->uq_key; umtx_pi_insert(new_pi); pi = new_pi; } } umtx_pi_ref(pi); umtxq_unlock(&uq->uq_key); /* * Care must be exercised when dealing with umtx structure. It * can fault on any access. */ for (;;) { /* * Try the uncontested case. This should be done in userland. */ rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); /* The address was invalid. */ if (rv == -1) { error = EFAULT; break; } /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) { error = 0; break; } /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) { error = EFAULT; break; } if (owner == UMUTEX_CONTESTED) { umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); error = umtx_pi_claim(pi, td); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); break; } error = umtxq_check_susp(td); if (error != 0) break; /* If this failed the lock has changed, restart. */ continue; } + if ((owner & ~UMUTEX_CONTESTED) == id) { + error = EDEADLK; + break; + } + if (try != 0) { error = EBUSY; break; } /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) break; umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); /* * Set the contested bit so that a release in user space * knows to use the system call for unlock. If this fails * either some one else has acquired the lock or it has been * released. */ rv = casueword32(&m->m_owner, owner, &old, owner | UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) { umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } umtxq_lock(&uq->uq_key); /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread * unlocking the umtx. */ if (old == owner) { error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, "umtxpi", timeout == NULL ? NULL : &timo); if (error != 0) continue; } else { umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); } error = umtxq_check_susp(td); if (error != 0) break; } umtxq_lock(&uq->uq_key); umtx_pi_unref(pi); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } /* * Unlock a PI mutex. */ static int do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) { struct umtx_key key; struct umtx_q *uq_first, *uq_first2, *uq_me; struct umtx_pi *pi, *pi2; uint32_t owner, old, id; int error; int count; int pri; id = td->td_tid; /* * Make sure we own this mtx. */ error = fueword32(&m->m_owner, &owner); if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); if (error == -1) return (EFAULT); if (old == owner) return (0); owner = old; } /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); count = umtxq_count_pi(&key, &uq_first); if (uq_first != NULL) { mtx_lock_spin(&umtx_lock); pi = uq_first->uq_pi_blocked; KASSERT(pi != NULL, ("pi == NULL?")); if (pi->pi_owner != curthread) { mtx_unlock_spin(&umtx_lock); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); /* userland messed the mutex */ return (EPERM); } uq_me = curthread->td_umtxq; pi->pi_owner = NULL; TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link); /* get highest priority thread which is still sleeping. */ uq_first = TAILQ_FIRST(&pi->pi_blocked); while (uq_first != NULL && (uq_first->uq_flags & UQF_UMTXQ) == 0) { uq_first = TAILQ_NEXT(uq_first, uq_lockq); } pri = PRI_MAX; TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); if (uq_first2 != NULL) { if (pri > UPRI(uq_first2->uq_thread)) pri = UPRI(uq_first2->uq_thread); } } thread_lock(curthread); sched_lend_user_prio(curthread, pri); thread_unlock(curthread); mtx_unlock_spin(&umtx_lock); if (uq_first) umtxq_signal_thread(uq_first); } umtxq_unlock(&key); /* * When unlocking the umtx, it must be marked as unowned if * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ error = casueword32(&m->m_owner, owner, &old, count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); umtxq_unbusy_unlocked(&key); umtx_key_release(&key); if (error == -1) return (EFAULT); if (old != owner) return (EINVAL); return (0); } /* * Lock a PP mutex. */ static int do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, struct _umtx_time *timeout, int try) { struct abs_timeout timo; struct umtx_q *uq, *uq2; struct umtx_pi *pi; uint32_t ceiling; uint32_t owner, id; int error, pri, old_inherited_pri, su, rv; id = td->td_tid; uq = td->td_umtxq; if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); if (timeout != NULL) abs_timeout_init2(&timo, timeout); su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); for (;;) { old_inherited_pri = uq->uq_inherited_pri; umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); rv = fueword32(&m->m_ceilings[0], &ceiling); if (rv == -1) { error = EFAULT; goto out; } ceiling = RTP_PRIO_MAX - ceiling; if (ceiling > RTP_PRIO_MAX) { error = EINVAL; goto out; } mtx_lock_spin(&umtx_lock); if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { mtx_unlock_spin(&umtx_lock); error = EINVAL; goto out; } if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; thread_lock(td); if (uq->uq_inherited_pri < UPRI(td)) sched_lend_user_prio(td, uq->uq_inherited_pri); thread_unlock(td); } mtx_unlock_spin(&umtx_lock); rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) { error = EFAULT; break; } if (owner == UMUTEX_CONTESTED) { error = 0; break; } if (try != 0) { error = EBUSY; break; } /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) break; umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "umtxpp", timeout == NULL ? NULL : &timo); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); mtx_lock_spin(&umtx_lock); uq->uq_inherited_pri = old_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { uq2 = TAILQ_FIRST(&pi->pi_blocked); if (uq2 != NULL) { if (pri > UPRI(uq2->uq_thread)) pri = UPRI(uq2->uq_thread); } } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; thread_lock(td); sched_lend_user_prio(td, pri); thread_unlock(td); mtx_unlock_spin(&umtx_lock); } if (error != 0) { mtx_lock_spin(&umtx_lock); uq->uq_inherited_pri = old_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { uq2 = TAILQ_FIRST(&pi->pi_blocked); if (uq2 != NULL) { if (pri > UPRI(uq2->uq_thread)) pri = UPRI(uq2->uq_thread); } } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; thread_lock(td); sched_lend_user_prio(td, pri); thread_unlock(td); mtx_unlock_spin(&umtx_lock); } out: umtxq_unbusy_unlocked(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } /* * Unlock a PP mutex. */ static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) { struct umtx_key key; struct umtx_q *uq, *uq2; struct umtx_pi *pi; uint32_t owner, id; uint32_t rceiling; int error, pri, new_inherited_pri, su; id = td->td_tid; uq = td->td_umtxq; su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); /* * Make sure we own this mtx. */ error = fueword32(&m->m_owner, &owner); if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); if (error != 0) return (error); if (rceiling == -1) new_inherited_pri = PRI_MAX; else { rceiling = RTP_PRIO_MAX - rceiling; if (rceiling > RTP_PRIO_MAX) return (EINVAL); new_inherited_pri = PRI_MIN_REALTIME + rceiling; } if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); umtxq_unlock(&key); /* * For priority protected mutex, always set unlocked state * to UMUTEX_CONTESTED, so that userland always enters kernel * to lock the mutex, it is necessary because thread priority * has to be adjusted for such mutex. */ error = suword32(&m->m_owner, UMUTEX_CONTESTED); umtxq_lock(&key); if (error == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); if (error == -1) error = EFAULT; else { mtx_lock_spin(&umtx_lock); if (su != 0) uq->uq_inherited_pri = new_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { uq2 = TAILQ_FIRST(&pi->pi_blocked); if (uq2 != NULL) { if (pri > UPRI(uq2->uq_thread)) pri = UPRI(uq2->uq_thread); } } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; thread_lock(td); sched_lend_user_prio(td, pri); thread_unlock(td); mtx_unlock_spin(&umtx_lock); } umtx_key_release(&key); return (error); } static int do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, uint32_t *old_ceiling) { struct umtx_q *uq; uint32_t save_ceiling; uint32_t owner, id; uint32_t flags; int error, rv; error = fueword32(&m->m_flags, &flags); if (error == -1) return (EFAULT); if ((flags & UMUTEX_PRIO_PROTECT) == 0) return (EINVAL); if (ceiling > RTP_PRIO_MAX) return (EINVAL); id = td->td_tid; uq = td->td_umtxq; if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); for (;;) { umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); rv = fueword32(&m->m_ceilings[0], &save_ceiling); if (rv == -1) { error = EFAULT; break; } rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); if (rv == -1) { error = EFAULT; break; } if (owner == UMUTEX_CONTESTED) { suword32(&m->m_ceilings[0], ceiling); suword32(&m->m_owner, UMUTEX_CONTESTED); error = 0; break; } if ((owner & ~UMUTEX_CONTESTED) == id) { suword32(&m->m_ceilings[0], ceiling); error = 0; break; } /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) break; /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread * unlocking the umtx. */ umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "umtxpp", NULL); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); } umtxq_lock(&uq->uq_key); if (error == 0) umtxq_signal(&uq->uq_key, INT_MAX); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); if (error == 0 && old_ceiling != NULL) suword32(old_ceiling, save_ceiling); return (error); } /* * Lock a userland POSIX mutex. */ static int do_lock_umutex(struct thread *td, struct umutex *m, struct _umtx_time *timeout, int mode) { uint32_t flags; int error; error = fueword32(&m->m_flags, &flags); if (error == -1) return (EFAULT); switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: error = do_lock_normal(td, m, flags, timeout, mode); break; case UMUTEX_PRIO_INHERIT: error = do_lock_pi(td, m, flags, timeout, mode); break; case UMUTEX_PRIO_PROTECT: error = do_lock_pp(td, m, flags, timeout, mode); break; default: return (EINVAL); } if (timeout == NULL) { if (error == EINTR && mode != _UMUTEX_WAIT) error = ERESTART; } else { /* Timed-locking is not restarted. */ if (error == ERESTART) error = EINTR; } return (error); } /* * Unlock a userland POSIX mutex. */ static int do_unlock_umutex(struct thread *td, struct umutex *m) { uint32_t flags; int error; error = fueword32(&m->m_flags, &flags); if (error == -1) return (EFAULT); switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: return (do_unlock_normal(td, m, flags)); case UMUTEX_PRIO_INHERIT: return (do_unlock_pi(td, m, flags)); case UMUTEX_PRIO_PROTECT: return (do_unlock_pp(td, m, flags)); } return (EINVAL); } static int do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, struct timespec *timeout, u_long wflags) { struct abs_timeout timo; struct umtx_q *uq; uint32_t flags, clockid, hasw; int error; uq = td->td_umtxq; error = fueword32(&cv->c_flags, &flags); if (error == -1) return (EFAULT); error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); if ((wflags & CVWAIT_CLOCKID) != 0) { error = fueword32(&cv->c_clockid, &clockid); if (error == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } if (clockid < CLOCK_REALTIME || clockid >= CLOCK_THREAD_CPUTIME_ID) { /* hmm, only HW clock id will work. */ umtx_key_release(&uq->uq_key); return (EINVAL); } } else { clockid = CLOCK_REALTIME; } umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); /* * Set c_has_waiters to 1 before releasing user mutex, also * don't modify cache line when unnecessary. */ error = fueword32(&cv->c_has_waiters, &hasw); if (error == 0 && hasw == 0) suword32(&cv->c_has_waiters, 1); umtxq_unbusy_unlocked(&uq->uq_key); error = do_unlock_umutex(td, m); if (timeout != NULL) abs_timeout_init(&timo, clockid, ((wflags & CVWAIT_ABSTIME) != 0), timeout); umtxq_lock(&uq->uq_key); if (error == 0) { error = umtxq_sleep(uq, "ucond", timeout == NULL ? NULL : &timo); } if ((uq->uq_flags & UQF_UMTXQ) == 0) error = 0; else { /* * This must be timeout,interrupted by signal or * surprious wakeup, clear c_has_waiter flag when * necessary. */ umtxq_busy(&uq->uq_key); if ((uq->uq_flags & UQF_UMTXQ) != 0) { int oldlen = uq->uq_cur_queue->length; umtxq_remove(uq); if (oldlen == 1) { umtxq_unlock(&uq->uq_key); suword32(&cv->c_has_waiters, 0); umtxq_lock(&uq->uq_key); } } umtxq_unbusy(&uq->uq_key); if (error == ERESTART) error = EINTR; } umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } /* * Signal a userland condition variable. */ static int do_cv_signal(struct thread *td, struct ucond *cv) { struct umtx_key key; int error, cnt, nwake; uint32_t flags; error = fueword32(&cv->c_flags, &flags); if (error == -1) return (EFAULT); if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); cnt = umtxq_count(&key); nwake = umtxq_signal(&key, 1); if (cnt <= nwake) { umtxq_unlock(&key); error = suword32(&cv->c_has_waiters, 0); if (error == -1) error = EFAULT; umtxq_lock(&key); } umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); return (error); } static int do_cv_broadcast(struct thread *td, struct ucond *cv) { struct umtx_key key; int error; uint32_t flags; error = fueword32(&cv->c_flags, &flags); if (error == -1) return (EFAULT); if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); umtxq_signal(&key, INT_MAX); umtxq_unlock(&key); error = suword32(&cv->c_has_waiters, 0); if (error == -1) error = EFAULT; umtxq_unbusy_unlocked(&key); umtx_key_release(&key); return (error); } static int do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx_time *timeout) { struct abs_timeout timo; struct umtx_q *uq; uint32_t flags, wrflags; int32_t state, oldstate; int32_t blocked_readers; int error, rv; uq = td->td_umtxq; error = fueword32(&rwlock->rw_flags, &flags); if (error == -1) return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); if (timeout != NULL) abs_timeout_init2(&timo, timeout); wrflags = URWLOCK_WRITE_OWNER; if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER)) wrflags |= URWLOCK_WRITE_WAITERS; for (;;) { rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } /* try to lock it */ while (!(state & wrflags)) { if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) { umtx_key_release(&uq->uq_key); return (EAGAIN); } rv = casueword32(&rwlock->rw_state, state, &oldstate, state + 1); if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } if (oldstate == state) { umtx_key_release(&uq->uq_key); return (0); } error = umtxq_check_susp(td); if (error != 0) break; state = oldstate; } if (error) break; /* grab monitor lock */ umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); /* * re-read the state, in case it changed between the try-lock above * and the check below */ rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) error = EFAULT; /* set read contention bit */ while (error == 0 && (state & wrflags) && !(state & URWLOCK_READ_WAITERS)) { rv = casueword32(&rwlock->rw_state, state, &oldstate, state | URWLOCK_READ_WAITERS); if (rv == -1) { error = EFAULT; break; } if (oldstate == state) goto sleep; state = oldstate; error = umtxq_check_susp(td); if (error != 0) break; } if (error != 0) { umtxq_unbusy_unlocked(&uq->uq_key); break; } /* state is changed while setting flags, restart */ if (!(state & wrflags)) { umtxq_unbusy_unlocked(&uq->uq_key); error = umtxq_check_susp(td); if (error != 0) break; continue; } sleep: /* contention bit is set, before sleeping, increase read waiter count */ rv = fueword32(&rwlock->rw_blocked_readers, &blocked_readers); if (rv == -1) { umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } suword32(&rwlock->rw_blocked_readers, blocked_readers+1); while (state & wrflags) { umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "urdlck", timeout == NULL ? NULL : &timo); umtxq_busy(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); if (error) break; rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) { error = EFAULT; break; } } /* decrease read waiter count, and may clear read contention bit */ rv = fueword32(&rwlock->rw_blocked_readers, &blocked_readers); if (rv == -1) { umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } suword32(&rwlock->rw_blocked_readers, blocked_readers-1); if (blocked_readers == 1) { rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) error = EFAULT; while (error == 0) { rv = casueword32(&rwlock->rw_state, state, &oldstate, state & ~URWLOCK_READ_WAITERS); if (rv == -1) { error = EFAULT; break; } if (oldstate == state) break; state = oldstate; error = umtxq_check_susp(td); } } umtxq_unbusy_unlocked(&uq->uq_key); if (error != 0) break; } umtx_key_release(&uq->uq_key); if (error == ERESTART) error = EINTR; return (error); } static int do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout) { struct abs_timeout timo; struct umtx_q *uq; uint32_t flags; int32_t state, oldstate; int32_t blocked_writers; int32_t blocked_readers; int error, rv; uq = td->td_umtxq; error = fueword32(&rwlock->rw_flags, &flags); if (error == -1) return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); if (timeout != NULL) abs_timeout_init2(&timo, timeout); blocked_readers = 0; for (;;) { rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { rv = casueword32(&rwlock->rw_state, state, &oldstate, state | URWLOCK_WRITE_OWNER); if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } if (oldstate == state) { umtx_key_release(&uq->uq_key); return (0); } state = oldstate; error = umtxq_check_susp(td); if (error != 0) break; } if (error) { if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) && blocked_readers != 0) { umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); } break; } /* grab monitor lock */ umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); /* * re-read the state, in case it changed between the try-lock above * and the check below */ rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) error = EFAULT; while (error == 0 && ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) && (state & URWLOCK_WRITE_WAITERS) == 0) { rv = casueword32(&rwlock->rw_state, state, &oldstate, state | URWLOCK_WRITE_WAITERS); if (rv == -1) { error = EFAULT; break; } if (oldstate == state) goto sleep; state = oldstate; error = umtxq_check_susp(td); if (error != 0) break; } if (error != 0) { umtxq_unbusy_unlocked(&uq->uq_key); break; } if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { umtxq_unbusy_unlocked(&uq->uq_key); error = umtxq_check_susp(td); if (error != 0) break; continue; } sleep: rv = fueword32(&rwlock->rw_blocked_writers, &blocked_writers); if (rv == -1) { umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } suword32(&rwlock->rw_blocked_writers, blocked_writers+1); while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) { umtxq_lock(&uq->uq_key); umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "uwrlck", timeout == NULL ? NULL : &timo); umtxq_busy(&uq->uq_key); umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE); umtxq_unlock(&uq->uq_key); if (error) break; rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) { error = EFAULT; break; } } rv = fueword32(&rwlock->rw_blocked_writers, &blocked_writers); if (rv == -1) { umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } suword32(&rwlock->rw_blocked_writers, blocked_writers-1); if (blocked_writers == 1) { rv = fueword32(&rwlock->rw_state, &state); if (rv == -1) { umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } for (;;) { rv = casueword32(&rwlock->rw_state, state, &oldstate, state & ~URWLOCK_WRITE_WAITERS); if (rv == -1) { error = EFAULT; break; } if (oldstate == state) break; state = oldstate; error = umtxq_check_susp(td); /* * We are leaving the URWLOCK_WRITE_WAITERS * behind, but this should not harm the * correctness. */ if (error != 0) break; } rv = fueword32(&rwlock->rw_blocked_readers, &blocked_readers); if (rv == -1) { umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } } else blocked_readers = 0; umtxq_unbusy_unlocked(&uq->uq_key); } umtx_key_release(&uq->uq_key); if (error == ERESTART) error = EINTR; return (error); } static int do_rw_unlock(struct thread *td, struct urwlock *rwlock) { struct umtx_q *uq; uint32_t flags; int32_t state, oldstate; int error, rv, q, count; uq = td->td_umtxq; error = fueword32(&rwlock->rw_flags, &flags); if (error == -1) return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); error = fueword32(&rwlock->rw_state, &state); if (error == -1) { error = EFAULT; goto out; } if (state & URWLOCK_WRITE_OWNER) { for (;;) { rv = casueword32(&rwlock->rw_state, state, &oldstate, state & ~URWLOCK_WRITE_OWNER); if (rv == -1) { error = EFAULT; goto out; } if (oldstate != state) { state = oldstate; if (!(oldstate & URWLOCK_WRITE_OWNER)) { error = EPERM; goto out; } error = umtxq_check_susp(td); if (error != 0) goto out; } else break; } } else if (URWLOCK_READER_COUNT(state) != 0) { for (;;) { rv = casueword32(&rwlock->rw_state, state, &oldstate, state - 1); if (rv == -1) { error = EFAULT; goto out; } if (oldstate != state) { state = oldstate; if (URWLOCK_READER_COUNT(oldstate) == 0) { error = EPERM; goto out; } error = umtxq_check_susp(td); if (error != 0) goto out; } else break; } } else { error = EPERM; goto out; } count = 0; if (!(flags & URWLOCK_PREFER_READER)) { if (state & URWLOCK_WRITE_WAITERS) { count = 1; q = UMTX_EXCLUSIVE_QUEUE; } else if (state & URWLOCK_READ_WAITERS) { count = INT_MAX; q = UMTX_SHARED_QUEUE; } } else { if (state & URWLOCK_READ_WAITERS) { count = INT_MAX; q = UMTX_SHARED_QUEUE; } else if (state & URWLOCK_WRITE_WAITERS) { count = 1; q = UMTX_EXCLUSIVE_QUEUE; } } if (count) { umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_signal_queue(&uq->uq_key, count, q); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); } out: umtx_key_release(&uq->uq_key); return (error); } #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) static int do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) { struct abs_timeout timo; struct umtx_q *uq; uint32_t flags, count, count1; int error, rv; uq = td->td_umtxq; error = fueword32(&sem->_flags, &flags); if (error == -1) return (EFAULT); error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); if (timeout != NULL) abs_timeout_init2(&timo, timeout); umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); rv = casueword32(&sem->_has_waiters, 0, &count1, 1); if (rv == 0) rv = fueword32(&sem->_count, &count); if (rv == -1 || count != 0) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (rv == -1 ? EFAULT : 0); } umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); if ((uq->uq_flags & UQF_UMTXQ) == 0) error = 0; else { umtxq_remove(uq); /* A relative timeout cannot be restarted. */ if (error == ERESTART && timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) error = EINTR; } umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } /* * Signal a userland semaphore. */ static int do_sem_wake(struct thread *td, struct _usem *sem) { struct umtx_key key; int error, cnt; uint32_t flags; error = fueword32(&sem->_flags, &flags); if (error == -1) return (EFAULT); if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); cnt = umtxq_count(&key); if (cnt > 0) { umtxq_signal(&key, 1); /* * Check if count is greater than 0, this means the memory is * still being referenced by user code, so we can safely * update _has_waiters flag. */ if (cnt == 1) { umtxq_unlock(&key); error = suword32(&sem->_has_waiters, 0); umtxq_lock(&key); if (error == -1) error = EFAULT; } } umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); return (error); } #endif static int do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) { struct abs_timeout timo; struct umtx_q *uq; uint32_t count, flags; int error, rv; uq = td->td_umtxq; flags = fuword32(&sem->_flags); error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); if (timeout != NULL) abs_timeout_init2(&timo, timeout); umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); rv = fueword32(&sem->_count, &count); if (rv == -1) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (EFAULT); } for (;;) { if (USEM_COUNT(count) != 0) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (0); } if (count == USEM_HAS_WAITERS) break; rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); if (rv == -1) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (EFAULT); } if (count == 0) break; } umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); if ((uq->uq_flags & UQF_UMTXQ) == 0) error = 0; else { umtxq_remove(uq); /* A relative timeout cannot be restarted. */ if (error == ERESTART && timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) error = EINTR; } umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } /* * Signal a userland semaphore. */ static int do_sem2_wake(struct thread *td, struct _usem2 *sem) { struct umtx_key key; int error, cnt, rv; uint32_t count, flags; rv = fueword32(&sem->_flags, &flags); if (rv == -1) return (EFAULT); if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); cnt = umtxq_count(&key); if (cnt > 0) { umtxq_signal(&key, 1); /* * If this was the last sleeping thread, clear the waiters * flag in _count. */ if (cnt == 1) { umtxq_unlock(&key); rv = fueword32(&sem->_count, &count); while (rv != -1 && count & USEM_HAS_WAITERS) rv = casueword32(&sem->_count, count, &count, count & ~USEM_HAS_WAITERS); if (rv == -1) error = EFAULT; umtxq_lock(&key); } } umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); return (error); } inline int umtx_copyin_timeout(const void *addr, struct timespec *tsp) { int error; error = copyin(addr, tsp, sizeof(struct timespec)); if (error == 0) { if (tsp->tv_sec < 0 || tsp->tv_nsec >= 1000000000 || tsp->tv_nsec < 0) error = EINVAL; } return (error); } static inline int umtx_copyin_umtx_time(const void *addr, size_t size, struct _umtx_time *tp) { int error; if (size <= sizeof(struct timespec)) { tp->_clockid = CLOCK_REALTIME; tp->_flags = 0; error = copyin(addr, &tp->_timeout, sizeof(struct timespec)); } else error = copyin(addr, tp, sizeof(struct _umtx_time)); if (error != 0) return (error); if (tp->_timeout.tv_sec < 0 || tp->_timeout.tv_nsec >= 1000000000 || tp->_timeout.tv_nsec < 0) return (EINVAL); return (0); } static int __umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap) { return (EOPNOTSUPP); } static int __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time timeout, *tm_p; int error; if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_wait(td, uap->obj, uap->val, tm_p, 0, 0); } static int __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time timeout, *tm_p; int error; if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_wait(td, uap->obj, uap->val, tm_p, 1, 0); } static int __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_wait(td, uap->obj, uap->val, tm_p, 1, 1); } static int __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) { return (kern_umtx_wake(td, uap->obj, uap->val, 0)); } #define BATCH_SIZE 128 static int __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap) { int count = uap->val; void *uaddrs[BATCH_SIZE]; char **upp = (char **)uap->obj; int tocopy; int error = 0; int i, pos = 0; while (count > 0) { tocopy = count; if (tocopy > BATCH_SIZE) tocopy = BATCH_SIZE; error = copyin(upp+pos, uaddrs, tocopy * sizeof(char *)); if (error != 0) break; for (i = 0; i < tocopy; ++i) kern_umtx_wake(td, uaddrs[i], INT_MAX, 1); count -= tocopy; pos += tocopy; } return (error); } static int __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) { return (kern_umtx_wake(td, uap->obj, uap->val, 1)); } static int __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_lock_umutex(td, uap->obj, tm_p, 0); } static int __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) { return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY); } static int __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT); } static int __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) { return do_wake_umutex(td, uap->obj); } static int __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) { return do_unlock_umutex(td, uap->obj); } static int __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) { return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1); } static int __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) { struct timespec *ts, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) ts = NULL; else { error = umtx_copyin_timeout(uap->uaddr2, &timeout); if (error != 0) return (error); ts = &timeout; } return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); } static int __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) { return do_cv_signal(td, uap->obj); } static int __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) { return do_cv_broadcast(td, uap->obj); } static int __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) { error = do_rw_rdlock(td, uap->obj, uap->val, 0); } else { error = umtx_copyin_umtx_time(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); } return (error); } static int __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) { error = do_rw_wrlock(td, uap->obj, 0); } else { error = umtx_copyin_umtx_time(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); error = do_rw_wrlock(td, uap->obj, &timeout); } return (error); } static int __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) { return do_rw_unlock(td, uap->obj); } #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) static int __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return (do_sem_wait(td, uap->obj, tm_p)); } static int __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap) { return do_sem_wake(td, uap->obj); } #endif static int __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap) { return do_wake2_umutex(td, uap->obj, uap->val); } static int __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return (do_sem2_wait(td, uap->obj, tm_p)); } static int __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap) { return do_sem2_wake(td, uap->obj); } typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); static _umtx_op_func op_table[] = { __umtx_op_unimpl, /* UMTX_OP_RESERVED0 */ __umtx_op_unimpl, /* UMTX_OP_RESERVED1 */ __umtx_op_wait, /* UMTX_OP_WAIT */ __umtx_op_wake, /* UMTX_OP_WAKE */ __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */ __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */ __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/ __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */ __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */ __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */ __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */ __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */ __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ __umtx_op_wait_umutex, /* UMTX_OP_MUTEX_WAIT */ __umtx_op_wake_umutex, /* UMTX_OP_MUTEX_WAKE */ #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */ __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */ #else __umtx_op_unimpl, /* UMTX_OP_SEM_WAIT */ __umtx_op_unimpl, /* UMTX_OP_SEM_WAKE */ #endif __umtx_op_nwake_private, /* UMTX_OP_NWAKE_PRIVATE */ __umtx_op_wake2_umutex, /* UMTX_OP_MUTEX_WAKE2 */ __umtx_op_sem2_wait, /* UMTX_OP_SEM2_WAIT */ __umtx_op_sem2_wake, /* UMTX_OP_SEM2_WAKE */ }; int sys__umtx_op(struct thread *td, struct _umtx_op_args *uap) { if ((unsigned)uap->op < UMTX_OP_MAX) return (*op_table[uap->op])(td, uap); return (EINVAL); } #ifdef COMPAT_FREEBSD32 struct timespec32 { int32_t tv_sec; int32_t tv_nsec; }; struct umtx_time32 { struct timespec32 timeout; uint32_t flags; uint32_t clockid; }; static inline int umtx_copyin_timeout32(void *addr, struct timespec *tsp) { struct timespec32 ts32; int error; error = copyin(addr, &ts32, sizeof(struct timespec32)); if (error == 0) { if (ts32.tv_sec < 0 || ts32.tv_nsec >= 1000000000 || ts32.tv_nsec < 0) error = EINVAL; else { tsp->tv_sec = ts32.tv_sec; tsp->tv_nsec = ts32.tv_nsec; } } return (error); } static inline int umtx_copyin_umtx_time32(const void *addr, size_t size, struct _umtx_time *tp) { struct umtx_time32 t32; int error; t32.clockid = CLOCK_REALTIME; t32.flags = 0; if (size <= sizeof(struct timespec32)) error = copyin(addr, &t32.timeout, sizeof(struct timespec32)); else error = copyin(addr, &t32, sizeof(struct umtx_time32)); if (error != 0) return (error); if (t32.timeout.tv_sec < 0 || t32.timeout.tv_nsec >= 1000000000 || t32.timeout.tv_nsec < 0) return (EINVAL); tp->_timeout.tv_sec = t32.timeout.tv_sec; tp->_timeout.tv_nsec = t32.timeout.tv_nsec; tp->_flags = t32.flags; tp->_clockid = t32.clockid; return (0); } static int __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time32(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_wait(td, uap->obj, uap->val, tm_p, 1, 0); } static int __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_lock_umutex(td, uap->obj, tm_p, 0); } static int __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time32(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT); } static int __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) { struct timespec *ts, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) ts = NULL; else { error = umtx_copyin_timeout32(uap->uaddr2, &timeout); if (error != 0) return (error); ts = &timeout; } return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); } static int __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) { error = do_rw_rdlock(td, uap->obj, uap->val, 0); } else { error = umtx_copyin_umtx_time32(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); } return (error); } static int __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) { error = do_rw_wrlock(td, uap->obj, 0); } else { error = umtx_copyin_umtx_time32(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); error = do_rw_wrlock(td, uap->obj, &timeout); } return (error); } static int __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time32( uap->uaddr2, (size_t)uap->uaddr1,&timeout); if (error != 0) return (error); tm_p = &timeout; } return do_wait(td, uap->obj, uap->val, tm_p, 1, 1); } #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) static int __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time32(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return (do_sem_wait(td, uap->obj, tm_p)); } #endif static int __umtx_op_sem2_wait_compat32(struct thread *td, struct _umtx_op_args *uap) { struct _umtx_time *tm_p, timeout; int error; /* Allow a null timespec (wait forever). */ if (uap->uaddr2 == NULL) tm_p = NULL; else { error = umtx_copyin_umtx_time32(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } return (do_sem2_wait(td, uap->obj, tm_p)); } static int __umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap) { int count = uap->val; uint32_t uaddrs[BATCH_SIZE]; uint32_t **upp = (uint32_t **)uap->obj; int tocopy; int error = 0; int i, pos = 0; while (count > 0) { tocopy = count; if (tocopy > BATCH_SIZE) tocopy = BATCH_SIZE; error = copyin(upp+pos, uaddrs, tocopy * sizeof(uint32_t)); if (error != 0) break; for (i = 0; i < tocopy; ++i) kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i], INT_MAX, 1); count -= tocopy; pos += tocopy; } return (error); } static _umtx_op_func op_table_compat32[] = { __umtx_op_unimpl, /* UMTX_OP_RESERVED0 */ __umtx_op_unimpl, /* UMTX_OP_RESERVED1 */ __umtx_op_wait_compat32, /* UMTX_OP_WAIT */ __umtx_op_wake, /* UMTX_OP_WAKE */ __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */ __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */ __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/ __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */ __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */ __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */ __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */ __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */ __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ __umtx_op_wait_umutex_compat32, /* UMTX_OP_MUTEX_WAIT */ __umtx_op_wake_umutex, /* UMTX_OP_MUTEX_WAKE */ #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */ __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */ #else __umtx_op_unimpl, /* UMTX_OP_SEM_WAIT */ __umtx_op_unimpl, /* UMTX_OP_SEM_WAKE */ #endif __umtx_op_nwake_private32, /* UMTX_OP_NWAKE_PRIVATE */ __umtx_op_wake2_umutex, /* UMTX_OP_MUTEX_WAKE2 */ __umtx_op_sem2_wait_compat32, /* UMTX_OP_SEM2_WAIT */ __umtx_op_sem2_wake, /* UMTX_OP_SEM2_WAKE */ }; int freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap) { if ((unsigned)uap->op < UMTX_OP_MAX) return (*op_table_compat32[uap->op])(td, (struct _umtx_op_args *)uap); return (EINVAL); } #endif void umtx_thread_init(struct thread *td) { td->td_umtxq = umtxq_alloc(); td->td_umtxq->uq_thread = td; } void umtx_thread_fini(struct thread *td) { umtxq_free(td->td_umtxq); } /* * It will be called when new thread is created, e.g fork(). */ void umtx_thread_alloc(struct thread *td) { struct umtx_q *uq; uq = td->td_umtxq; uq->uq_inherited_pri = PRI_MAX; KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); KASSERT(uq->uq_thread == td, ("uq_thread != td")); KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); } /* * exec() hook. */ static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, struct image_params *imgp __unused) { umtx_thread_cleanup(curthread); } /* * thread_exit() hook. */ void umtx_thread_exit(struct thread *td) { umtx_thread_cleanup(td); } /* * clean up umtx data. */ static void umtx_thread_cleanup(struct thread *td) { struct umtx_q *uq; struct umtx_pi *pi; if ((uq = td->td_umtxq) == NULL) return; mtx_lock_spin(&umtx_lock); uq->uq_inherited_pri = PRI_MAX; while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { pi->pi_owner = NULL; TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); } mtx_unlock_spin(&umtx_lock); thread_lock(td); sched_lend_user_prio(td, PRI_MAX); thread_unlock(td); } diff --git a/sys/mips/atheros/ar71xx_gpio.c b/sys/mips/atheros/ar71xx_gpio.c index 45ac8ffda2ed..b48e4a6de21b 100644 --- a/sys/mips/atheros/ar71xx_gpio.c +++ b/sys/mips/atheros/ar71xx_gpio.c @@ -1,485 +1,489 @@ /*- * Copyright (c) 2009, Oleksandr Tymoshenko * Copyright (c) 2009, Luiz Otavio O Souza. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * GPIO driver for AR71xx */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT) /* * Helpers */ static void ar71xx_gpio_function_enable(struct ar71xx_gpio_softc *sc, uint32_t mask); static void ar71xx_gpio_function_disable(struct ar71xx_gpio_softc *sc, uint32_t mask); static void ar71xx_gpio_pin_configure(struct ar71xx_gpio_softc *sc, struct gpio_pin *pin, uint32_t flags); /* * Driver stuff */ static int ar71xx_gpio_probe(device_t dev); static int ar71xx_gpio_attach(device_t dev); static int ar71xx_gpio_detach(device_t dev); static int ar71xx_gpio_filter(void *arg); static void ar71xx_gpio_intr(void *arg); /* * GPIO interface */ static int ar71xx_gpio_pin_max(device_t dev, int *maxpin); static int ar71xx_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps); static int ar71xx_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags); static int ar71xx_gpio_pin_getname(device_t dev, uint32_t pin, char *name); static int ar71xx_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags); static int ar71xx_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value); static int ar71xx_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val); static int ar71xx_gpio_pin_toggle(device_t dev, uint32_t pin); static void ar71xx_gpio_function_enable(struct ar71xx_gpio_softc *sc, uint32_t mask) { if (ar71xx_soc == AR71XX_SOC_AR9341 || ar71xx_soc == AR71XX_SOC_AR9342 || ar71xx_soc == AR71XX_SOC_AR9344) GPIO_SET_BITS(sc, AR934X_GPIO_REG_FUNC, mask); else GPIO_SET_BITS(sc, AR71XX_GPIO_FUNCTION, mask); } static void ar71xx_gpio_function_disable(struct ar71xx_gpio_softc *sc, uint32_t mask) { if (ar71xx_soc == AR71XX_SOC_AR9341 || ar71xx_soc == AR71XX_SOC_AR9342 || ar71xx_soc == AR71XX_SOC_AR9344) GPIO_CLEAR_BITS(sc, AR934X_GPIO_REG_FUNC, mask); else GPIO_CLEAR_BITS(sc, AR71XX_GPIO_FUNCTION, mask); } static void ar71xx_gpio_pin_configure(struct ar71xx_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { uint32_t mask; mask = 1 << pin->gp_pin; /* * Manage input/output */ if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; GPIO_SET_BITS(sc, AR71XX_GPIO_OE, mask); } else { pin->gp_flags |= GPIO_PIN_INPUT; GPIO_CLEAR_BITS(sc, AR71XX_GPIO_OE, mask); } } } static int ar71xx_gpio_pin_max(device_t dev, int *maxpin) { switch (ar71xx_soc) { case AR71XX_SOC_AR9130: case AR71XX_SOC_AR9132: *maxpin = AR91XX_GPIO_PINS - 1; break; case AR71XX_SOC_AR7240: case AR71XX_SOC_AR7241: case AR71XX_SOC_AR7242: *maxpin = AR724X_GPIO_PINS - 1; break; case AR71XX_SOC_AR9330: case AR71XX_SOC_AR9331: *maxpin = AR933X_GPIO_COUNT - 1; break; case AR71XX_SOC_AR9341: case AR71XX_SOC_AR9342: case AR71XX_SOC_AR9344: *maxpin = AR934X_GPIO_COUNT - 1; break; default: *maxpin = AR71XX_GPIO_PINS - 1; } return (0); } static int ar71xx_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct ar71xx_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *caps = sc->gpio_pins[i].gp_caps; GPIO_UNLOCK(sc); return (0); } static int ar71xx_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct ar71xx_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *flags = sc->gpio_pins[i].gp_flags; GPIO_UNLOCK(sc); return (0); } static int ar71xx_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct ar71xx_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME); GPIO_UNLOCK(sc); return (0); } static int ar71xx_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { int i; struct ar71xx_gpio_softc *sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); ar71xx_gpio_pin_configure(sc, &sc->gpio_pins[i], flags); return (0); } static int ar71xx_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct ar71xx_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); if (value) GPIO_WRITE(sc, AR71XX_GPIO_SET, (1 << pin)); else GPIO_WRITE(sc, AR71XX_GPIO_CLEAR, (1 << pin)); return (0); } static int ar71xx_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct ar71xx_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); *val = (GPIO_READ(sc, AR71XX_GPIO_IN) & (1 << pin)) ? 1 : 0; return (0); } static int ar71xx_gpio_pin_toggle(device_t dev, uint32_t pin) { int res, i; struct ar71xx_gpio_softc *sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); res = (GPIO_READ(sc, AR71XX_GPIO_IN) & (1 << pin)) ? 1 : 0; if (res) GPIO_WRITE(sc, AR71XX_GPIO_CLEAR, (1 << pin)); else GPIO_WRITE(sc, AR71XX_GPIO_SET, (1 << pin)); return (0); } static int ar71xx_gpio_filter(void *arg) { /* TODO: something useful */ return (FILTER_STRAY); } static void ar71xx_gpio_intr(void *arg) { struct ar71xx_gpio_softc *sc = arg; GPIO_LOCK(sc); /* TODO: something useful */ GPIO_UNLOCK(sc); } static int ar71xx_gpio_probe(device_t dev) { device_set_desc(dev, "Atheros AR71XX GPIO driver"); return (0); } static int ar71xx_gpio_attach(device_t dev) { struct ar71xx_gpio_softc *sc = device_get_softc(dev); - int error = 0; int i, j, maxpin; int mask, pinon; uint32_t oe; KASSERT((device_get_unit(dev) == 0), ("ar71xx_gpio: Only one gpio module supported")); mtx_init(&sc->gpio_mtx, device_get_nameunit(dev), NULL, MTX_DEF); /* Map control/status registers. */ sc->gpio_mem_rid = 0; sc->gpio_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->gpio_mem_rid, RF_ACTIVE); if (sc->gpio_mem_res == NULL) { device_printf(dev, "couldn't map memory\n"); - error = ENXIO; ar71xx_gpio_detach(dev); - return(error); + return (ENXIO); } if ((sc->gpio_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->gpio_irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { device_printf(dev, "unable to allocate IRQ resource\n"); + ar71xx_gpio_detach(dev); return (ENXIO); } if ((bus_setup_intr(dev, sc->gpio_irq_res, INTR_TYPE_MISC, ar71xx_gpio_filter, ar71xx_gpio_intr, sc, &sc->gpio_ih))) { device_printf(dev, "WARNING: unable to register interrupt handler\n"); + ar71xx_gpio_detach(dev); return (ENXIO); } sc->dev = dev; /* Enable function bits that are required */ if (resource_int_value(device_get_name(dev), device_get_unit(dev), "function_set", &mask) == 0) { device_printf(dev, "function_set: 0x%x\n", mask); ar71xx_gpio_function_enable(sc, mask); } /* Disable function bits that are required */ if (resource_int_value(device_get_name(dev), device_get_unit(dev), "function_clear", &mask) == 0) { device_printf(dev, "function_clear: 0x%x\n", mask); ar71xx_gpio_function_disable(sc, mask); } /* Disable interrupts for all pins. */ GPIO_WRITE(sc, AR71XX_GPIO_INT_MASK, 0); /* Initialise all pins specified in the mask, up to the pin count */ (void) ar71xx_gpio_pin_max(dev, &maxpin); if (resource_int_value(device_get_name(dev), device_get_unit(dev), "pinmask", &mask) != 0) mask = 0; if (resource_int_value(device_get_name(dev), device_get_unit(dev), "pinon", &pinon) != 0) pinon = 0; device_printf(dev, "gpio pinmask=0x%x\n", mask); for (j = 0; j <= maxpin; j++) { if ((mask & (1 << j)) == 0) continue; sc->gpio_npins++; } /* Iniatilize the GPIO pins, keep the loader settings. */ oe = GPIO_READ(sc, AR71XX_GPIO_OE); sc->gpio_pins = malloc(sizeof(*sc->gpio_pins) * sc->gpio_npins, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0, j = 0; j <= maxpin; j++) { if ((mask & (1 << j)) == 0) continue; snprintf(sc->gpio_pins[i].gp_name, GPIOMAXNAME, "pin %d", j); sc->gpio_pins[i].gp_pin = j; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; if (oe & (1 << j)) sc->gpio_pins[i].gp_flags = GPIO_PIN_OUTPUT; else sc->gpio_pins[i].gp_flags = GPIO_PIN_INPUT; i++; } /* Turn on the hinted pins. */ for (i = 0; i < sc->gpio_npins; i++) { j = sc->gpio_pins[i].gp_pin; if ((pinon & (1 << j)) != 0) { ar71xx_gpio_pin_setflags(dev, j, GPIO_PIN_OUTPUT); ar71xx_gpio_pin_set(dev, j, 1); } } device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); } static int ar71xx_gpio_detach(device_t dev) { struct ar71xx_gpio_softc *sc = device_get_softc(dev); KASSERT(mtx_initialized(&sc->gpio_mtx), ("gpio mutex not initialized")); bus_generic_detach(dev); - + if (sc->gpio_ih) + bus_teardown_intr(dev, sc->gpio_irq_res, sc->gpio_ih); + if (sc->gpio_irq_res) + bus_release_resource(dev, SYS_RES_IRQ, sc->gpio_irq_rid, + sc->gpio_irq_res); if (sc->gpio_mem_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->gpio_mem_rid, sc->gpio_mem_res); - - free(sc->gpio_pins, M_DEVBUF); + if (sc->gpio_pins) + free(sc->gpio_pins, M_DEVBUF); mtx_destroy(&sc->gpio_mtx); return(0); } static device_method_t ar71xx_gpio_methods[] = { DEVMETHOD(device_probe, ar71xx_gpio_probe), DEVMETHOD(device_attach, ar71xx_gpio_attach), DEVMETHOD(device_detach, ar71xx_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, ar71xx_gpio_pin_max), DEVMETHOD(gpio_pin_getname, ar71xx_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, ar71xx_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, ar71xx_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, ar71xx_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, ar71xx_gpio_pin_get), DEVMETHOD(gpio_pin_set, ar71xx_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, ar71xx_gpio_pin_toggle), {0, 0}, }; static driver_t ar71xx_gpio_driver = { "gpio", ar71xx_gpio_methods, sizeof(struct ar71xx_gpio_softc), }; static devclass_t ar71xx_gpio_devclass; DRIVER_MODULE(ar71xx_gpio, apb, ar71xx_gpio_driver, ar71xx_gpio_devclass, 0, 0); diff --git a/sys/mips/atheros/ar71xx_gpiovar.h b/sys/mips/atheros/ar71xx_gpiovar.h index a1c6e2f1c5a6..32337ccea87a 100644 --- a/sys/mips/atheros/ar71xx_gpiovar.h +++ b/sys/mips/atheros/ar71xx_gpiovar.h @@ -1,70 +1,70 @@ /*- * Copyright (c) 2009, Oleksandr Tymoshenko * Copyright (c) 2009, Luiz Otavio O Souza. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __AR71XX_GPIOVAR_H__ #define __AR71XX_GPIOVAR_H__ #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->gpio_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->gpio_mtx) #define GPIO_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->gpio_mtx, MA_OWNED) /* * register space access macros */ #define GPIO_WRITE(sc, reg, val) do { \ bus_write_4(sc->gpio_mem_res, (reg), (val)); \ } while (0) #define GPIO_READ(sc, reg) bus_read_4(sc->gpio_mem_res, (reg)) #define GPIO_SET_BITS(sc, reg, bits) \ GPIO_WRITE(sc, reg, GPIO_READ(sc, (reg)) | (bits)) #define GPIO_CLEAR_BITS(sc, reg, bits) \ GPIO_WRITE(sc, reg, GPIO_READ(sc, (reg)) & ~(bits)) #define AR71XX_GPIO_PINS 12 #define AR724X_GPIO_PINS 18 #define AR91XX_GPIO_PINS 22 struct ar71xx_gpio_softc { device_t dev; - struct mtx gpio_mtx; - struct resource *gpio_mem_res; - int gpio_mem_rid; - struct resource *gpio_irq_res; - int gpio_irq_rid; - void *gpio_ih; + struct mtx gpio_mtx; + struct resource *gpio_mem_res; + int gpio_mem_rid; + struct resource *gpio_irq_res; + int gpio_irq_rid; + void *gpio_ih; int gpio_npins; struct gpio_pin *gpio_pins; }; #endif /* __AR71XX_GPIOVAR_H__ */ diff --git a/sys/mips/cavium/octeon_gpio.c b/sys/mips/cavium/octeon_gpio.c index 36868481a996..1053976c508f 100644 --- a/sys/mips/cavium/octeon_gpio.c +++ b/sys/mips/cavium/octeon_gpio.c @@ -1,486 +1,491 @@ /*- * Copyright (c) 2011, Oleksandr Tymoshenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * GPIO driver for Cavium Octeon */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT) struct octeon_gpio_pin { const char *name; int pin; int flags; }; /* * on CAP100 GPIO 7 is "Factory defaults" button * */ static struct octeon_gpio_pin octeon_gpio_pins[] = { { "F/D", 7, GPIO_PIN_INPUT}, { NULL, 0, 0}, }; /* * Helpers */ static void octeon_gpio_pin_configure(struct octeon_gpio_softc *sc, struct gpio_pin *pin, uint32_t flags); /* * Driver stuff */ static void octeon_gpio_identify(driver_t *, device_t); static int octeon_gpio_probe(device_t dev); static int octeon_gpio_attach(device_t dev); static int octeon_gpio_detach(device_t dev); static int octeon_gpio_filter(void *arg); static void octeon_gpio_intr(void *arg); /* * GPIO interface */ static int octeon_gpio_pin_max(device_t dev, int *maxpin); static int octeon_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps); static int octeon_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags); static int octeon_gpio_pin_getname(device_t dev, uint32_t pin, char *name); static int octeon_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags); static int octeon_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value); static int octeon_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val); static int octeon_gpio_pin_toggle(device_t dev, uint32_t pin); static void octeon_gpio_pin_configure(struct octeon_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { uint32_t mask; cvmx_gpio_bit_cfgx_t gpio_cfgx; mask = 1 << pin->gp_pin; GPIO_LOCK(sc); /* * Manage input/output */ if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { gpio_cfgx.u64 = cvmx_read_csr(CVMX_GPIO_BIT_CFGX(pin->gp_pin)); pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; gpio_cfgx.s.tx_oe = 1; } else { pin->gp_flags |= GPIO_PIN_INPUT; gpio_cfgx.s.tx_oe = 0; } if (flags & GPIO_PIN_INVIN) gpio_cfgx.s.rx_xor = 1; else gpio_cfgx.s.rx_xor = 0; cvmx_write_csr(CVMX_GPIO_BIT_CFGX(pin->gp_pin), gpio_cfgx.u64); } GPIO_UNLOCK(sc); } static int octeon_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = OCTEON_GPIO_PINS - 1; return (0); } static int octeon_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct octeon_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *caps = sc->gpio_pins[i].gp_caps; GPIO_UNLOCK(sc); return (0); } static int octeon_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct octeon_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *flags = sc->gpio_pins[i].gp_flags; GPIO_UNLOCK(sc); return (0); } static int octeon_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct octeon_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME); GPIO_UNLOCK(sc); return (0); } static int octeon_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { int i; struct octeon_gpio_softc *sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); octeon_gpio_pin_configure(sc, &sc->gpio_pins[i], flags); return (0); } static int octeon_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct octeon_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); if (value) cvmx_gpio_set(1 << pin); else cvmx_gpio_clear(1 << pin); GPIO_UNLOCK(sc); return (0); } static int octeon_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct octeon_gpio_softc *sc = device_get_softc(dev); int i; uint64_t state; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); state = cvmx_gpio_read(); *val = (state & (1 << pin)) ? 1 : 0; GPIO_UNLOCK(sc); return (0); } static int octeon_gpio_pin_toggle(device_t dev, uint32_t pin) { int i; uint64_t state; struct octeon_gpio_softc *sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); /* * XXX: Need to check if read returns actual state of output * pins or we need to keep this information by ourself */ state = cvmx_gpio_read(); if (state & (1 << pin)) cvmx_gpio_clear(1 << pin); else cvmx_gpio_set(1 << pin); GPIO_UNLOCK(sc); return (0); } static int octeon_gpio_filter(void *arg) { cvmx_gpio_bit_cfgx_t gpio_cfgx; void **cookie = arg; struct octeon_gpio_softc *sc = *cookie; long int irq = (cookie - sc->gpio_intr_cookies); if ((irq < 0) || (irq >= OCTEON_GPIO_IRQS)) return (FILTER_STRAY); gpio_cfgx.u64 = cvmx_read_csr(CVMX_GPIO_BIT_CFGX(irq)); /* Clear rising edge detector */ if (gpio_cfgx.s.int_type == OCTEON_GPIO_IRQ_EDGE) cvmx_gpio_interrupt_clear(1 << irq); /* disable interrupt */ gpio_cfgx.s.int_en = 0; cvmx_write_csr(CVMX_GPIO_BIT_CFGX(irq), gpio_cfgx.u64); return (FILTER_SCHEDULE_THREAD); } static void octeon_gpio_intr(void *arg) { cvmx_gpio_bit_cfgx_t gpio_cfgx; void **cookie = arg; struct octeon_gpio_softc *sc = *cookie; long int irq = (cookie - sc->gpio_intr_cookies); if ((irq < 0) || (irq >= OCTEON_GPIO_IRQS)) { printf("%s: invalid GPIO IRQ: %ld\n", __func__, irq); return; } GPIO_LOCK(sc); gpio_cfgx.u64 = cvmx_read_csr(CVMX_GPIO_BIT_CFGX(irq)); /* disable interrupt */ gpio_cfgx.s.int_en = 1; cvmx_write_csr(CVMX_GPIO_BIT_CFGX(irq), gpio_cfgx.u64); /* TODO: notify bus here or something */ printf("GPIO IRQ for pin %ld\n", irq); GPIO_UNLOCK(sc); } static void octeon_gpio_identify(driver_t *drv, device_t parent) { BUS_ADD_CHILD(parent, 0, "gpio", 0); } static int octeon_gpio_probe(device_t dev) { device_set_desc(dev, "Cavium Octeon GPIO driver"); return (0); } static int octeon_gpio_attach(device_t dev) { struct octeon_gpio_softc *sc = device_get_softc(dev); struct octeon_gpio_pin *pinp; cvmx_gpio_bit_cfgx_t gpio_cfgx; int i; KASSERT((device_get_unit(dev) == 0), ("octeon_gpio: Only one gpio module supported")); mtx_init(&sc->gpio_mtx, device_get_nameunit(dev), NULL, MTX_DEF); for ( i = 0; i < OCTEON_GPIO_IRQS; i++) { if ((sc->gpio_irq_res[i] = bus_alloc_resource(dev, SYS_RES_IRQ, &sc->gpio_irq_rid[i], OCTEON_IRQ_GPIO0 + i, OCTEON_IRQ_GPIO0 + i, 1, RF_SHAREABLE | RF_ACTIVE)) == NULL) { device_printf(dev, "unable to allocate IRQ resource\n"); + octeon_gpio_detach(dev); return (ENXIO); } sc->gpio_intr_cookies[i] = sc; if ((bus_setup_intr(dev, sc->gpio_irq_res[i], INTR_TYPE_MISC, octeon_gpio_filter, octeon_gpio_intr, &(sc->gpio_intr_cookies[i]), &sc->gpio_ih[i]))) { device_printf(dev, "WARNING: unable to register interrupt handler\n"); + octeon_gpio_detach(dev); return (ENXIO); } } sc->dev = dev; /* Configure all pins as input */ /* disable interrupts for all pins */ pinp = octeon_gpio_pins; i = 0; while (pinp->name) { strncpy(sc->gpio_pins[i].gp_name, pinp->name, GPIOMAXNAME); sc->gpio_pins[i].gp_pin = pinp->pin; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; sc->gpio_pins[i].gp_flags = 0; octeon_gpio_pin_configure(sc, &sc->gpio_pins[i], pinp->flags); pinp++; i++; } sc->gpio_npins = i; #if 0 /* * Sample: how to enable edge-triggered interrupt * for GPIO pin */ gpio_cfgx.u64 = cvmx_read_csr(CVMX_GPIO_BIT_CFGX(7)); gpio_cfgx.s.int_en = 1; gpio_cfgx.s.int_type = OCTEON_GPIO_IRQ_EDGE; cvmx_write_csr(CVMX_GPIO_BIT_CFGX(7), gpio_cfgx.u64); #endif if (bootverbose) { for (i = 0; i < 16; i++) { gpio_cfgx.u64 = cvmx_read_csr(CVMX_GPIO_BIT_CFGX(i)); device_printf(dev, "[pin%d] output=%d, invinput=%d, intr=%d, intr_type=%s\n", i, gpio_cfgx.s.tx_oe, gpio_cfgx.s.rx_xor, gpio_cfgx.s.int_en, gpio_cfgx.s.int_type ? "rising edge" : "level"); } } device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); } static int octeon_gpio_detach(device_t dev) { struct octeon_gpio_softc *sc = device_get_softc(dev); int i; KASSERT(mtx_initialized(&sc->gpio_mtx), ("gpio mutex not initialized")); for ( i = 0; i < OCTEON_GPIO_IRQS; i++) { - bus_release_resource(dev, SYS_RES_IRQ, - sc->gpio_irq_rid[i], sc->gpio_irq_res[i]); + if (sc->gpio_ih[i]) + bus_teardown_intr(dev, sc->gpio_irq_res[i], + sc->gpio_ih[i]); + if (sc->gpio_irq_res[i]) + bus_release_resource(dev, SYS_RES_IRQ, + sc->gpio_irq_rid[i], sc->gpio_irq_res[i]); } bus_generic_detach(dev); - mtx_destroy(&sc->gpio_mtx); return(0); } static device_method_t octeon_gpio_methods[] = { DEVMETHOD(device_identify, octeon_gpio_identify), DEVMETHOD(device_probe, octeon_gpio_probe), DEVMETHOD(device_attach, octeon_gpio_attach), DEVMETHOD(device_detach, octeon_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, octeon_gpio_pin_max), DEVMETHOD(gpio_pin_getname, octeon_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, octeon_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, octeon_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, octeon_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, octeon_gpio_pin_get), DEVMETHOD(gpio_pin_set, octeon_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, octeon_gpio_pin_toggle), {0, 0}, }; static driver_t octeon_gpio_driver = { "gpio", octeon_gpio_methods, sizeof(struct octeon_gpio_softc), }; static devclass_t octeon_gpio_devclass; DRIVER_MODULE(octeon_gpio, ciu, octeon_gpio_driver, octeon_gpio_devclass, 0, 0); diff --git a/sys/mips/cavium/octeon_gpiovar.h b/sys/mips/cavium/octeon_gpiovar.h index 5717b7cc8636..a9b814fefb8d 100644 --- a/sys/mips/cavium/octeon_gpiovar.h +++ b/sys/mips/cavium/octeon_gpiovar.h @@ -1,55 +1,55 @@ /*- * Copyright (c) 2011, Oleksandr Tymoshenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __OCTEON_GPIOVAR_H__ #define __OCTEON_GPIOVAR_H__ #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->gpio_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->gpio_mtx) #define GPIO_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->gpio_mtx, MA_OWNED) #define OCTEON_GPIO_IRQ_LEVEL 0 #define OCTEON_GPIO_IRQ_EDGE 1 #define OCTEON_GPIO_PINS 24 #define OCTEON_GPIO_IRQS 16 struct octeon_gpio_softc { device_t dev; - struct mtx gpio_mtx; - struct resource *gpio_irq_res[OCTEON_GPIO_IRQS]; - int gpio_irq_rid[OCTEON_GPIO_IRQS]; - void *gpio_ih[OCTEON_GPIO_IRQS]; - void *gpio_intr_cookies[OCTEON_GPIO_IRQS]; + struct mtx gpio_mtx; + struct resource *gpio_irq_res[OCTEON_GPIO_IRQS]; + int gpio_irq_rid[OCTEON_GPIO_IRQS]; + void *gpio_ih[OCTEON_GPIO_IRQS]; + void *gpio_intr_cookies[OCTEON_GPIO_IRQS]; int gpio_npins; struct gpio_pin gpio_pins[OCTEON_GPIO_PINS]; }; #endif /* __OCTEON_GPIOVAR_H__ */ diff --git a/sys/mips/rt305x/rt305x_gpio.c b/sys/mips/rt305x/rt305x_gpio.c index a5d72549bd03..bf03bd149fcd 100644 --- a/sys/mips/rt305x/rt305x_gpio.c +++ b/sys/mips/rt305x/rt305x_gpio.c @@ -1,607 +1,611 @@ /*- * Copyright (c) 2010-2011, Aleksandr Rybalko * Copyright (c) 2009, Oleksandr Tymoshenko * Copyright (c) 2009, Luiz Otavio O Souza. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * GPIO driver for RT305X SoC. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #ifdef notyet #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | GPIO_PIN_INVIN | \ GPIO_PIN_INVOUT | GPIO_PIN_REPORT ) #else #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | GPIO_PIN_INVIN | \ GPIO_PIN_INVOUT ) #endif /* * Helpers */ static void rt305x_gpio_pin_configure(struct rt305x_gpio_softc *sc, struct gpio_pin *pin, uint32_t flags); /* * Driver stuff */ static int rt305x_gpio_probe(device_t dev); static int rt305x_gpio_attach(device_t dev); static int rt305x_gpio_detach(device_t dev); static int rt305x_gpio_intr(void *arg); int rt305x_get_int_mask (device_t); void rt305x_set_int_mask (device_t, uint32_t); int rt305x_get_int_status(device_t); void rt305x_set_int_status(device_t, uint32_t); /* * GPIO interface */ static int rt305x_gpio_pin_max(device_t dev, int *maxpin); static int rt305x_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps); static int rt305x_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags); static int rt305x_gpio_pin_getname(device_t dev, uint32_t pin, char *name); static int rt305x_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags); static int rt305x_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value); static int rt305x_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val); static int rt305x_gpio_pin_toggle(device_t dev, uint32_t pin); static void rt305x_gpio_pin_configure(struct rt305x_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { GPIO_LOCK(sc); /* * Manage input/output */ if (flags & (GPIO_PIN_INPUT|GPIO_PIN_OUTPUT)) { pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { pin->gp_flags |= GPIO_PIN_OUTPUT; GPIO_BIT_SET(sc, pin->gp_pin, DIR); } else { pin->gp_flags |= GPIO_PIN_INPUT; GPIO_BIT_CLR(sc, pin->gp_pin, DIR); } } if (flags & GPIO_PIN_INVOUT) { pin->gp_flags |= GPIO_PIN_INVOUT; GPIO_BIT_SET(sc, pin->gp_pin, POL); } else { pin->gp_flags &= ~GPIO_PIN_INVOUT; GPIO_BIT_CLR(sc, pin->gp_pin, POL); } if (flags & GPIO_PIN_INVIN) { pin->gp_flags |= GPIO_PIN_INVIN; GPIO_BIT_SET(sc, pin->gp_pin, POL); } else { pin->gp_flags &= ~GPIO_PIN_INVIN; GPIO_BIT_CLR(sc, pin->gp_pin, POL); } #ifdef notyet /* Enable interrupt bits for rising/falling transitions */ if (flags & GPIO_PIN_REPORT) { pin->gp_flags |= GPIO_PIN_REPORT; GPIO_BIT_SET(sc, pin->gp_pin, RENA); GPIO_BIT_SET(sc, pin->gp_pin, FENA); device_printf(sc->dev, "Will report interrupt on pin %d\n", pin->gp_pin); } else { pin->gp_flags &= ~GPIO_PIN_REPORT; GPIO_BIT_CLR(sc, pin->gp_pin, RENA); GPIO_BIT_CLR(sc, pin->gp_pin, FENA); } #else /* Disable generating interrupts for now */ GPIO_BIT_CLR(sc, pin->gp_pin, RENA); GPIO_BIT_CLR(sc, pin->gp_pin, FENA); #endif GPIO_UNLOCK(sc); } static int rt305x_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = NGPIO - 1; return (0); } static int rt305x_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct rt305x_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *caps = sc->gpio_pins[i].gp_caps; GPIO_UNLOCK(sc); return (0); } static int rt305x_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct rt305x_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *flags = sc->gpio_pins[i].gp_flags; GPIO_UNLOCK(sc); return (0); } static int rt305x_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct rt305x_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME); GPIO_UNLOCK(sc); return (0); } static int rt305x_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { int i; struct rt305x_gpio_softc *sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); rt305x_gpio_pin_configure(sc, &sc->gpio_pins[i], flags); return (0); } static int rt305x_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct rt305x_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); if (value) GPIO_BIT_SET(sc, i, DATA); else GPIO_BIT_CLR(sc, i, DATA); GPIO_UNLOCK(sc); return (0); } static int rt305x_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct rt305x_gpio_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); *val = GPIO_BIT_GET(sc, i, DATA); GPIO_UNLOCK(sc); return (0); } static int rt305x_gpio_pin_toggle(device_t dev, uint32_t pin) { int i; struct rt305x_gpio_softc *sc = device_get_softc(dev); for (i = 0; i < sc->gpio_npins; i++) { if (sc->gpio_pins[i].gp_pin == pin) break; } if (i >= sc->gpio_npins) return (EINVAL); GPIO_LOCK(sc); GPIO_BIT_SET(sc, i, TOG); GPIO_UNLOCK(sc); return (0); } static int rt305x_gpio_intr(void *arg) { struct rt305x_gpio_softc *sc = arg; #ifdef notyet uint32_t i; #endif uint64_t input, value; #ifdef notyet uint64_t reset_pin; char notify[16]; char pinname[6]; #endif /* Read all reported pins */ input = GPIO_READ_ALL(sc, INT); /* Clear int status */ GPIO_WRITE_ALL(sc, INT, input); /* Clear report for OUTs */ input &= ~GPIO_READ_ALL(sc, DIR); value = input & GPIO_READ_ALL(sc, DATA); if (!input) goto intr_done; #ifdef notyet /* if reset_gpio and this pin is input */ if (sc->reset_gpio >= 0 && (input & (1 << sc->reset_gpio))) { /* get reset_gpio pin value */ reset_pin = (value & (1 << sc->reset_gpio))?1:0; if ( sc->reset_gpio_last != reset_pin ) { /* * if now reset is high, check how long * and do reset if less than 2 seconds */ if ( reset_pin && (time_uptime - sc->reset_gpio_ontime) < 2 ) shutdown_nice(0); sc->reset_gpio_last = reset_pin; sc->reset_gpio_ontime = time_uptime; } } for ( i = 0; i < NGPIO; i ++ ) { /* Next if output pin */ if ( !(( input >> i) & 1) ) continue; if ( (((value & input) >> i) & 1) != sc->gpio_pins[i].gp_last ) { /* !system=GPIO subsystem=pin7 type=PIN_HIGH period=3 */ snprintf(notify , sizeof(notify ), "period=%d", (uint32_t)time_uptime - sc->gpio_pins[i].gp_time); snprintf(pinname, sizeof(pinname), "pin%02d", i); devctl_notify("GPIO", pinname, (((value & input) >> i) & 1)?"PIN_HIGH":"PIN_LOW", notify); printf("GPIO[%s] %s %s\n", pinname, (((value & input) >> i) & 1)?"PIN_HIGH":"PIN_LOW", notify); sc->gpio_pins[i].gp_last = ((value & input) >> i) & 1; sc->gpio_pins[i].gp_time = time_uptime; } } #endif intr_done: return (FILTER_HANDLED); } static int rt305x_gpio_probe(device_t dev) { device_set_desc(dev, "RT305X GPIO driver"); return (0); } static uint64_t rt305x_gpio_init(device_t dev) { uint64_t avl = ~0ULL; uint32_t gmode = rt305x_sysctl_get(SYSCTL_GPIOMODE); if (!(gmode & SYSCTL_GPIOMODE_RGMII_GPIO_MODE)) avl &= ~RGMII_GPIO_MODE_MASK; if (!(gmode & SYSCTL_GPIOMODE_SDRAM_GPIO_MODE)) avl &= ~SDRAM_GPIO_MODE_MASK; if (!(gmode & SYSCTL_GPIOMODE_MDIO_GPIO_MODE)) avl &= ~MDIO_GPIO_MODE_MASK; if (!(gmode & SYSCTL_GPIOMODE_JTAG_GPIO_MODE)) avl &= ~JTAG_GPIO_MODE_MASK; if (!(gmode & SYSCTL_GPIOMODE_UARTL_GPIO_MODE)) avl &= ~UARTL_GPIO_MODE_MASK; if (!(gmode & SYSCTL_GPIOMODE_SPI_GPIO_MODE)) avl &= ~SPI_GPIO_MODE_MASK; if (!(gmode & SYSCTL_GPIOMODE_I2C_GPIO_MODE)) avl &= ~I2C_GPIO_MODE_MASK; if ((gmode & SYSCTL_GPIOMODE_UARTF_SHARE_MODE_GPIO) != SYSCTL_GPIOMODE_UARTF_SHARE_MODE_GPIO) avl &= ~I2C_GPIO_MODE_MASK; /* D-Link DAP-1350 Board have * MDIO_GPIO_MODE * UARTF_GPIO_MODE * SPI_GPIO_MODE * I2C_GPIO_MODE * So we have * 00000001 10000000 01111111 11111110 */ return (avl); } #define DAP1350_RESET_GPIO 10 static int rt305x_gpio_attach(device_t dev) { struct rt305x_gpio_softc *sc = device_get_softc(dev); - int error = 0, i; + int i; uint64_t avlpins = 0; sc->reset_gpio = DAP1350_RESET_GPIO; KASSERT((device_get_unit(dev) == 0), ("rt305x_gpio_gpio: Only one gpio module supported")); mtx_init(&sc->gpio_mtx, device_get_nameunit(dev), NULL, MTX_DEF); /* Map control/status registers. */ sc->gpio_mem_rid = 0; sc->gpio_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->gpio_mem_rid, RF_ACTIVE); if (sc->gpio_mem_res == NULL) { device_printf(dev, "couldn't map memory\n"); - error = ENXIO; rt305x_gpio_detach(dev); - return(error); + return (ENXIO); } if ((sc->gpio_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->gpio_irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { device_printf(dev, "unable to allocate IRQ resource\n"); + rt305x_gpio_detach(dev); return (ENXIO); } if ((bus_setup_intr(dev, sc->gpio_irq_res, INTR_TYPE_MISC, /* rt305x_gpio_filter, */ rt305x_gpio_intr, NULL, sc, &sc->gpio_ih))) { device_printf(dev, "WARNING: unable to register interrupt handler\n"); + rt305x_gpio_detach(dev); return (ENXIO); } sc->dev = dev; avlpins = rt305x_gpio_init(dev); /* Configure all pins as input */ /* disable interrupts for all pins */ /* TODO */ sc->gpio_npins = NGPIO; resource_int_value(device_get_name(dev), device_get_unit(dev), "pins", &sc->gpio_npins); for (i = 0; i < sc->gpio_npins; i++) { sc->gpio_pins[i].gp_pin = i; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; sc->gpio_pins[i].gp_flags = 0; } /* Setup reset pin interrupt */ if (TUNABLE_INT_FETCH("reset_gpio", &sc->reset_gpio)) { device_printf(dev, "\tHinted reset_gpio %d\n", sc->reset_gpio); } #ifdef notyet if (sc->reset_gpio != -1) { rt305x_gpio_pin_setflags(dev, sc->reset_gpio, GPIO_PIN_INPUT|GPIO_PIN_INVOUT| GPIO_PIN_INVOUT|GPIO_PIN_REPORT); device_printf(dev, "\tUse reset_gpio %d\n", sc->reset_gpio); } #else if (sc->reset_gpio != -1) { rt305x_gpio_pin_setflags(dev, sc->reset_gpio, GPIO_PIN_INPUT|GPIO_PIN_INVOUT); device_printf(dev, "\tUse reset_gpio %d\n", sc->reset_gpio); } #endif device_add_child(dev, "gpioc", -1); device_add_child(dev, "gpiobus", -1); return (bus_generic_attach(dev)); } static int rt305x_gpio_detach(device_t dev) { struct rt305x_gpio_softc *sc = device_get_softc(dev); KASSERT(mtx_initialized(&sc->gpio_mtx), ("gpio mutex not initialized")); bus_generic_detach(dev); - + if (sc->gpio_ih) + bus_teardown_intr(dev, sc->gpio_irq_res, sc->gpio_ih); + if (sc->gpio_irq_res) + bus_release_resource(dev, SYS_RES_IRQ, sc->gpio_irq_rid, + sc->gpio_irq_res); if (sc->gpio_mem_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->gpio_mem_rid, sc->gpio_mem_res); - mtx_destroy(&sc->gpio_mtx); return(0); } #ifdef notyet static struct resource * rt305x_gpio_alloc_resource(device_t bus, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { struct obio_softc *sc = device_get_softc(bus); struct resource *rv; struct rman *rm; switch (type) { case SYS_RES_GPIO: rm = &sc->gpio_rman; break; default: printf("%s: unknown resource type %d\n", __func__, type); return (0); } rv = rman_reserve_resource(rm, start, end, count, flags, child); if (rv == 0) { printf("%s: could not reserve resource\n", __func__); return (0); } rman_set_rid(rv, *rid); return (rv); } static int rt305x_gpio_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { return (rman_activate_resource(r)); } static int rt305x_gpio_deactivate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { return (rman_deactivate_resource(r)); } static int rt305x_gpio_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { rman_release_resource(r); return (0); } #endif static device_method_t rt305x_gpio_methods[] = { DEVMETHOD(device_probe, rt305x_gpio_probe), DEVMETHOD(device_attach, rt305x_gpio_attach), DEVMETHOD(device_detach, rt305x_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_pin_max, rt305x_gpio_pin_max), DEVMETHOD(gpio_pin_getname, rt305x_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, rt305x_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, rt305x_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, rt305x_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, rt305x_gpio_pin_get), DEVMETHOD(gpio_pin_set, rt305x_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, rt305x_gpio_pin_toggle), {0, 0}, }; static driver_t rt305x_gpio_driver = { "gpio", rt305x_gpio_methods, sizeof(struct rt305x_gpio_softc), }; static devclass_t rt305x_gpio_devclass; DRIVER_MODULE(rt305x_gpio, obio, rt305x_gpio_driver, rt305x_gpio_devclass, 0, 0); diff --git a/sys/mips/rt305x/rt305x_gpiovar.h b/sys/mips/rt305x/rt305x_gpiovar.h index ed79e2db90be..85cd6f6429da 100644 --- a/sys/mips/rt305x/rt305x_gpiovar.h +++ b/sys/mips/rt305x/rt305x_gpiovar.h @@ -1,48 +1,48 @@ /*- * Copyright (c) 2010 Aleksandr Rybalko. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _RT305X_GPIOVAR_H_ #define _RT305X_GPIOVAR_H_ struct rt305x_gpio_softc { device_t dev; - struct mtx gpio_mtx; - struct resource *gpio_mem_res; - int gpio_mem_rid; - struct resource *gpio_irq_res; - int gpio_irq_rid; - void *gpio_ih; + struct mtx gpio_mtx; + struct resource *gpio_mem_res; + int gpio_mem_rid; + struct resource *gpio_irq_res; + int gpio_irq_rid; + void *gpio_ih; int gpio_npins; struct gpio_pin gpio_pins[NGPIO]; int reset_gpio; int reset_gpio_last; time_t reset_gpio_ontime; }; #endif /* _RT305X_GPIOVAR_H_ */ diff --git a/sys/modules/ctl/Makefile b/sys/modules/ctl/Makefile index 77dc4e12b382..e97ec38d2b55 100644 --- a/sys/modules/ctl/Makefile +++ b/sys/modules/ctl/Makefile @@ -1,29 +1,32 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../cam/ctl KMOD= ctl SRCS= ctl.c SRCS+= ctl_backend.c SRCS+= ctl_backend_block.c SRCS+= ctl_backend_ramdisk.c SRCS+= ctl_cmd_table.c SRCS+= ctl_frontend.c SRCS+= ctl_frontend_cam_sim.c SRCS+= ctl_frontend_internal.c SRCS+= ctl_frontend_iscsi.c SRCS+= ctl_scsi_all.c SRCS+= ctl_tpc.c SRCS+= ctl_tpc_local.c SRCS+= ctl_error.c SRCS+= ctl_util.c SRCS+= scsi_ctl.c SRCS+= bus_if.h SRCS+= device_if.h SRCS+= vnode_if.h +SRCS+= icl_conn_if.h SRCS+= opt_cam.h #CFLAGS+=-DICL_KERNEL_PROXY +MFILES= kern/bus_if.m kern/device_if.m dev/iscsi/icl_conn_if.m + .include diff --git a/sys/modules/iscsi/Makefile b/sys/modules/iscsi/Makefile index 2bff545ef2ff..74a971c92aae 100644 --- a/sys/modules/iscsi/Makefile +++ b/sys/modules/iscsi/Makefile @@ -1,15 +1,20 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../dev/iscsi/ KMOD= iscsi SRCS= iscsi.c SRCS+= icl.c SRCS+= icl_proxy.c +SRCS+= icl_soft.c SRCS+= opt_cam.h SRCS+= bus_if.h SRCS+= device_if.h +SRCS+= icl_conn_if.c +SRCS+= icl_conn_if.h #CFLAGS+=-DICL_KERNEL_PROXY +MFILES= kern/bus_if.m kern/device_if.m dev/iscsi/icl_conn_if.m + .include diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 9b6c695d37fb..64b99fce0918 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -1,1020 +1,1022 @@ /*- * Copyright (c) 1986, 1989, 1991, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)proc.h 8.15 (Berkeley) 5/19/95 * $FreeBSD$ */ #ifndef _SYS_PROC_H_ #define _SYS_PROC_H_ #include /* For struct callout. */ #include /* For struct klist. */ #include #ifndef _KERNEL #include #endif #include #include #include #include #include #include #include /* XXX. */ #include #include #include #include #include #ifndef _KERNEL #include /* For structs itimerval, timeval. */ #else #include #endif #include #include #include /* Machine-dependent proc substruct. */ /* * One structure allocated per session. * * List of locks * (m) locked by s_mtx mtx * (e) locked by proctree_lock sx * (c) const until freeing */ struct session { u_int s_count; /* Ref cnt; pgrps in session - atomic. */ struct proc *s_leader; /* (m + e) Session leader. */ struct vnode *s_ttyvp; /* (m) Vnode of controlling tty. */ struct cdev_priv *s_ttydp; /* (m) Device of controlling tty. */ struct tty *s_ttyp; /* (e) Controlling tty. */ pid_t s_sid; /* (c) Session ID. */ /* (m) Setlogin() name: */ char s_login[roundup(MAXLOGNAME, sizeof(long))]; struct mtx s_mtx; /* Mutex to protect members. */ }; /* * One structure allocated per process group. * * List of locks * (m) locked by pg_mtx mtx * (e) locked by proctree_lock sx * (c) const until freeing */ struct pgrp { LIST_ENTRY(pgrp) pg_hash; /* (e) Hash chain. */ LIST_HEAD(, proc) pg_members; /* (m + e) Pointer to pgrp members. */ struct session *pg_session; /* (c) Pointer to session. */ struct sigiolst pg_sigiolst; /* (m) List of sigio sources. */ pid_t pg_id; /* (c) Process group id. */ int pg_jobc; /* (m) Job control process count. */ struct mtx pg_mtx; /* Mutex to protect members */ }; /* * pargs, used to hold a copy of the command line, if it had a sane length. */ struct pargs { u_int ar_ref; /* Reference count. */ u_int ar_length; /* Length. */ u_char ar_args[1]; /* Arguments. */ }; /*- * Description of a process. * * This structure contains the information needed to manage a thread of * control, known in UN*X as a process; it has references to substructures * containing descriptions of things that the process uses, but may share * with related processes. The process structure and the substructures * are always addressable except for those marked "(CPU)" below, * which might be addressable only on a processor on which the process * is running. * * Below is a key of locks used to protect each member of struct proc. The * lock is indicated by a reference to a specific character in parens in the * associated comment. * * - not yet protected * a - only touched by curproc or parent during fork/wait * b - created at fork, never changes * (exception aiods switch vmspaces, but they are also * marked 'P_SYSTEM' so hopefully it will be left alone) * c - locked by proc mtx * d - locked by allproc_lock lock * e - locked by proctree_lock lock * f - session mtx * g - process group mtx * h - callout_lock mtx * i - by curproc or the master session mtx * j - locked by proc slock * k - only accessed by curthread * k*- only accessed by curthread and from an interrupt * l - the attaching proc or attaching proc parent * m - Giant * n - not locked, lazy * o - ktrace lock * q - td_contested lock * r - p_peers lock * t - thread lock * u - process stat lock * w - process timer lock * x - created at fork, only changes during single threading in exec * y - created at first aio, doesn't change until exit or exec at which * point we are single-threaded and only curthread changes it * z - zombie threads lock * * If the locking key specifies two identifiers (for example, p_pptr) then * either lock is sufficient for read access, but both locks must be held * for write access. */ struct cpuset; struct kaioinfo; struct kaudit_record; struct kdtrace_proc; struct kdtrace_thread; struct mqueue_notifier; struct nlminfo; struct p_sched; struct proc; struct procdesc; struct racct; struct sbuf; struct sleepqueue; struct td_sched; struct thread; struct trapframe; struct turnstile; /* * XXX: Does this belong in resource.h or resourcevar.h instead? * Resource usage extension. The times in rusage structs in the kernel are * never up to date. The actual times are kept as runtimes and tick counts * (with control info in the "previous" times), and are converted when * userland asks for rusage info. Backwards compatibility prevents putting * this directly in the user-visible rusage struct. * * Locking for p_rux: (cu) means (u) for p_rux and (c) for p_crux. * Locking for td_rux: (t) for all fields. */ struct rusage_ext { uint64_t rux_runtime; /* (cu) Real time. */ uint64_t rux_uticks; /* (cu) Statclock hits in user mode. */ uint64_t rux_sticks; /* (cu) Statclock hits in sys mode. */ uint64_t rux_iticks; /* (cu) Statclock hits in intr mode. */ uint64_t rux_uu; /* (c) Previous user time in usec. */ uint64_t rux_su; /* (c) Previous sys time in usec. */ uint64_t rux_tu; /* (c) Previous total time in usec. */ }; /* * Kernel runnable context (thread). * This is what is put to sleep and reactivated. * Thread context. Processes may have multiple threads. */ struct thread { struct mtx *volatile td_lock; /* replaces sched lock */ struct proc *td_proc; /* (*) Associated process. */ TAILQ_ENTRY(thread) td_plist; /* (*) All threads in this proc. */ TAILQ_ENTRY(thread) td_runq; /* (t) Run queue. */ TAILQ_ENTRY(thread) td_slpq; /* (t) Sleep queue. */ TAILQ_ENTRY(thread) td_lockq; /* (t) Lock queue. */ LIST_ENTRY(thread) td_hash; /* (d) Hash chain. */ struct cpuset *td_cpuset; /* (t) CPU affinity mask. */ struct seltd *td_sel; /* Select queue/channel. */ struct sleepqueue *td_sleepqueue; /* (k) Associated sleep queue. */ struct turnstile *td_turnstile; /* (k) Associated turnstile. */ struct rl_q_entry *td_rlqe; /* (k) Associated range lock entry. */ struct umtx_q *td_umtxq; /* (c?) Link for when we're blocked. */ lwpid_t td_tid; /* (b) Thread ID. */ sigqueue_t td_sigqueue; /* (c) Sigs arrived, not delivered. */ #define td_siglist td_sigqueue.sq_signals u_char td_lend_user_pri; /* (t) Lend user pri. */ /* Cleared during fork1() */ #define td_startzero td_flags int td_flags; /* (t) TDF_* flags. */ int td_inhibitors; /* (t) Why can not run. */ int td_pflags; /* (k) Private thread (TDP_*) flags. */ int td_dupfd; /* (k) Ret value from fdopen. XXX */ int td_sqqueue; /* (t) Sleepqueue queue blocked on. */ void *td_wchan; /* (t) Sleep address. */ const char *td_wmesg; /* (t) Reason for sleep. */ int td_lastcpu; /* (t) Last cpu we were on. */ int td_oncpu; /* (t) Which cpu we are on. */ volatile u_char td_owepreempt; /* (k*) Preempt on last critical_exit */ u_char td_tsqueue; /* (t) Turnstile queue blocked on. */ short td_locks; /* (k) Count of non-spin locks. */ short td_rw_rlocks; /* (k) Count of rwlock read locks. */ short td_lk_slocks; /* (k) Count of lockmgr shared locks. */ short td_stopsched; /* (k) Scheduler stopped. */ struct turnstile *td_blocked; /* (t) Lock thread is blocked on. */ const char *td_lockname; /* (t) Name of lock blocked on. */ LIST_HEAD(, turnstile) td_contested; /* (q) Contested locks. */ struct lock_list_entry *td_sleeplocks; /* (k) Held sleep locks. */ int td_intr_nesting_level; /* (k) Interrupt recursion. */ int td_pinned; /* (k) Temporary cpu pin count. */ struct ucred *td_ucred; /* (k) Reference to credentials. */ u_int td_estcpu; /* (t) estimated cpu utilization */ int td_slptick; /* (t) Time at sleep. */ int td_blktick; /* (t) Time spent blocked. */ int td_swvoltick; /* (t) Time at last SW_VOL switch. */ u_int td_cow; /* (*) Number of copy-on-write faults */ struct rusage td_ru; /* (t) rusage information. */ struct rusage_ext td_rux; /* (t) Internal rusage information. */ uint64_t td_incruntime; /* (t) Cpu ticks to transfer to proc. */ uint64_t td_runtime; /* (t) How many cpu ticks we've run. */ u_int td_pticks; /* (t) Statclock hits for profiling */ u_int td_sticks; /* (t) Statclock hits in system mode. */ u_int td_iticks; /* (t) Statclock hits in intr mode. */ u_int td_uticks; /* (t) Statclock hits in user mode. */ int td_intrval; /* (t) Return value for sleepq. */ sigset_t td_oldsigmask; /* (k) Saved mask from pre sigpause. */ volatile u_int td_generation; /* (k) For detection of preemption */ stack_t td_sigstk; /* (k) Stack ptr and on-stack flag. */ int td_xsig; /* (c) Signal for ptrace */ u_long td_profil_addr; /* (k) Temporary addr until AST. */ u_int td_profil_ticks; /* (k) Temporary ticks until AST. */ char td_name[MAXCOMLEN + 1]; /* (*) Thread name. */ struct file *td_fpop; /* (k) file referencing cdev under op */ int td_dbgflags; /* (c) Userland debugger flags */ struct ksiginfo td_dbgksi; /* (c) ksi reflected to debugger. */ int td_ng_outbound; /* (k) Thread entered ng from above. */ struct osd td_osd; /* (k) Object specific data. */ struct vm_map_entry *td_map_def_user; /* (k) Deferred entries. */ pid_t td_dbg_forked; /* (c) Child pid for debugger. */ u_int td_vp_reserv; /* (k) Count of reserved vnodes. */ int td_no_sleeping; /* (k) Sleeping disabled count. */ int td_dom_rr_idx; /* (k) RR Numa domain selection. */ #define td_endzero td_sigmask /* Copied during fork1() or create_thread(). */ #define td_startcopy td_endzero sigset_t td_sigmask; /* (c) Current signal mask. */ u_char td_rqindex; /* (t) Run queue index. */ u_char td_base_pri; /* (t) Thread base kernel priority. */ u_char td_priority; /* (t) Thread active priority. */ u_char td_pri_class; /* (t) Scheduling class. */ u_char td_user_pri; /* (t) User pri from estcpu and nice. */ u_char td_base_user_pri; /* (t) Base user pri */ #define td_endcopy td_pcb /* * Fields that must be manually set in fork1() or create_thread() * or already have been set in the allocator, constructor, etc. */ struct pcb *td_pcb; /* (k) Kernel VA of pcb and kstack. */ enum { TDS_INACTIVE = 0x0, TDS_INHIBITED, TDS_CAN_RUN, TDS_RUNQ, TDS_RUNNING } td_state; /* (t) thread state */ union { register_t tdu_retval[2]; off_t tdu_off; } td_uretoff; /* (k) Syscall aux returns. */ #define td_retval td_uretoff.tdu_retval struct callout td_slpcallout; /* (h) Callout for sleep. */ struct trapframe *td_frame; /* (k) */ struct vm_object *td_kstack_obj;/* (a) Kstack object. */ vm_offset_t td_kstack; /* (a) Kernel VA of kstack. */ int td_kstack_pages; /* (a) Size of the kstack. */ volatile u_int td_critnest; /* (k*) Critical section nest level. */ struct mdthread td_md; /* (k) Any machine-dependent fields. */ struct td_sched *td_sched; /* (*) Scheduler-specific data. */ struct kaudit_record *td_ar; /* (k) Active audit record, if any. */ struct lpohead td_lprof[2]; /* (a) lock profiling objects. */ struct kdtrace_thread *td_dtrace; /* (*) DTrace-specific data. */ int td_errno; /* Error returned by last syscall. */ struct vnet *td_vnet; /* (k) Effective vnet. */ const char *td_vnet_lpush; /* (k) Debugging vnet push / pop. */ struct trapframe *td_intr_frame;/* (k) Frame of the current irq */ struct proc *td_rfppwait_p; /* (k) The vforked child */ struct vm_page **td_ma; /* (k) uio pages held */ int td_ma_cnt; /* (k) size of *td_ma */ }; struct mtx *thread_lock_block(struct thread *); void thread_lock_unblock(struct thread *, struct mtx *); void thread_lock_set(struct thread *, struct mtx *); #define THREAD_LOCK_ASSERT(td, type) \ do { \ struct mtx *__m = (td)->td_lock; \ if (__m != &blocked_lock) \ mtx_assert(__m, (type)); \ } while (0) #ifdef INVARIANTS #define THREAD_LOCKPTR_ASSERT(td, lock) \ do { \ struct mtx *__m = (td)->td_lock; \ KASSERT((__m == &blocked_lock || __m == (lock)), \ ("Thread %p lock %p does not match %p", td, __m, (lock))); \ } while (0) #else #define THREAD_LOCKPTR_ASSERT(td, lock) #endif /* * Flags kept in td_flags: * To change these you MUST have the scheduler lock. */ #define TDF_BORROWING 0x00000001 /* Thread is borrowing pri from another. */ #define TDF_INPANIC 0x00000002 /* Caused a panic, let it drive crashdump. */ #define TDF_INMEM 0x00000004 /* Thread's stack is in memory. */ #define TDF_SINTR 0x00000008 /* Sleep is interruptible. */ #define TDF_TIMEOUT 0x00000010 /* Timing out during sleep. */ #define TDF_IDLETD 0x00000020 /* This is a per-CPU idle thread. */ #define TDF_CANSWAP 0x00000040 /* Thread can be swapped. */ #define TDF_SLEEPABORT 0x00000080 /* sleepq_abort was called. */ #define TDF_KTH_SUSP 0x00000100 /* kthread is suspended */ #define TDF_ALLPROCSUSP 0x00000200 /* suspended by SINGLE_ALLPROC */ #define TDF_BOUNDARY 0x00000400 /* Thread suspended at user boundary */ #define TDF_ASTPENDING 0x00000800 /* Thread has some asynchronous events. */ #define TDF_TIMOFAIL 0x00001000 /* Timeout from sleep after we were awake. */ #define TDF_SBDRY 0x00002000 /* Stop only on usermode boundary. */ #define TDF_UPIBLOCKED 0x00004000 /* Thread blocked on user PI mutex. */ #define TDF_NEEDSUSPCHK 0x00008000 /* Thread may need to suspend. */ #define TDF_NEEDRESCHED 0x00010000 /* Thread needs to yield. */ #define TDF_NEEDSIGCHK 0x00020000 /* Thread may need signal delivery. */ #define TDF_NOLOAD 0x00040000 /* Ignore during load avg calculations. */ #define TDF_UNUSED19 0x00080000 /* --available-- */ #define TDF_THRWAKEUP 0x00100000 /* Libthr thread must not suspend itself. */ #define TDF_UNUSED21 0x00200000 /* --available-- */ #define TDF_SWAPINREQ 0x00400000 /* Swapin request due to wakeup. */ #define TDF_UNUSED23 0x00800000 /* --available-- */ #define TDF_SCHED0 0x01000000 /* Reserved for scheduler private use */ #define TDF_SCHED1 0x02000000 /* Reserved for scheduler private use */ #define TDF_SCHED2 0x04000000 /* Reserved for scheduler private use */ #define TDF_SCHED3 0x08000000 /* Reserved for scheduler private use */ #define TDF_ALRMPEND 0x10000000 /* Pending SIGVTALRM needs to be posted. */ #define TDF_PROFPEND 0x20000000 /* Pending SIGPROF needs to be posted. */ #define TDF_MACPEND 0x40000000 /* AST-based MAC event pending. */ /* Userland debug flags */ #define TDB_SUSPEND 0x00000001 /* Thread is suspended by debugger */ #define TDB_XSIG 0x00000002 /* Thread is exchanging signal under trace */ #define TDB_USERWR 0x00000004 /* Debugger modified memory or registers */ #define TDB_SCE 0x00000008 /* Thread performs syscall enter */ #define TDB_SCX 0x00000010 /* Thread performs syscall exit */ #define TDB_EXEC 0x00000020 /* TDB_SCX from exec(2) family */ #define TDB_FORK 0x00000040 /* TDB_SCX from fork(2) that created new process */ #define TDB_STOPATFORK 0x00000080 /* Stop at the return from fork (child only) */ #define TDB_CHILD 0x00000100 /* New child indicator for ptrace() */ /* * "Private" flags kept in td_pflags: * These are only written by curthread and thus need no locking. */ #define TDP_OLDMASK 0x00000001 /* Need to restore mask after suspend. */ #define TDP_INKTR 0x00000002 /* Thread is currently in KTR code. */ #define TDP_INKTRACE 0x00000004 /* Thread is currently in KTRACE code. */ #define TDP_BUFNEED 0x00000008 /* Do not recurse into the buf flush */ #define TDP_COWINPROGRESS 0x00000010 /* Snapshot copy-on-write in progress. */ #define TDP_ALTSTACK 0x00000020 /* Have alternate signal stack. */ #define TDP_DEADLKTREAT 0x00000040 /* Lock aquisition - deadlock treatment. */ #define TDP_NOFAULTING 0x00000080 /* Do not handle page faults. */ #define TDP_UNUSED9 0x00000100 /* --available-- */ #define TDP_OWEUPC 0x00000200 /* Call addupc() at next AST. */ #define TDP_ITHREAD 0x00000400 /* Thread is an interrupt thread. */ #define TDP_SYNCIO 0x00000800 /* Local override, disable async i/o. */ #define TDP_SCHED1 0x00001000 /* Reserved for scheduler private use */ #define TDP_SCHED2 0x00002000 /* Reserved for scheduler private use */ #define TDP_SCHED3 0x00004000 /* Reserved for scheduler private use */ #define TDP_SCHED4 0x00008000 /* Reserved for scheduler private use */ #define TDP_GEOM 0x00010000 /* Settle GEOM before finishing syscall */ #define TDP_SOFTDEP 0x00020000 /* Stuck processing softdep worklist */ #define TDP_NORUNNINGBUF 0x00040000 /* Ignore runningbufspace check */ #define TDP_WAKEUP 0x00080000 /* Don't sleep in umtx cond_wait */ #define TDP_INBDFLUSH 0x00100000 /* Already in BO_BDFLUSH, do not recurse */ #define TDP_KTHREAD 0x00200000 /* This is an official kernel thread */ #define TDP_CALLCHAIN 0x00400000 /* Capture thread's callchain */ #define TDP_IGNSUSP 0x00800000 /* Permission to ignore the MNTK_SUSPEND* */ #define TDP_AUDITREC 0x01000000 /* Audit record pending on thread */ #define TDP_RFPPWAIT 0x02000000 /* Handle RFPPWAIT on syscall exit */ #define TDP_RESETSPUR 0x04000000 /* Reset spurious page fault history. */ #define TDP_NERRNO 0x08000000 /* Last errno is already in td_errno */ #define TDP_UIOHELD 0x10000000 /* Current uio has pages held in td_ma */ #define TDP_UNUSED29 0x20000000 /* --available-- */ #define TDP_EXECVMSPC 0x40000000 /* Execve destroyed old vmspace */ /* * Reasons that the current thread can not be run yet. * More than one may apply. */ #define TDI_SUSPENDED 0x0001 /* On suspension queue. */ #define TDI_SLEEPING 0x0002 /* Actually asleep! (tricky). */ #define TDI_SWAPPED 0x0004 /* Stack not in mem. Bad juju if run. */ #define TDI_LOCK 0x0008 /* Stopped on a lock. */ #define TDI_IWAIT 0x0010 /* Awaiting interrupt. */ #define TD_IS_SLEEPING(td) ((td)->td_inhibitors & TDI_SLEEPING) #define TD_ON_SLEEPQ(td) ((td)->td_wchan != NULL) #define TD_IS_SUSPENDED(td) ((td)->td_inhibitors & TDI_SUSPENDED) #define TD_IS_SWAPPED(td) ((td)->td_inhibitors & TDI_SWAPPED) #define TD_ON_LOCK(td) ((td)->td_inhibitors & TDI_LOCK) #define TD_AWAITING_INTR(td) ((td)->td_inhibitors & TDI_IWAIT) #define TD_IS_RUNNING(td) ((td)->td_state == TDS_RUNNING) #define TD_ON_RUNQ(td) ((td)->td_state == TDS_RUNQ) #define TD_CAN_RUN(td) ((td)->td_state == TDS_CAN_RUN) #define TD_IS_INHIBITED(td) ((td)->td_state == TDS_INHIBITED) #define TD_ON_UPILOCK(td) ((td)->td_flags & TDF_UPIBLOCKED) #define TD_IS_IDLETHREAD(td) ((td)->td_flags & TDF_IDLETD) #define TD_SET_INHIB(td, inhib) do { \ (td)->td_state = TDS_INHIBITED; \ (td)->td_inhibitors |= (inhib); \ } while (0) #define TD_CLR_INHIB(td, inhib) do { \ if (((td)->td_inhibitors & (inhib)) && \ (((td)->td_inhibitors &= ~(inhib)) == 0)) \ (td)->td_state = TDS_CAN_RUN; \ } while (0) #define TD_SET_SLEEPING(td) TD_SET_INHIB((td), TDI_SLEEPING) #define TD_SET_SWAPPED(td) TD_SET_INHIB((td), TDI_SWAPPED) #define TD_SET_LOCK(td) TD_SET_INHIB((td), TDI_LOCK) #define TD_SET_SUSPENDED(td) TD_SET_INHIB((td), TDI_SUSPENDED) #define TD_SET_IWAIT(td) TD_SET_INHIB((td), TDI_IWAIT) #define TD_SET_EXITING(td) TD_SET_INHIB((td), TDI_EXITING) #define TD_CLR_SLEEPING(td) TD_CLR_INHIB((td), TDI_SLEEPING) #define TD_CLR_SWAPPED(td) TD_CLR_INHIB((td), TDI_SWAPPED) #define TD_CLR_LOCK(td) TD_CLR_INHIB((td), TDI_LOCK) #define TD_CLR_SUSPENDED(td) TD_CLR_INHIB((td), TDI_SUSPENDED) #define TD_CLR_IWAIT(td) TD_CLR_INHIB((td), TDI_IWAIT) #define TD_SET_RUNNING(td) (td)->td_state = TDS_RUNNING #define TD_SET_RUNQ(td) (td)->td_state = TDS_RUNQ #define TD_SET_CAN_RUN(td) (td)->td_state = TDS_CAN_RUN /* * Process structure. */ struct proc { LIST_ENTRY(proc) p_list; /* (d) List of all processes. */ TAILQ_HEAD(, thread) p_threads; /* (c) all threads. */ struct mtx p_slock; /* process spin lock */ struct ucred *p_ucred; /* (c) Process owner's identity. */ struct filedesc *p_fd; /* (b) Open files. */ struct filedesc_to_leader *p_fdtol; /* (b) Tracking node */ struct pstats *p_stats; /* (b) Accounting/statistics (CPU). */ struct plimit *p_limit; /* (c) Process limits. */ struct callout p_limco; /* (c) Limit callout handle */ struct sigacts *p_sigacts; /* (x) Signal actions, state (CPU). */ int p_flag; /* (c) P_* flags. */ int p_flag2; /* (c) P2_* flags. */ enum { PRS_NEW = 0, /* In creation */ PRS_NORMAL, /* threads can be run. */ PRS_ZOMBIE } p_state; /* (j/c) Process status. */ pid_t p_pid; /* (b) Process identifier. */ LIST_ENTRY(proc) p_hash; /* (d) Hash chain. */ LIST_ENTRY(proc) p_pglist; /* (g + e) List of processes in pgrp. */ struct proc *p_pptr; /* (c + e) Pointer to parent process. */ LIST_ENTRY(proc) p_sibling; /* (e) List of sibling processes. */ LIST_HEAD(, proc) p_children; /* (e) Pointer to list of children. */ struct proc *p_reaper; /* (e) My reaper. */ LIST_HEAD(, proc) p_reaplist; /* (e) List of my descendants (if I am reaper). */ LIST_ENTRY(proc) p_reapsibling; /* (e) List of siblings - descendants of the same reaper. */ struct mtx p_mtx; /* (n) Lock for this struct. */ struct mtx p_statmtx; /* Lock for the stats */ struct mtx p_itimmtx; /* Lock for the virt/prof timers */ struct mtx p_profmtx; /* Lock for the profiling */ struct ksiginfo *p_ksi; /* Locked by parent proc lock */ sigqueue_t p_sigqueue; /* (c) Sigs not delivered to a td. */ #define p_siglist p_sigqueue.sq_signals /* The following fields are all zeroed upon creation in fork. */ #define p_startzero p_oppid pid_t p_oppid; /* (c + e) Save ppid in ptrace. XXX */ struct vmspace *p_vmspace; /* (b) Address space. */ u_int p_swtick; /* (c) Tick when swapped in or out. */ struct itimerval p_realtimer; /* (c) Alarm timer. */ struct rusage p_ru; /* (a) Exit information. */ struct rusage_ext p_rux; /* (cu) Internal resource usage. */ struct rusage_ext p_crux; /* (c) Internal child resource usage. */ int p_profthreads; /* (c) Num threads in addupc_task. */ volatile int p_exitthreads; /* (j) Number of threads exiting */ int p_traceflag; /* (o) Kernel trace points. */ struct vnode *p_tracevp; /* (c + o) Trace to vnode. */ struct ucred *p_tracecred; /* (o) Credentials to trace with. */ struct vnode *p_textvp; /* (b) Vnode of executable. */ u_int p_lock; /* (c) Proclock (prevent swap) count. */ struct sigiolst p_sigiolst; /* (c) List of sigio sources. */ int p_sigparent; /* (c) Signal to parent on exit. */ int p_sig; /* (n) For core dump/debugger XXX. */ u_long p_code; /* (n) For core dump/debugger XXX. */ u_int p_stops; /* (c) Stop event bitmask. */ u_int p_stype; /* (c) Stop event type. */ char p_step; /* (c) Process is stopped. */ u_char p_pfsflags; /* (c) Procfs flags. */ struct nlminfo *p_nlminfo; /* (?) Only used by/for lockd. */ struct kaioinfo *p_aioinfo; /* (y) ASYNC I/O info. */ struct thread *p_singlethread;/* (c + j) If single threading this is it */ int p_suspcount; /* (j) Num threads in suspended mode. */ struct thread *p_xthread; /* (c) Trap thread */ int p_boundary_count;/* (j) Num threads at user boundary */ int p_pendingcnt; /* how many signals are pending */ struct itimers *p_itimers; /* (c) POSIX interval timers. */ struct procdesc *p_procdesc; /* (e) Process descriptor, if any. */ u_int p_treeflag; /* (e) P_TREE flags */ /* End area that is zeroed on creation. */ #define p_endzero p_magic /* The following fields are all copied upon creation in fork. */ #define p_startcopy p_endzero u_int p_magic; /* (b) Magic number. */ int p_osrel; /* (x) osreldate for the binary (from ELF note, if any) */ char p_comm[MAXCOMLEN + 1]; /* (b) Process name. */ struct pgrp *p_pgrp; /* (c + e) Pointer to process group. */ struct sysentvec *p_sysent; /* (b) Syscall dispatch info. */ struct pargs *p_args; /* (c) Process arguments. */ rlim_t p_cpulimit; /* (c) Current CPU limit in seconds. */ signed char p_nice; /* (c) Process "nice" value. */ int p_fibnum; /* in this routing domain XXX MRT */ pid_t p_reapsubtree; /* (e) Pid of the direct child of the reaper which spawned our subtree. */ /* End area that is copied on creation. */ #define p_endcopy p_xstat u_short p_xstat; /* (c) Exit status; also stop sig. */ struct knlist p_klist; /* (c) Knotes attached to this proc. */ int p_numthreads; /* (c) Number of threads. */ struct mdproc p_md; /* Any machine-dependent fields. */ struct callout p_itcallout; /* (h + c) Interval timer callout. */ u_short p_acflag; /* (c) Accounting flags. */ struct proc *p_peers; /* (r) */ struct proc *p_leader; /* (b) */ void *p_emuldata; /* (c) Emulator state data. */ struct label *p_label; /* (*) Proc (not subject) MAC label. */ struct p_sched *p_sched; /* (*) Scheduler-specific data. */ STAILQ_HEAD(, ktr_request) p_ktr; /* (o) KTR event queue. */ LIST_HEAD(, mqueue_notifier) p_mqnotifier; /* (c) mqueue notifiers.*/ struct kdtrace_proc *p_dtrace; /* (*) DTrace-specific data. */ struct cv p_pwait; /* (*) wait cv for exit/exec. */ struct cv p_dbgwait; /* (*) wait cv for debugger attach after fork. */ uint64_t p_prev_runtime; /* (c) Resource usage accounting. */ struct racct *p_racct; /* (b) Resource accounting. */ u_char p_throttled; /* (c) Flag for racct pcpu throttling */ /* * An orphan is the child that has beed re-parented to the * debugger as a result of attaching to it. Need to keep * track of them for parent to be able to collect the exit * status of what used to be children. */ LIST_ENTRY(proc) p_orphan; /* (e) List of orphan processes. */ LIST_HEAD(, proc) p_orphans; /* (e) Pointer to list of orphans. */ }; #define p_session p_pgrp->pg_session #define p_pgid p_pgrp->pg_id #define NOCPU (-1) /* For when we aren't on a CPU. */ #define NOCPU_OLD (255) #define MAXCPU_OLD (254) #define PROC_SLOCK(p) mtx_lock_spin(&(p)->p_slock) #define PROC_SUNLOCK(p) mtx_unlock_spin(&(p)->p_slock) #define PROC_SLOCK_ASSERT(p, type) mtx_assert(&(p)->p_slock, (type)) #define PROC_STATLOCK(p) mtx_lock_spin(&(p)->p_statmtx) #define PROC_STATUNLOCK(p) mtx_unlock_spin(&(p)->p_statmtx) #define PROC_STATLOCK_ASSERT(p, type) mtx_assert(&(p)->p_statmtx, (type)) #define PROC_ITIMLOCK(p) mtx_lock_spin(&(p)->p_itimmtx) #define PROC_ITIMUNLOCK(p) mtx_unlock_spin(&(p)->p_itimmtx) #define PROC_ITIMLOCK_ASSERT(p, type) mtx_assert(&(p)->p_itimmtx, (type)) #define PROC_PROFLOCK(p) mtx_lock_spin(&(p)->p_profmtx) #define PROC_PROFUNLOCK(p) mtx_unlock_spin(&(p)->p_profmtx) #define PROC_PROFLOCK_ASSERT(p, type) mtx_assert(&(p)->p_profmtx, (type)) /* These flags are kept in p_flag. */ #define P_ADVLOCK 0x00001 /* Process may hold a POSIX advisory lock. */ #define P_CONTROLT 0x00002 /* Has a controlling terminal. */ #define P_KTHREAD 0x00004 /* Kernel thread (*). */ #define P_FOLLOWFORK 0x00008 /* Attach parent debugger to children. */ #define P_PPWAIT 0x00010 /* Parent is waiting for child to exec/exit. */ #define P_PROFIL 0x00020 /* Has started profiling. */ #define P_STOPPROF 0x00040 /* Has thread requesting to stop profiling. */ #define P_HADTHREADS 0x00080 /* Has had threads (no cleanup shortcuts) */ #define P_SUGID 0x00100 /* Had set id privileges since last exec. */ #define P_SYSTEM 0x00200 /* System proc: no sigs, stats or swapping. */ #define P_SINGLE_EXIT 0x00400 /* Threads suspending should exit, not wait. */ #define P_TRACED 0x00800 /* Debugged process being traced. */ #define P_WAITED 0x01000 /* Someone is waiting for us. */ #define P_WEXIT 0x02000 /* Working on exiting. */ #define P_EXEC 0x04000 /* Process called exec. */ #define P_WKILLED 0x08000 /* Killed, go to kernel/user boundary ASAP. */ #define P_CONTINUED 0x10000 /* Proc has continued from a stopped state. */ #define P_STOPPED_SIG 0x20000 /* Stopped due to SIGSTOP/SIGTSTP. */ #define P_STOPPED_TRACE 0x40000 /* Stopped because of tracing. */ #define P_STOPPED_SINGLE 0x80000 /* Only 1 thread can continue (not to user). */ #define P_PROTECTED 0x100000 /* Do not kill on memory overcommit. */ #define P_SIGEVENT 0x200000 /* Process pending signals changed. */ #define P_SINGLE_BOUNDARY 0x400000 /* Threads should suspend at user boundary. */ #define P_HWPMC 0x800000 /* Process is using HWPMCs */ #define P_JAILED 0x1000000 /* Process is in jail. */ #define P_TOTAL_STOP 0x2000000 /* Stopped in proc_stop_total. */ #define P_INEXEC 0x4000000 /* Process is in execve(). */ #define P_STATCHILD 0x8000000 /* Child process stopped or exited. */ #define P_INMEM 0x10000000 /* Loaded into memory. */ #define P_SWAPPINGOUT 0x20000000 /* Process is being swapped out. */ #define P_SWAPPINGIN 0x40000000 /* Process is being swapped in. */ #define P_PPTRACE 0x80000000 /* PT_TRACEME by vforked child. */ #define P_STOPPED (P_STOPPED_SIG|P_STOPPED_SINGLE|P_STOPPED_TRACE) #define P_SHOULDSTOP(p) ((p)->p_flag & P_STOPPED) #define P_KILLED(p) ((p)->p_flag & P_WKILLED) /* These flags are kept in p_flag2. */ #define P2_INHERIT_PROTECTED 0x00000001 /* New children get P_PROTECTED. */ #define P2_NOTRACE 0x00000002 /* No ptrace(2) attach or coredumps. */ #define P2_NOTRACE_EXEC 0x00000004 /* Keep P2_NOPTRACE on exec(2). */ /* Flags protected by proctree_lock, kept in p_treeflags. */ #define P_TREE_ORPHANED 0x00000001 /* Reparented, on orphan list */ #define P_TREE_FIRST_ORPHAN 0x00000002 /* First element of orphan list */ #define P_TREE_REAPER 0x00000004 /* Reaper of subtree */ /* * These were process status values (p_stat), now they are only used in * legacy conversion code. */ #define SIDL 1 /* Process being created by fork. */ #define SRUN 2 /* Currently runnable. */ #define SSLEEP 3 /* Sleeping on an address. */ #define SSTOP 4 /* Process debugging or suspension. */ #define SZOMB 5 /* Awaiting collection by parent. */ #define SWAIT 6 /* Waiting for interrupt. */ #define SLOCK 7 /* Blocked on a lock. */ #define P_MAGIC 0xbeefface #ifdef _KERNEL /* Types and flags for mi_switch(). */ #define SW_TYPE_MASK 0xff /* First 8 bits are switch type */ #define SWT_NONE 0 /* Unspecified switch. */ #define SWT_PREEMPT 1 /* Switching due to preemption. */ #define SWT_OWEPREEMPT 2 /* Switching due to opepreempt. */ #define SWT_TURNSTILE 3 /* Turnstile contention. */ #define SWT_SLEEPQ 4 /* Sleepq wait. */ #define SWT_SLEEPQTIMO 5 /* Sleepq timeout wait. */ #define SWT_RELINQUISH 6 /* yield call. */ #define SWT_NEEDRESCHED 7 /* NEEDRESCHED was set. */ #define SWT_IDLE 8 /* Switching from the idle thread. */ #define SWT_IWAIT 9 /* Waiting for interrupts. */ #define SWT_SUSPEND 10 /* Thread suspended. */ #define SWT_REMOTEPREEMPT 11 /* Remote processor preempted. */ #define SWT_REMOTEWAKEIDLE 12 /* Remote processor preempted idle. */ #define SWT_COUNT 13 /* Number of switch types. */ /* Flags */ #define SW_VOL 0x0100 /* Voluntary switch. */ #define SW_INVOL 0x0200 /* Involuntary switch. */ #define SW_PREEMPT 0x0400 /* The invol switch is a preemption */ /* How values for thread_single(). */ #define SINGLE_NO_EXIT 0 #define SINGLE_EXIT 1 #define SINGLE_BOUNDARY 2 #define SINGLE_ALLPROC 3 #ifdef MALLOC_DECLARE MALLOC_DECLARE(M_PARGS); MALLOC_DECLARE(M_PGRP); MALLOC_DECLARE(M_SESSION); MALLOC_DECLARE(M_SUBPROC); #endif #define FOREACH_PROC_IN_SYSTEM(p) \ LIST_FOREACH((p), &allproc, p_list) #define FOREACH_THREAD_IN_PROC(p, td) \ TAILQ_FOREACH((td), &(p)->p_threads, td_plist) #define FIRST_THREAD_IN_PROC(p) TAILQ_FIRST(&(p)->p_threads) /* * We use process IDs <= pid_max <= PID_MAX; PID_MAX + 1 must also fit * in a pid_t, as it is used to represent "no process group". */ #define PID_MAX 99999 #define NO_PID 100000 extern pid_t pid_max; #define SESS_LEADER(p) ((p)->p_session->s_leader == (p)) #define STOPEVENT(p, e, v) do { \ + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, \ + "checking stopevent %d", (e)); \ if ((p)->p_stops & (e)) { \ PROC_LOCK(p); \ stopevent((p), (e), (v)); \ PROC_UNLOCK(p); \ } \ } while (0) #define _STOPEVENT(p, e, v) do { \ PROC_LOCK_ASSERT(p, MA_OWNED); \ WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &p->p_mtx.lock_object, \ "checking stopevent %d", (e)); \ if ((p)->p_stops & (e)) \ stopevent((p), (e), (v)); \ } while (0) /* Lock and unlock a process. */ #define PROC_LOCK(p) mtx_lock(&(p)->p_mtx) #define PROC_TRYLOCK(p) mtx_trylock(&(p)->p_mtx) #define PROC_UNLOCK(p) mtx_unlock(&(p)->p_mtx) #define PROC_LOCKED(p) mtx_owned(&(p)->p_mtx) #define PROC_LOCK_ASSERT(p, type) mtx_assert(&(p)->p_mtx, (type)) /* Lock and unlock a process group. */ #define PGRP_LOCK(pg) mtx_lock(&(pg)->pg_mtx) #define PGRP_UNLOCK(pg) mtx_unlock(&(pg)->pg_mtx) #define PGRP_LOCKED(pg) mtx_owned(&(pg)->pg_mtx) #define PGRP_LOCK_ASSERT(pg, type) mtx_assert(&(pg)->pg_mtx, (type)) #define PGRP_LOCK_PGSIGNAL(pg) do { \ if ((pg) != NULL) \ PGRP_LOCK(pg); \ } while (0) #define PGRP_UNLOCK_PGSIGNAL(pg) do { \ if ((pg) != NULL) \ PGRP_UNLOCK(pg); \ } while (0) /* Lock and unlock a session. */ #define SESS_LOCK(s) mtx_lock(&(s)->s_mtx) #define SESS_UNLOCK(s) mtx_unlock(&(s)->s_mtx) #define SESS_LOCKED(s) mtx_owned(&(s)->s_mtx) #define SESS_LOCK_ASSERT(s, type) mtx_assert(&(s)->s_mtx, (type)) /* Hold process U-area in memory, normally for ptrace/procfs work. */ #define PHOLD(p) do { \ PROC_LOCK(p); \ _PHOLD(p); \ PROC_UNLOCK(p); \ } while (0) #define _PHOLD(p) do { \ PROC_LOCK_ASSERT((p), MA_OWNED); \ KASSERT(!((p)->p_flag & P_WEXIT) || (p) == curproc, \ ("PHOLD of exiting process")); \ (p)->p_lock++; \ if (((p)->p_flag & P_INMEM) == 0) \ faultin((p)); \ } while (0) #define PROC_ASSERT_HELD(p) do { \ KASSERT((p)->p_lock > 0, ("process not held")); \ } while (0) #define PRELE(p) do { \ PROC_LOCK((p)); \ _PRELE((p)); \ PROC_UNLOCK((p)); \ } while (0) #define _PRELE(p) do { \ PROC_LOCK_ASSERT((p), MA_OWNED); \ PROC_ASSERT_HELD(p); \ (--(p)->p_lock); \ if (((p)->p_flag & P_WEXIT) && (p)->p_lock == 0) \ wakeup(&(p)->p_lock); \ } while (0) #define PROC_ASSERT_NOT_HELD(p) do { \ KASSERT((p)->p_lock == 0, ("process held")); \ } while (0) /* Check whether a thread is safe to be swapped out. */ #define thread_safetoswapout(td) ((td)->td_flags & TDF_CANSWAP) /* Control whether or not it is safe for curthread to sleep. */ #define THREAD_NO_SLEEPING() ((curthread)->td_no_sleeping++) #define THREAD_SLEEPING_OK() ((curthread)->td_no_sleeping--) #define THREAD_CAN_SLEEP() ((curthread)->td_no_sleeping == 0) #define PIDHASH(pid) (&pidhashtbl[(pid) & pidhash]) extern LIST_HEAD(pidhashhead, proc) *pidhashtbl; extern u_long pidhash; #define TIDHASH(tid) (&tidhashtbl[(tid) & tidhash]) extern LIST_HEAD(tidhashhead, thread) *tidhashtbl; extern u_long tidhash; extern struct rwlock tidhash_lock; #define PGRPHASH(pgid) (&pgrphashtbl[(pgid) & pgrphash]) extern LIST_HEAD(pgrphashhead, pgrp) *pgrphashtbl; extern u_long pgrphash; extern struct sx allproc_lock; extern int allproc_gen; extern struct sx proctree_lock; extern struct mtx ppeers_lock; extern struct proc proc0; /* Process slot for swapper. */ extern struct thread thread0; /* Primary thread in proc0. */ extern struct vmspace vmspace0; /* VM space for proc0. */ extern int hogticks; /* Limit on kernel cpu hogs. */ extern int lastpid; extern int nprocs, maxproc; /* Current and max number of procs. */ extern int maxprocperuid; /* Max procs per uid. */ extern u_long ps_arg_cache_limit; LIST_HEAD(proclist, proc); TAILQ_HEAD(procqueue, proc); TAILQ_HEAD(threadqueue, thread); extern struct proclist allproc; /* List of all processes. */ extern struct proclist zombproc; /* List of zombie processes. */ extern struct proc *initproc, *pageproc; /* Process slots for init, pager. */ extern struct uma_zone *proc_zone; struct proc *pfind(pid_t); /* Find process by id. */ struct proc *pfind_locked(pid_t pid); struct pgrp *pgfind(pid_t); /* Find process group by id. */ struct proc *zpfind(pid_t); /* Find zombie process by id. */ /* * pget() flags. */ #define PGET_HOLD 0x00001 /* Hold the process. */ #define PGET_CANSEE 0x00002 /* Check against p_cansee(). */ #define PGET_CANDEBUG 0x00004 /* Check against p_candebug(). */ #define PGET_ISCURRENT 0x00008 /* Check that the found process is current. */ #define PGET_NOTWEXIT 0x00010 /* Check that the process is not in P_WEXIT. */ #define PGET_NOTINEXEC 0x00020 /* Check that the process is not in P_INEXEC. */ #define PGET_NOTID 0x00040 /* Do not assume tid if pid > PID_MAX. */ #define PGET_WANTREAD (PGET_HOLD | PGET_CANDEBUG | PGET_NOTWEXIT) int pget(pid_t pid, int flags, struct proc **pp); void ast(struct trapframe *framep); struct thread *choosethread(void); int cr_cansignal(struct ucred *cred, struct proc *proc, int signum); int enterpgrp(struct proc *p, pid_t pgid, struct pgrp *pgrp, struct session *sess); int enterthispgrp(struct proc *p, struct pgrp *pgrp); void faultin(struct proc *p); void fixjobc(struct proc *p, struct pgrp *pgrp, int entering); int fork1(struct thread *, int, int, struct proc **, int *, int); void fork_exit(void (*)(void *, struct trapframe *), void *, struct trapframe *); void fork_return(struct thread *, struct trapframe *); int inferior(struct proc *p); void kern_yield(int); void kick_proc0(void); int leavepgrp(struct proc *p); int maybe_preempt(struct thread *td); void maybe_yield(void); void mi_switch(int flags, struct thread *newtd); int p_candebug(struct thread *td, struct proc *p); int p_cansee(struct thread *td, struct proc *p); int p_cansched(struct thread *td, struct proc *p); int p_cansignal(struct thread *td, struct proc *p, int signum); int p_canwait(struct thread *td, struct proc *p); struct pargs *pargs_alloc(int len); void pargs_drop(struct pargs *pa); void pargs_hold(struct pargs *pa); int proc_getargv(struct thread *td, struct proc *p, struct sbuf *sb); int proc_getauxv(struct thread *td, struct proc *p, struct sbuf *sb); int proc_getenvv(struct thread *td, struct proc *p, struct sbuf *sb); void procinit(void); void proc_linkup0(struct proc *p, struct thread *td); void proc_linkup(struct proc *p, struct thread *td); struct proc *proc_realparent(struct proc *child); void proc_reap(struct thread *td, struct proc *p, int *status, int options); void proc_reparent(struct proc *child, struct proc *newparent); struct pstats *pstats_alloc(void); void pstats_fork(struct pstats *src, struct pstats *dst); void pstats_free(struct pstats *ps); void reaper_abandon_children(struct proc *p, bool exiting); int securelevel_ge(struct ucred *cr, int level); int securelevel_gt(struct ucred *cr, int level); void sess_hold(struct session *); void sess_release(struct session *); int setrunnable(struct thread *); void setsugid(struct proc *p); int should_yield(void); int sigonstack(size_t sp); void stopevent(struct proc *, u_int, u_int); struct thread *tdfind(lwpid_t, pid_t); void threadinit(void); void tidhash_add(struct thread *); void tidhash_remove(struct thread *); void cpu_idle(int); int cpu_idle_wakeup(int); extern void (*cpu_idle_hook)(sbintime_t); /* Hook to machdep CPU idler. */ void cpu_switch(struct thread *, struct thread *, struct mtx *); void cpu_throw(struct thread *, struct thread *) __dead2; void unsleep(struct thread *); void userret(struct thread *, struct trapframe *); void cpu_exit(struct thread *); void exit1(struct thread *, int) __dead2; struct syscall_args; int cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa); void cpu_fork(struct thread *, struct proc *, struct thread *, int); void cpu_set_fork_handler(struct thread *, void (*)(void *), void *); void cpu_set_syscall_retval(struct thread *, int); void cpu_set_upcall(struct thread *td, struct thread *td0); void cpu_set_upcall_kse(struct thread *, void (*)(void *), void *, stack_t *); int cpu_set_user_tls(struct thread *, void *tls_base); void cpu_thread_alloc(struct thread *); void cpu_thread_clean(struct thread *); void cpu_thread_exit(struct thread *); void cpu_thread_free(struct thread *); void cpu_thread_swapin(struct thread *); void cpu_thread_swapout(struct thread *); struct thread *thread_alloc(int pages); int thread_alloc_stack(struct thread *, int pages); void thread_exit(void) __dead2; void thread_free(struct thread *td); void thread_link(struct thread *td, struct proc *p); void thread_reap(void); int thread_single(struct proc *p, int how); void thread_single_end(struct proc *p, int how); void thread_stash(struct thread *td); void thread_stopped(struct proc *p); void childproc_stopped(struct proc *child, int reason); void childproc_continued(struct proc *child); void childproc_exited(struct proc *child); int thread_suspend_check(int how); bool thread_suspend_check_needed(void); void thread_suspend_switch(struct thread *, struct proc *p); void thread_suspend_one(struct thread *td); void thread_unlink(struct thread *td); void thread_unsuspend(struct proc *p); int thread_unsuspend_one(struct thread *td, struct proc *p); void thread_wait(struct proc *p); struct thread *thread_find(struct proc *p, lwpid_t tid); void stop_all_proc(void); void resume_all_proc(void); static __inline int curthread_pflags_set(int flags) { struct thread *td; int save; td = curthread; save = ~flags | (td->td_pflags & flags); td->td_pflags |= flags; return (save); } static __inline void curthread_pflags_restore(int save) { curthread->td_pflags &= save; } #endif /* _KERNEL */ #endif /* !_SYS_PROC_H_ */