Index: projects/clang390-import/cddl/contrib/opensolaris/common/ctf/ctf_types.c =================================================================== --- projects/clang390-import/cddl/contrib/opensolaris/common/ctf/ctf_types.c (revision 305080) +++ projects/clang390-import/cddl/contrib/opensolaris/common/ctf/ctf_types.c (revision 305081) @@ -1,870 +1,887 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include ssize_t ctf_get_ctt_size(const ctf_file_t *fp, const ctf_type_t *tp, ssize_t *sizep, ssize_t *incrementp) { ssize_t size, increment; if (fp->ctf_version > CTF_VERSION_1 && tp->ctt_size == CTF_LSIZE_SENT) { size = CTF_TYPE_LSIZE(tp); increment = sizeof (ctf_type_t); } else { size = tp->ctt_size; increment = sizeof (ctf_stype_t); } if (sizep) *sizep = size; if (incrementp) *incrementp = increment; return (size); } /* * Iterate over the members of a STRUCT or UNION. We pass the name, member * type, and offset of each member to the specified callback function. */ int ctf_member_iter(ctf_file_t *fp, ctf_id_t type, ctf_member_f *func, void *arg) { ctf_file_t *ofp = fp; const ctf_type_t *tp; ssize_t size, increment; uint_t kind, n; int rc; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (CTF_ERR); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ (void) ctf_get_ctt_size(fp, tp, &size, &increment); kind = LCTF_INFO_KIND(fp, tp->ctt_info); if (kind != CTF_K_STRUCT && kind != CTF_K_UNION) return (ctf_set_errno(ofp, ECTF_NOTSOU)); if (fp->ctf_version == CTF_VERSION_1 || size < CTF_LSTRUCT_THRESH) { const ctf_member_t *mp = (const ctf_member_t *) ((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, mp++) { const char *name = ctf_strptr(fp, mp->ctm_name); if ((rc = func(name, mp->ctm_type, mp->ctm_offset, arg)) != 0) return (rc); } } else { const ctf_lmember_t *lmp = (const ctf_lmember_t *) ((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, lmp++) { const char *name = ctf_strptr(fp, lmp->ctlm_name); if ((rc = func(name, lmp->ctlm_type, (ulong_t)CTF_LMEM_OFFSET(lmp), arg)) != 0) return (rc); } } return (0); } /* * Iterate over the members of an ENUM. We pass the string name and associated * integer value of each enum element to the specified callback function. */ int ctf_enum_iter(ctf_file_t *fp, ctf_id_t type, ctf_enum_f *func, void *arg) { ctf_file_t *ofp = fp; const ctf_type_t *tp; const ctf_enum_t *ep; ssize_t increment; uint_t n; int rc; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (CTF_ERR); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ if (LCTF_INFO_KIND(fp, tp->ctt_info) != CTF_K_ENUM) return (ctf_set_errno(ofp, ECTF_NOTENUM)); (void) ctf_get_ctt_size(fp, tp, NULL, &increment); ep = (const ctf_enum_t *)((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, ep++) { const char *name = ctf_strptr(fp, ep->cte_name); if ((rc = func(name, ep->cte_value, arg)) != 0) return (rc); } return (0); } /* * Iterate over every root (user-visible) type in the given CTF container. * We pass the type ID of each type to the specified callback function. */ int ctf_type_iter(ctf_file_t *fp, ctf_type_f *func, void *arg) { ctf_id_t id, max = fp->ctf_typemax; int rc, child = (fp->ctf_flags & LCTF_CHILD); for (id = 1; id <= max; id++) { const ctf_type_t *tp = LCTF_INDEX_TO_TYPEPTR(fp, id); if (CTF_INFO_ISROOT(tp->ctt_info) && (rc = func(CTF_INDEX_TO_TYPE(id, child), arg)) != 0) return (rc); } return (0); } /* * Follow a given type through the graph for TYPEDEF, VOLATILE, CONST, and * RESTRICT nodes until we reach a "base" type node. This is useful when * we want to follow a type ID to a node that has members or a size. To guard * against infinite loops, we implement simplified cycle detection and check * each link against itself, the previous node, and the topmost node. */ ctf_id_t ctf_type_resolve(ctf_file_t *fp, ctf_id_t type) { ctf_id_t prev = type, otype = type; ctf_file_t *ofp = fp; const ctf_type_t *tp; while ((tp = ctf_lookup_by_id(&fp, type)) != NULL) { switch (LCTF_INFO_KIND(fp, tp->ctt_info)) { case CTF_K_TYPEDEF: case CTF_K_VOLATILE: case CTF_K_CONST: case CTF_K_RESTRICT: if (tp->ctt_type == type || tp->ctt_type == otype || tp->ctt_type == prev) { ctf_dprintf("type %ld cycle detected\n", otype); return (ctf_set_errno(ofp, ECTF_CORRUPT)); } prev = type; type = tp->ctt_type; break; default: return (type); } } return (CTF_ERR); /* errno is set for us */ } /* * Lookup the given type ID and print a string name for it into buf. Return * the actual number of bytes (not including \0) needed to format the name. */ static ssize_t ctf_type_qlname(ctf_file_t *fp, ctf_id_t type, char *buf, size_t len, const char *qname) { ctf_decl_t cd; ctf_decl_node_t *cdp; ctf_decl_prec_t prec, lp, rp; int ptr, arr; uint_t k; if (fp == NULL && type == CTF_ERR) return (-1); /* simplify caller code by permitting CTF_ERR */ ctf_decl_init(&cd, buf, len); ctf_decl_push(&cd, fp, type); if (cd.cd_err != 0) { ctf_decl_fini(&cd); return (ctf_set_errno(fp, cd.cd_err)); } /* * If the type graph's order conflicts with lexical precedence order * for pointers or arrays, then we need to surround the declarations at * the corresponding lexical precedence with parentheses. This can * result in either a parenthesized pointer (*) as in int (*)() or * int (*)[], or in a parenthesized pointer and array as in int (*[])(). */ ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER; arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY; rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1; lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1; k = CTF_K_POINTER; /* avoid leading whitespace (see below) */ for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) { for (cdp = ctf_list_next(&cd.cd_nodes[prec]); cdp != NULL; cdp = ctf_list_next(cdp)) { ctf_file_t *rfp = fp; const ctf_type_t *tp = ctf_lookup_by_id(&rfp, cdp->cd_type); const char *name = ctf_strptr(rfp, tp->ctt_name); if (k != CTF_K_POINTER && k != CTF_K_ARRAY) ctf_decl_sprintf(&cd, " "); if (lp == prec) { ctf_decl_sprintf(&cd, "("); lp = -1; } switch (cdp->cd_kind) { case CTF_K_INTEGER: case CTF_K_FLOAT: case CTF_K_TYPEDEF: if (qname != NULL) ctf_decl_sprintf(&cd, "%s`", qname); ctf_decl_sprintf(&cd, "%s", name); break; case CTF_K_POINTER: ctf_decl_sprintf(&cd, "*"); break; case CTF_K_ARRAY: ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n); break; case CTF_K_FUNCTION: ctf_decl_sprintf(&cd, "()"); break; case CTF_K_STRUCT: case CTF_K_FORWARD: ctf_decl_sprintf(&cd, "struct "); if (qname != NULL) ctf_decl_sprintf(&cd, "%s`", qname); ctf_decl_sprintf(&cd, "%s", name); break; case CTF_K_UNION: ctf_decl_sprintf(&cd, "union "); if (qname != NULL) ctf_decl_sprintf(&cd, "%s`", qname); ctf_decl_sprintf(&cd, "%s", name); break; case CTF_K_ENUM: ctf_decl_sprintf(&cd, "enum "); if (qname != NULL) ctf_decl_sprintf(&cd, "%s`", qname); ctf_decl_sprintf(&cd, "%s", name); break; case CTF_K_VOLATILE: ctf_decl_sprintf(&cd, "volatile"); break; case CTF_K_CONST: ctf_decl_sprintf(&cd, "const"); break; case CTF_K_RESTRICT: ctf_decl_sprintf(&cd, "restrict"); break; } k = cdp->cd_kind; } if (rp == prec) ctf_decl_sprintf(&cd, ")"); } if (cd.cd_len >= len) (void) ctf_set_errno(fp, ECTF_NAMELEN); ctf_decl_fini(&cd); return (cd.cd_len); } ssize_t ctf_type_lname(ctf_file_t *fp, ctf_id_t type, char *buf, size_t len) { return (ctf_type_qlname(fp, type, buf, len, NULL)); } /* * Lookup the given type ID and print a string name for it into buf. If buf * is too small, return NULL: the ECTF_NAMELEN error is set on 'fp' for us. */ char * ctf_type_name(ctf_file_t *fp, ctf_id_t type, char *buf, size_t len) { ssize_t rv = ctf_type_qlname(fp, type, buf, len, NULL); return (rv >= 0 && rv < len ? buf : NULL); } char * ctf_type_qname(ctf_file_t *fp, ctf_id_t type, char *buf, size_t len, const char *qname) { ssize_t rv = ctf_type_qlname(fp, type, buf, len, qname); return (rv >= 0 && rv < len ? buf : NULL); } /* * Resolve the type down to a base type node, and then return the size * of the type storage in bytes. */ ssize_t ctf_type_size(ctf_file_t *fp, ctf_id_t type) { const ctf_type_t *tp; ssize_t size; ctf_arinfo_t ar; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (-1); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (-1); /* errno is set for us */ switch (LCTF_INFO_KIND(fp, tp->ctt_info)) { case CTF_K_POINTER: return (fp->ctf_dmodel->ctd_pointer); case CTF_K_FUNCTION: return (0); /* function size is only known by symtab */ case CTF_K_ENUM: return (fp->ctf_dmodel->ctd_int); case CTF_K_ARRAY: /* * Array size is not directly returned by stabs data. Instead, * it defines the element type and requires the user to perform * the multiplication. If ctf_get_ctt_size() returns zero, the * current version of ctfconvert does not compute member sizes * and we compute the size here on its behalf. */ if ((size = ctf_get_ctt_size(fp, tp, NULL, NULL)) > 0) return (size); if (ctf_array_info(fp, type, &ar) == CTF_ERR || (size = ctf_type_size(fp, ar.ctr_contents)) == CTF_ERR) return (-1); /* errno is set for us */ return (size * ar.ctr_nelems); default: return (ctf_get_ctt_size(fp, tp, NULL, NULL)); } } /* * Resolve the type down to a base type node, and then return the alignment * needed for the type storage in bytes. */ ssize_t ctf_type_align(ctf_file_t *fp, ctf_id_t type) { const ctf_type_t *tp; ctf_arinfo_t r; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (-1); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (-1); /* errno is set for us */ switch (LCTF_INFO_KIND(fp, tp->ctt_info)) { case CTF_K_POINTER: case CTF_K_FUNCTION: return (fp->ctf_dmodel->ctd_pointer); case CTF_K_ARRAY: if (ctf_array_info(fp, type, &r) == CTF_ERR) return (-1); /* errno is set for us */ return (ctf_type_align(fp, r.ctr_contents)); case CTF_K_STRUCT: case CTF_K_UNION: { uint_t n = LCTF_INFO_VLEN(fp, tp->ctt_info); ssize_t size, increment; size_t align = 0; const void *vmp; (void) ctf_get_ctt_size(fp, tp, &size, &increment); vmp = (uchar_t *)tp + increment; if (LCTF_INFO_KIND(fp, tp->ctt_info) == CTF_K_STRUCT) n = MIN(n, 1); /* only use first member for structs */ if (fp->ctf_version == CTF_VERSION_1 || size < CTF_LSTRUCT_THRESH) { const ctf_member_t *mp = vmp; for (; n != 0; n--, mp++) { ssize_t am = ctf_type_align(fp, mp->ctm_type); align = MAX(align, am); } } else { const ctf_lmember_t *lmp = vmp; for (; n != 0; n--, lmp++) { ssize_t am = ctf_type_align(fp, lmp->ctlm_type); align = MAX(align, am); } } return (align); } case CTF_K_ENUM: return (fp->ctf_dmodel->ctd_int); default: return (ctf_get_ctt_size(fp, tp, NULL, NULL)); } } /* * Return the kind (CTF_K_* constant) for the specified type ID. */ int ctf_type_kind(ctf_file_t *fp, ctf_id_t type) { const ctf_type_t *tp; if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ return (LCTF_INFO_KIND(fp, tp->ctt_info)); } /* * If the type is one that directly references another type (such as POINTER), * then return the ID of the type to which it refers. */ ctf_id_t ctf_type_reference(ctf_file_t *fp, ctf_id_t type) { ctf_file_t *ofp = fp; const ctf_type_t *tp; if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ switch (LCTF_INFO_KIND(fp, tp->ctt_info)) { case CTF_K_POINTER: case CTF_K_TYPEDEF: case CTF_K_VOLATILE: case CTF_K_CONST: case CTF_K_RESTRICT: return (tp->ctt_type); default: return (ctf_set_errno(ofp, ECTF_NOTREF)); } } /* * Find a pointer to type by looking in fp->ctf_ptrtab. If we can't find a * pointer to the given type, see if we can compute a pointer to the type * resulting from resolving the type down to its base type and use that * instead. This helps with cases where the CTF data includes "struct foo *" * but not "foo_t *" and the user accesses "foo_t *" in the debugger. */ ctf_id_t ctf_type_pointer(ctf_file_t *fp, ctf_id_t type) { ctf_file_t *ofp = fp; ctf_id_t ntype; if (ctf_lookup_by_id(&fp, type) == NULL) return (CTF_ERR); /* errno is set for us */ if ((ntype = fp->ctf_ptrtab[CTF_TYPE_TO_INDEX(type)]) != 0) return (CTF_INDEX_TO_TYPE(ntype, (fp->ctf_flags & LCTF_CHILD))); if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (ctf_set_errno(ofp, ECTF_NOTYPE)); if (ctf_lookup_by_id(&fp, type) == NULL) return (ctf_set_errno(ofp, ECTF_NOTYPE)); if ((ntype = fp->ctf_ptrtab[CTF_TYPE_TO_INDEX(type)]) != 0) return (CTF_INDEX_TO_TYPE(ntype, (fp->ctf_flags & LCTF_CHILD))); return (ctf_set_errno(ofp, ECTF_NOTYPE)); } /* * Return the encoding for the specified INTEGER or FLOAT. */ int ctf_type_encoding(ctf_file_t *fp, ctf_id_t type, ctf_encoding_t *ep) { ctf_file_t *ofp = fp; const ctf_type_t *tp; ssize_t increment; uint_t data; if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ (void) ctf_get_ctt_size(fp, tp, NULL, &increment); switch (LCTF_INFO_KIND(fp, tp->ctt_info)) { case CTF_K_INTEGER: data = *(const uint_t *)((uintptr_t)tp + increment); ep->cte_format = CTF_INT_ENCODING(data); ep->cte_offset = CTF_INT_OFFSET(data); ep->cte_bits = CTF_INT_BITS(data); break; case CTF_K_FLOAT: data = *(const uint_t *)((uintptr_t)tp + increment); ep->cte_format = CTF_FP_ENCODING(data); ep->cte_offset = CTF_FP_OFFSET(data); ep->cte_bits = CTF_FP_BITS(data); break; default: return (ctf_set_errno(ofp, ECTF_NOTINTFP)); } return (0); } int ctf_type_cmp(ctf_file_t *lfp, ctf_id_t ltype, ctf_file_t *rfp, ctf_id_t rtype) { int rval; if (ltype < rtype) rval = -1; else if (ltype > rtype) rval = 1; else rval = 0; if (lfp == rfp) return (rval); if (CTF_TYPE_ISPARENT(ltype) && lfp->ctf_parent != NULL) lfp = lfp->ctf_parent; if (CTF_TYPE_ISPARENT(rtype) && rfp->ctf_parent != NULL) rfp = rfp->ctf_parent; if (lfp < rfp) return (-1); if (lfp > rfp) return (1); return (rval); } /* * Return a boolean value indicating if two types are compatible integers or * floating-pointer values. This function returns true if the two types are * the same, or if they have the same ASCII name and encoding properties. * This function could be extended to test for compatibility for other kinds. */ int ctf_type_compat(ctf_file_t *lfp, ctf_id_t ltype, ctf_file_t *rfp, ctf_id_t rtype) { const ctf_type_t *ltp, *rtp; ctf_encoding_t le, re; ctf_arinfo_t la, ra; uint_t lkind, rkind; if (ctf_type_cmp(lfp, ltype, rfp, rtype) == 0) return (1); ltype = ctf_type_resolve(lfp, ltype); lkind = ctf_type_kind(lfp, ltype); rtype = ctf_type_resolve(rfp, rtype); rkind = ctf_type_kind(rfp, rtype); if (lkind != rkind || (ltp = ctf_lookup_by_id(&lfp, ltype)) == NULL || (rtp = ctf_lookup_by_id(&rfp, rtype)) == NULL || strcmp(ctf_strptr(lfp, ltp->ctt_name), ctf_strptr(rfp, rtp->ctt_name)) != 0) return (0); switch (lkind) { case CTF_K_INTEGER: case CTF_K_FLOAT: return (ctf_type_encoding(lfp, ltype, &le) == 0 && ctf_type_encoding(rfp, rtype, &re) == 0 && bcmp(&le, &re, sizeof (ctf_encoding_t)) == 0); case CTF_K_POINTER: return (ctf_type_compat(lfp, ctf_type_reference(lfp, ltype), rfp, ctf_type_reference(rfp, rtype))); case CTF_K_ARRAY: return (ctf_array_info(lfp, ltype, &la) == 0 && ctf_array_info(rfp, rtype, &ra) == 0 && la.ctr_nelems == ra.ctr_nelems && ctf_type_compat( lfp, la.ctr_contents, rfp, ra.ctr_contents) && ctf_type_compat(lfp, la.ctr_index, rfp, ra.ctr_index)); case CTF_K_STRUCT: case CTF_K_UNION: return (ctf_type_size(lfp, ltype) == ctf_type_size(rfp, rtype)); case CTF_K_ENUM: case CTF_K_FORWARD: return (1); /* no other checks required for these type kinds */ default: return (0); /* should not get here since we did a resolve */ } } -/* - * Return the type and offset for a given member of a STRUCT or UNION. - */ -int -ctf_member_info(ctf_file_t *fp, ctf_id_t type, const char *name, +static int +_ctf_member_info(ctf_file_t *fp, ctf_id_t type, const char *name, ulong_t off, ctf_membinfo_t *mip) { ctf_file_t *ofp = fp; const ctf_type_t *tp; ssize_t size, increment; uint_t kind, n; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (CTF_ERR); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ (void) ctf_get_ctt_size(fp, tp, &size, &increment); kind = LCTF_INFO_KIND(fp, tp->ctt_info); if (kind != CTF_K_STRUCT && kind != CTF_K_UNION) return (ctf_set_errno(ofp, ECTF_NOTSOU)); if (fp->ctf_version == CTF_VERSION_1 || size < CTF_LSTRUCT_THRESH) { const ctf_member_t *mp = (const ctf_member_t *) ((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, mp++) { + if (mp->ctm_name == 0 && + _ctf_member_info(fp, mp->ctm_type, name, + mp->ctm_offset + off, mip) == 0) + return (0); if (strcmp(ctf_strptr(fp, mp->ctm_name), name) == 0) { mip->ctm_type = mp->ctm_type; - mip->ctm_offset = mp->ctm_offset; + mip->ctm_offset = mp->ctm_offset + off; return (0); } } } else { const ctf_lmember_t *lmp = (const ctf_lmember_t *) ((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, lmp++) { + if (lmp->ctlm_name == 0 && + _ctf_member_info(fp, lmp->ctlm_name, name, + (ulong_t)CTF_LMEM_OFFSET(lmp) + off, mip) == 0) + return (0); if (strcmp(ctf_strptr(fp, lmp->ctlm_name), name) == 0) { mip->ctm_type = lmp->ctlm_type; - mip->ctm_offset = (ulong_t)CTF_LMEM_OFFSET(lmp); + mip->ctm_offset = + (ulong_t)CTF_LMEM_OFFSET(lmp) + off; return (0); } } } return (ctf_set_errno(ofp, ECTF_NOMEMBNAM)); +} + +/* + * Return the type and offset for a given member of a STRUCT or UNION. + */ +int +ctf_member_info(ctf_file_t *fp, ctf_id_t type, const char *name, + ctf_membinfo_t *mip) +{ + + return (_ctf_member_info(fp, type, name, 0, mip)); } /* * Return the array type, index, and size information for the specified ARRAY. */ int ctf_array_info(ctf_file_t *fp, ctf_id_t type, ctf_arinfo_t *arp) { ctf_file_t *ofp = fp; const ctf_type_t *tp; const ctf_array_t *ap; ssize_t increment; if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ if (LCTF_INFO_KIND(fp, tp->ctt_info) != CTF_K_ARRAY) return (ctf_set_errno(ofp, ECTF_NOTARRAY)); (void) ctf_get_ctt_size(fp, tp, NULL, &increment); ap = (const ctf_array_t *)((uintptr_t)tp + increment); arp->ctr_contents = ap->cta_contents; arp->ctr_index = ap->cta_index; arp->ctr_nelems = ap->cta_nelems; return (0); } /* * Convert the specified value to the corresponding enum member name, if a * matching name can be found. Otherwise NULL is returned. */ const char * ctf_enum_name(ctf_file_t *fp, ctf_id_t type, int value) { ctf_file_t *ofp = fp; const ctf_type_t *tp; const ctf_enum_t *ep; ssize_t increment; uint_t n; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (NULL); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (NULL); /* errno is set for us */ if (LCTF_INFO_KIND(fp, tp->ctt_info) != CTF_K_ENUM) { (void) ctf_set_errno(ofp, ECTF_NOTENUM); return (NULL); } (void) ctf_get_ctt_size(fp, tp, NULL, &increment); ep = (const ctf_enum_t *)((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, ep++) { if (ep->cte_value == value) return (ctf_strptr(fp, ep->cte_name)); } (void) ctf_set_errno(ofp, ECTF_NOENUMNAM); return (NULL); } /* * Convert the specified enum tag name to the corresponding value, if a * matching name can be found. Otherwise CTF_ERR is returned. */ int ctf_enum_value(ctf_file_t *fp, ctf_id_t type, const char *name, int *valp) { ctf_file_t *ofp = fp; const ctf_type_t *tp; const ctf_enum_t *ep; ssize_t size, increment; uint_t n; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (CTF_ERR); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ if (LCTF_INFO_KIND(fp, tp->ctt_info) != CTF_K_ENUM) { (void) ctf_set_errno(ofp, ECTF_NOTENUM); return (CTF_ERR); } (void) ctf_get_ctt_size(fp, tp, &size, &increment); ep = (const ctf_enum_t *)((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, ep++) { if (strcmp(ctf_strptr(fp, ep->cte_name), name) == 0) { if (valp != NULL) *valp = ep->cte_value; return (0); } } (void) ctf_set_errno(ofp, ECTF_NOENUMNAM); return (CTF_ERR); } /* * Recursively visit the members of any type. This function is used as the * engine for ctf_type_visit, below. We resolve the input type, recursively * invoke ourself for each type member if the type is a struct or union, and * then invoke the callback function on the current type. If any callback * returns non-zero, we abort and percolate the error code back up to the top. */ static int ctf_type_rvisit(ctf_file_t *fp, ctf_id_t type, ctf_visit_f *func, void *arg, const char *name, ulong_t offset, int depth) { ctf_id_t otype = type; const ctf_type_t *tp; ssize_t size, increment; uint_t kind, n; int rc; if ((type = ctf_type_resolve(fp, type)) == CTF_ERR) return (CTF_ERR); /* errno is set for us */ if ((tp = ctf_lookup_by_id(&fp, type)) == NULL) return (CTF_ERR); /* errno is set for us */ if ((rc = func(name, otype, offset, depth, arg)) != 0) return (rc); kind = LCTF_INFO_KIND(fp, tp->ctt_info); if (kind != CTF_K_STRUCT && kind != CTF_K_UNION) return (0); (void) ctf_get_ctt_size(fp, tp, &size, &increment); if (fp->ctf_version == CTF_VERSION_1 || size < CTF_LSTRUCT_THRESH) { const ctf_member_t *mp = (const ctf_member_t *) ((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, mp++) { if ((rc = ctf_type_rvisit(fp, mp->ctm_type, func, arg, ctf_strptr(fp, mp->ctm_name), offset + mp->ctm_offset, depth + 1)) != 0) return (rc); } } else { const ctf_lmember_t *lmp = (const ctf_lmember_t *) ((uintptr_t)tp + increment); for (n = LCTF_INFO_VLEN(fp, tp->ctt_info); n != 0; n--, lmp++) { if ((rc = ctf_type_rvisit(fp, lmp->ctlm_type, func, arg, ctf_strptr(fp, lmp->ctlm_name), offset + (ulong_t)CTF_LMEM_OFFSET(lmp), depth + 1)) != 0) return (rc); } } return (0); } /* * Recursively visit the members of any type. We pass the name, member * type, and offset of each member to the specified callback function. */ int ctf_type_visit(ctf_file_t *fp, ctf_id_t type, ctf_visit_f *func, void *arg) { return (ctf_type_rvisit(fp, type, func, arg, "", 0, 0)); } Index: projects/clang390-import/cddl/contrib/opensolaris =================================================================== --- projects/clang390-import/cddl/contrib/opensolaris (revision 305080) +++ projects/clang390-import/cddl/contrib/opensolaris (revision 305081) Property changes on: projects/clang390-import/cddl/contrib/opensolaris ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/cddl/contrib/opensolaris:r304537-305080 Index: projects/clang390-import/cddl =================================================================== --- projects/clang390-import/cddl (revision 305080) +++ projects/clang390-import/cddl (revision 305081) Property changes on: projects/clang390-import/cddl ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/cddl:r305029-305080 Index: projects/clang390-import/contrib/netbsd-tests/usr.bin/dirname/t_dirname.sh =================================================================== --- projects/clang390-import/contrib/netbsd-tests/usr.bin/dirname/t_dirname.sh (revision 305080) +++ projects/clang390-import/contrib/netbsd-tests/usr.bin/dirname/t_dirname.sh (revision 305081) @@ -1,52 +1,52 @@ # $NetBSD: t_dirname.sh,v 1.1 2012/03/17 16:33:13 jruoho Exp $ # # Copyright (c) 2008 The NetBSD Foundation, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # atf_test_case basic basic_head() { atf_set "descr" "Checks basic functionality" } basic_body() { # Begin FreeBSD - atf_tc_expect_fail "dirname //usr//bin doesn't return //usr like it used to; bug # 212193" + atf_expect_fail "dirname //usr//bin doesn't return //usr like it used to; bug # 212193" # End FreeBSD atf_check -o inline:"/\n" dirname / atf_check -o inline:"/\n" dirname // atf_check -o inline:"/usr\n" dirname /usr/bin/ atf_check -o inline:"//usr\n" dirname //usr//bin// atf_check -o inline:".\n" dirname usr atf_check -o inline:".\n" dirname "" atf_check -o inline:"/\n" dirname /usr atf_check -o inline:"/usr\n" dirname /usr/bin atf_check -o inline:"usr\n" dirname usr/bin } atf_init_test_cases() { atf_add_test_case basic } Index: projects/clang390-import/contrib/telnet/libtelnet/pk.c =================================================================== --- projects/clang390-import/contrib/telnet/libtelnet/pk.c (revision 305080) +++ projects/clang390-import/contrib/telnet/libtelnet/pk.c (revision 305081) @@ -1,265 +1,265 @@ /*- * Copyright (c) 1991, 1993 * Dave Safford. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); /* public key routines */ /* functions: genkeys(char *public, char *secret) common_key(char *secret, char *public, desData *deskey) pk_encode(char *in, *out, DesData *deskey); pk_decode(char *in, *out, DesData *deskey); where char public[HEXKEYBYTES + 1]; char secret[HEXKEYBYTES + 1]; */ #include #include #include #include #include #include #include "mp.h" #include "pk.h" static void adjust(char keyout[HEXKEYBYTES+1], char *keyin); /* * Choose top 128 bits of the common key to use as our idea key. */ static void extractideakey(MINT *ck, IdeaData *ideakey) { MINT *a; MINT *z; short r; int i; short base = (1 << 8); char *k; z = mp_itom(0); a = mp_itom(0); mp_madd(ck, z, a); for (i = 0; i < ((KEYSIZE - 128) / 8); i++) { mp_sdiv(a, base, a, &r); } k = (char *)ideakey; for (i = 0; i < 16; i++) { mp_sdiv(a, base, a, &r); *k++ = r; } mp_mfree(z); mp_mfree(a); } /* * Choose middle 64 bits of the common key to use as our des key, possibly * overwriting the lower order bits by setting parity. */ static void extractdeskey(MINT *ck, DesData *deskey) { MINT *a; MINT *z; short r; int i; short base = (1 << 8); char *k; z = mp_itom(0); a = mp_itom(0); mp_madd(ck, z, a); for (i = 0; i < ((KEYSIZE - 64) / 2) / 8; i++) { mp_sdiv(a, base, a, &r); } k = (char *)deskey; for (i = 0; i < 8; i++) { mp_sdiv(a, base, a, &r); *k++ = r; } mp_mfree(z); mp_mfree(a); } /* * get common key from my secret key and his public key */ void common_key(char *xsecret, char *xpublic, IdeaData *ideakey, DesData *deskey) { MINT *public; MINT *secret; MINT *common; MINT *modulus = mp_xtom(HEXMODULUS); public = mp_xtom(xpublic); secret = mp_xtom(xsecret); common = mp_itom(0); mp_pow(public, secret, modulus, common); extractdeskey(common, deskey); extractideakey(common, ideakey); des_set_odd_parity(deskey); mp_mfree(common); mp_mfree(secret); mp_mfree(public); mp_mfree(modulus); } /* * Generate a seed */ static void getseed(char *seed, int seedsize) { int i; srandomdev(); for (i = 0; i < seedsize; i++) { seed[i] = random() & 0xff; } } /* * Generate a random public/secret key pair */ void genkeys(char *public, char *secret) { size_t i; # define BASEBITS (8*sizeof(short) - 1) # define BASE (1 << BASEBITS) MINT *pk = mp_itom(0); MINT *sk = mp_itom(0); MINT *tmp; - MINT *base = mp_itom(BASE); + MINT *base = mp_itom((short)BASE); MINT *root = mp_itom(PROOT); MINT *modulus = mp_xtom(HEXMODULUS); short r; unsigned short seed[KEYSIZE/BASEBITS + 1]; char *xkey; getseed((char *)seed, sizeof(seed)); for (i = 0; i < KEYSIZE/BASEBITS + 1; i++) { r = seed[i] % BASE; tmp = mp_itom(r); mp_mult(sk, base, sk); mp_madd(sk, tmp, sk); mp_mfree(tmp); } tmp = mp_itom(0); mp_mdiv(sk, modulus, tmp, sk); mp_mfree(tmp); mp_pow(root, sk, modulus, pk); xkey = mp_mtox(sk); adjust(secret, xkey); xkey = mp_mtox(pk); adjust(public, xkey); mp_mfree(sk); mp_mfree(base); mp_mfree(pk); mp_mfree(root); mp_mfree(modulus); } /* * Adjust the input key so that it is 0-filled on the left */ static void adjust(char keyout[HEXKEYBYTES+1], char *keyin) { char *p; char *s; for (p = keyin; *p; p++) ; for (s = keyout + HEXKEYBYTES; p >= keyin; p--, s--) { *s = *p; } while (s >= keyout) { *s-- = '0'; } } static char hextab[17] = "0123456789ABCDEF"; /* given a DES key, cbc encrypt and translate input to terminated hex */ void pk_encode(char *in, char *out, DesData *key) { char buf[256]; DesData i; des_key_schedule k; int l,op,deslen; memset(&i,0,sizeof(i)); memset(buf,0,sizeof(buf)); deslen = ((strlen(in) + 7)/8)*8; des_key_sched(key, k); des_cbc_encrypt(in,buf,deslen, k,&i,DES_ENCRYPT); for (l=0,op=0;l> 4]; out[op++] = hextab[(buf[l] & 0x0f)]; } out[op] = '\0'; } /* given a DES key, translate input from hex and decrypt */ void pk_decode(char *in, char *out, DesData *key) { char buf[256]; DesData i; des_key_schedule k; int n1,n2,op; size_t l; memset(&i,0,sizeof(i)); memset(buf,0,sizeof(buf)); for (l=0,op=0;l '9') n1 = in[op] - 'A' + 10; else n1 = in[op] - '0'; if (in[op+1] > '9') n2 = in[op+1] - 'A' + 10; else n2 = in[op+1] - '0'; buf[l] = n1*16 +n2; } des_key_sched(key, k); des_cbc_encrypt(buf,out,strlen(in)/2, k,&i,DES_DECRYPT); out[strlen(in)/2] = '\0'; } Index: projects/clang390-import/crypto/openssh/auth-pam.c =================================================================== --- projects/clang390-import/crypto/openssh/auth-pam.c (revision 305080) +++ projects/clang390-import/crypto/openssh/auth-pam.c (revision 305081) @@ -1,1220 +1,1222 @@ /*- * Copyright (c) 2002 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and * NAI Labs, the Security Research Division of Network Associates, Inc. * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the * DARPA CHATS research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright (c) 2003,2004 Damien Miller * Copyright (c) 2003,2004 Darren Tucker * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* Based on FreeBSD: src/crypto/openssh/auth2-pam-freebsd.c,v 1.11 2003/03/31 13:48:18 des */ #include "includes.h" #include #include #include #include #include #include #include #include #ifdef USE_PAM #if defined(HAVE_SECURITY_PAM_APPL_H) #include #elif defined (HAVE_PAM_PAM_APPL_H) #include #endif /* OpenGroup RFC86.0 and XSSO specify no "const" on arguments */ #ifdef PAM_SUN_CODEBASE # define sshpam_const /* Solaris, HP-UX, AIX */ #else # define sshpam_const const /* LinuxPAM, OpenPAM */ #endif /* Ambiguity in spec: is it an array of pointers or a pointer to an array? */ #ifdef PAM_SUN_CODEBASE # define PAM_MSG_MEMBER(msg, n, member) ((*(msg))[(n)].member) #else # define PAM_MSG_MEMBER(msg, n, member) ((msg)[(n)]->member) #endif #include "xmalloc.h" #include "buffer.h" #include "key.h" #include "hostfile.h" #include "auth.h" #include "auth-pam.h" #include "canohost.h" #include "log.h" #include "msg.h" #include "packet.h" #include "misc.h" #include "servconf.h" #include "ssh2.h" #include "auth-options.h" #ifdef GSSAPI #include "ssh-gss.h" #endif #include "monitor_wrap.h" +#include "blacklist_client.h" extern ServerOptions options; extern Buffer loginmsg; extern int compat20; extern u_int utmp_len; /* so we don't silently change behaviour */ #ifdef USE_POSIX_THREADS # error "USE_POSIX_THREADS replaced by UNSUPPORTED_POSIX_THREADS_HACK" #endif /* * Formerly known as USE_POSIX_THREADS, using this is completely unsupported * and generally a bad idea. Use at own risk and do not expect support if * this breaks. */ #ifdef UNSUPPORTED_POSIX_THREADS_HACK #include /* * Avoid namespace clash when *not* using pthreads for systems *with* * pthreads, which unconditionally define pthread_t via sys/types.h * (e.g. Linux) */ typedef pthread_t sp_pthread_t; #else typedef pid_t sp_pthread_t; #endif struct pam_ctxt { sp_pthread_t pam_thread; int pam_psock; int pam_csock; int pam_done; }; static void sshpam_free_ctx(void *); static struct pam_ctxt *cleanup_ctxt; #ifndef UNSUPPORTED_POSIX_THREADS_HACK /* * Simulate threads with processes. */ static int sshpam_thread_status = -1; static mysig_t sshpam_oldsig; static void sshpam_sigchld_handler(int sig) { signal(SIGCHLD, SIG_DFL); if (cleanup_ctxt == NULL) return; /* handler called after PAM cleanup, shouldn't happen */ if (waitpid(cleanup_ctxt->pam_thread, &sshpam_thread_status, WNOHANG) <= 0) { /* PAM thread has not exitted, privsep slave must have */ kill(cleanup_ctxt->pam_thread, SIGTERM); if (waitpid(cleanup_ctxt->pam_thread, &sshpam_thread_status, 0) <= 0) return; /* could not wait */ } if (WIFSIGNALED(sshpam_thread_status) && WTERMSIG(sshpam_thread_status) == SIGTERM) return; /* terminated by pthread_cancel */ if (!WIFEXITED(sshpam_thread_status)) sigdie("PAM: authentication thread exited unexpectedly"); if (WEXITSTATUS(sshpam_thread_status) != 0) sigdie("PAM: authentication thread exited uncleanly"); } /* ARGSUSED */ static void pthread_exit(void *value) { _exit(0); } /* ARGSUSED */ static int pthread_create(sp_pthread_t *thread, const void *attr, void *(*thread_start)(void *), void *arg) { pid_t pid; struct pam_ctxt *ctx = arg; sshpam_thread_status = -1; switch ((pid = fork())) { case -1: error("fork(): %s", strerror(errno)); return (-1); case 0: close(ctx->pam_psock); ctx->pam_psock = -1; thread_start(arg); _exit(1); default: *thread = pid; close(ctx->pam_csock); ctx->pam_csock = -1; sshpam_oldsig = signal(SIGCHLD, sshpam_sigchld_handler); return (0); } } static int pthread_cancel(sp_pthread_t thread) { signal(SIGCHLD, sshpam_oldsig); return (kill(thread, SIGTERM)); } /* ARGSUSED */ static int pthread_join(sp_pthread_t thread, void **value) { int status; if (sshpam_thread_status != -1) return (sshpam_thread_status); signal(SIGCHLD, sshpam_oldsig); waitpid(thread, &status, 0); return (status); } #endif static pam_handle_t *sshpam_handle = NULL; static int sshpam_err = 0; static int sshpam_authenticated = 0; static int sshpam_session_open = 0; static int sshpam_cred_established = 0; static int sshpam_account_status = -1; static char **sshpam_env = NULL; static Authctxt *sshpam_authctxt = NULL; static const char *sshpam_password = NULL; static char badpw[] = "\b\n\r\177INCORRECT"; /* Some PAM implementations don't implement this */ #ifndef HAVE_PAM_GETENVLIST static char ** pam_getenvlist(pam_handle_t *pamh) { /* * XXX - If necessary, we can still support envrionment passing * for platforms without pam_getenvlist by searching for known * env vars (e.g. KRB5CCNAME) from the PAM environment. */ return NULL; } #endif /* * Some platforms, notably Solaris, do not enforce password complexity * rules during pam_chauthtok() if the real uid of the calling process * is 0, on the assumption that it's being called by "passwd" run by root. * This wraps pam_chauthtok and sets/restore the real uid so PAM will do * the right thing. */ #ifdef SSHPAM_CHAUTHTOK_NEEDS_RUID static int sshpam_chauthtok_ruid(pam_handle_t *pamh, int flags) { int result; if (sshpam_authctxt == NULL) fatal("PAM: sshpam_authctxt not initialized"); if (setreuid(sshpam_authctxt->pw->pw_uid, -1) == -1) fatal("%s: setreuid failed: %s", __func__, strerror(errno)); result = pam_chauthtok(pamh, flags); if (setreuid(0, -1) == -1) fatal("%s: setreuid failed: %s", __func__, strerror(errno)); return result; } # define pam_chauthtok(a,b) (sshpam_chauthtok_ruid((a), (b))) #endif void sshpam_password_change_required(int reqd) { debug3("%s %d", __func__, reqd); if (sshpam_authctxt == NULL) fatal("%s: PAM authctxt not initialized", __func__); sshpam_authctxt->force_pwchange = reqd; if (reqd) { no_port_forwarding_flag |= 2; no_agent_forwarding_flag |= 2; no_x11_forwarding_flag |= 2; } else { no_port_forwarding_flag &= ~2; no_agent_forwarding_flag &= ~2; no_x11_forwarding_flag &= ~2; } } /* Import regular and PAM environment from subprocess */ static void import_environments(Buffer *b) { char *env; u_int i, num_env; int err; debug3("PAM: %s entering", __func__); #ifndef UNSUPPORTED_POSIX_THREADS_HACK /* Import variables set by do_pam_account */ sshpam_account_status = buffer_get_int(b); sshpam_password_change_required(buffer_get_int(b)); /* Import environment from subprocess */ num_env = buffer_get_int(b); if (num_env > 1024) fatal("%s: received %u environment variables, expected <= 1024", __func__, num_env); sshpam_env = xcalloc(num_env + 1, sizeof(*sshpam_env)); debug3("PAM: num env strings %d", num_env); for(i = 0; i < num_env; i++) sshpam_env[i] = buffer_get_string(b, NULL); sshpam_env[num_env] = NULL; /* Import PAM environment from subprocess */ num_env = buffer_get_int(b); debug("PAM: num PAM env strings %d", num_env); for(i = 0; i < num_env; i++) { env = buffer_get_string(b, NULL); #ifdef HAVE_PAM_PUTENV /* Errors are not fatal here */ if ((err = pam_putenv(sshpam_handle, env)) != PAM_SUCCESS) { error("PAM: pam_putenv: %s", pam_strerror(sshpam_handle, sshpam_err)); } #endif } #endif } /* * Conversation function for authentication thread. */ static int sshpam_thread_conv(int n, sshpam_const struct pam_message **msg, struct pam_response **resp, void *data) { Buffer buffer; struct pam_ctxt *ctxt; struct pam_response *reply; int i; debug3("PAM: %s entering, %d messages", __func__, n); *resp = NULL; if (data == NULL) { error("PAM: conversation function passed a null context"); return (PAM_CONV_ERR); } ctxt = data; if (n <= 0 || n > PAM_MAX_NUM_MSG) return (PAM_CONV_ERR); if ((reply = calloc(n, sizeof(*reply))) == NULL) return (PAM_CONV_ERR); buffer_init(&buffer); for (i = 0; i < n; ++i) { switch (PAM_MSG_MEMBER(msg, i, msg_style)) { case PAM_PROMPT_ECHO_OFF: buffer_put_cstring(&buffer, PAM_MSG_MEMBER(msg, i, msg)); if (ssh_msg_send(ctxt->pam_csock, PAM_MSG_MEMBER(msg, i, msg_style), &buffer) == -1) goto fail; if (ssh_msg_recv(ctxt->pam_csock, &buffer) == -1) goto fail; if (buffer_get_char(&buffer) != PAM_AUTHTOK) goto fail; reply[i].resp = buffer_get_string(&buffer, NULL); break; case PAM_PROMPT_ECHO_ON: buffer_put_cstring(&buffer, PAM_MSG_MEMBER(msg, i, msg)); if (ssh_msg_send(ctxt->pam_csock, PAM_MSG_MEMBER(msg, i, msg_style), &buffer) == -1) goto fail; if (ssh_msg_recv(ctxt->pam_csock, &buffer) == -1) goto fail; if (buffer_get_char(&buffer) != PAM_AUTHTOK) goto fail; reply[i].resp = buffer_get_string(&buffer, NULL); break; case PAM_ERROR_MSG: buffer_put_cstring(&buffer, PAM_MSG_MEMBER(msg, i, msg)); if (ssh_msg_send(ctxt->pam_csock, PAM_MSG_MEMBER(msg, i, msg_style), &buffer) == -1) goto fail; break; case PAM_TEXT_INFO: buffer_put_cstring(&buffer, PAM_MSG_MEMBER(msg, i, msg)); if (ssh_msg_send(ctxt->pam_csock, PAM_MSG_MEMBER(msg, i, msg_style), &buffer) == -1) goto fail; break; default: goto fail; } buffer_clear(&buffer); } buffer_free(&buffer); *resp = reply; return (PAM_SUCCESS); fail: for(i = 0; i < n; i++) { free(reply[i].resp); } free(reply); buffer_free(&buffer); return (PAM_CONV_ERR); } /* * Authentication thread. */ static void * sshpam_thread(void *ctxtp) { struct pam_ctxt *ctxt = ctxtp; Buffer buffer; struct pam_conv sshpam_conv; int flags = (options.permit_empty_passwd == 0 ? PAM_DISALLOW_NULL_AUTHTOK : 0); #ifndef UNSUPPORTED_POSIX_THREADS_HACK extern char **environ; char **env_from_pam; u_int i; const char *pam_user; const char **ptr_pam_user = &pam_user; char *tz = getenv("TZ"); sshpam_err = pam_get_item(sshpam_handle, PAM_USER, (sshpam_const void **)ptr_pam_user); if (sshpam_err != PAM_SUCCESS) goto auth_fail; environ[0] = NULL; if (tz != NULL) if (setenv("TZ", tz, 1) == -1) error("PAM: could not set TZ environment: %s", strerror(errno)); if (sshpam_authctxt != NULL) { setproctitle("%s [pam]", sshpam_authctxt->valid ? pam_user : "unknown"); } #endif sshpam_conv.conv = sshpam_thread_conv; sshpam_conv.appdata_ptr = ctxt; if (sshpam_authctxt == NULL) fatal("%s: PAM authctxt not initialized", __func__); buffer_init(&buffer); sshpam_err = pam_set_item(sshpam_handle, PAM_CONV, (const void *)&sshpam_conv); if (sshpam_err != PAM_SUCCESS) goto auth_fail; sshpam_err = pam_authenticate(sshpam_handle, flags); if (sshpam_err != PAM_SUCCESS) goto auth_fail; if (compat20) { if (!do_pam_account()) { sshpam_err = PAM_ACCT_EXPIRED; goto auth_fail; } if (sshpam_authctxt->force_pwchange) { sshpam_err = pam_chauthtok(sshpam_handle, PAM_CHANGE_EXPIRED_AUTHTOK); if (sshpam_err != PAM_SUCCESS) goto auth_fail; sshpam_password_change_required(0); } } buffer_put_cstring(&buffer, "OK"); #ifndef UNSUPPORTED_POSIX_THREADS_HACK /* Export variables set by do_pam_account */ buffer_put_int(&buffer, sshpam_account_status); buffer_put_int(&buffer, sshpam_authctxt->force_pwchange); /* Export any environment strings set in child */ for(i = 0; environ[i] != NULL; i++) ; /* Count */ buffer_put_int(&buffer, i); for(i = 0; environ[i] != NULL; i++) buffer_put_cstring(&buffer, environ[i]); /* Export any environment strings set by PAM in child */ env_from_pam = pam_getenvlist(sshpam_handle); for(i = 0; env_from_pam != NULL && env_from_pam[i] != NULL; i++) ; /* Count */ buffer_put_int(&buffer, i); for(i = 0; env_from_pam != NULL && env_from_pam[i] != NULL; i++) buffer_put_cstring(&buffer, env_from_pam[i]); #endif /* UNSUPPORTED_POSIX_THREADS_HACK */ /* XXX - can't do much about an error here */ ssh_msg_send(ctxt->pam_csock, sshpam_err, &buffer); buffer_free(&buffer); pthread_exit(NULL); auth_fail: buffer_put_cstring(&buffer, pam_strerror(sshpam_handle, sshpam_err)); /* XXX - can't do much about an error here */ if (sshpam_err == PAM_ACCT_EXPIRED) ssh_msg_send(ctxt->pam_csock, PAM_ACCT_EXPIRED, &buffer); else ssh_msg_send(ctxt->pam_csock, PAM_AUTH_ERR, &buffer); buffer_free(&buffer); pthread_exit(NULL); return (NULL); /* Avoid warning for non-pthread case */ } void sshpam_thread_cleanup(void) { struct pam_ctxt *ctxt = cleanup_ctxt; debug3("PAM: %s entering", __func__); if (ctxt != NULL && ctxt->pam_thread != 0) { pthread_cancel(ctxt->pam_thread); pthread_join(ctxt->pam_thread, NULL); close(ctxt->pam_psock); close(ctxt->pam_csock); memset(ctxt, 0, sizeof(*ctxt)); cleanup_ctxt = NULL; } } static int sshpam_null_conv(int n, sshpam_const struct pam_message **msg, struct pam_response **resp, void *data) { debug3("PAM: %s entering, %d messages", __func__, n); return (PAM_CONV_ERR); } static struct pam_conv null_conv = { sshpam_null_conv, NULL }; static int sshpam_store_conv(int n, sshpam_const struct pam_message **msg, struct pam_response **resp, void *data) { struct pam_response *reply; int i; size_t len; debug3("PAM: %s called with %d messages", __func__, n); *resp = NULL; if (n <= 0 || n > PAM_MAX_NUM_MSG) return (PAM_CONV_ERR); if ((reply = calloc(n, sizeof(*reply))) == NULL) return (PAM_CONV_ERR); for (i = 0; i < n; ++i) { switch (PAM_MSG_MEMBER(msg, i, msg_style)) { case PAM_ERROR_MSG: case PAM_TEXT_INFO: len = strlen(PAM_MSG_MEMBER(msg, i, msg)); buffer_append(&loginmsg, PAM_MSG_MEMBER(msg, i, msg), len); buffer_append(&loginmsg, "\n", 1 ); reply[i].resp_retcode = PAM_SUCCESS; break; default: goto fail; } } *resp = reply; return (PAM_SUCCESS); fail: for(i = 0; i < n; i++) { free(reply[i].resp); } free(reply); return (PAM_CONV_ERR); } static struct pam_conv store_conv = { sshpam_store_conv, NULL }; void sshpam_cleanup(void) { if (sshpam_handle == NULL || (use_privsep && !mm_is_monitor())) return; debug("PAM: cleanup"); pam_set_item(sshpam_handle, PAM_CONV, (const void *)&null_conv); if (sshpam_session_open) { debug("PAM: closing session"); pam_close_session(sshpam_handle, PAM_SILENT); sshpam_session_open = 0; } if (sshpam_cred_established) { debug("PAM: deleting credentials"); pam_setcred(sshpam_handle, PAM_DELETE_CRED); sshpam_cred_established = 0; } sshpam_authenticated = 0; pam_end(sshpam_handle, sshpam_err); sshpam_handle = NULL; } static int sshpam_init(Authctxt *authctxt) { extern char *__progname; const char *pam_rhost, *pam_user, *user = authctxt->user; const char **ptr_pam_user = &pam_user; if (sshpam_handle != NULL) { /* We already have a PAM context; check if the user matches */ sshpam_err = pam_get_item(sshpam_handle, PAM_USER, (sshpam_const void **)ptr_pam_user); if (sshpam_err == PAM_SUCCESS && strcmp(user, pam_user) == 0) return (0); pam_end(sshpam_handle, sshpam_err); sshpam_handle = NULL; } debug("PAM: initializing for \"%s\"", user); sshpam_err = pam_start(SSHD_PAM_SERVICE, user, &store_conv, &sshpam_handle); sshpam_authctxt = authctxt; if (sshpam_err != PAM_SUCCESS) { pam_end(sshpam_handle, sshpam_err); sshpam_handle = NULL; return (-1); } pam_rhost = get_remote_name_or_ip(utmp_len, options.use_dns); debug("PAM: setting PAM_RHOST to \"%s\"", pam_rhost); sshpam_err = pam_set_item(sshpam_handle, PAM_RHOST, pam_rhost); if (sshpam_err != PAM_SUCCESS) { pam_end(sshpam_handle, sshpam_err); sshpam_handle = NULL; return (-1); } #ifdef PAM_TTY_KLUDGE /* * Some silly PAM modules (e.g. pam_time) require a TTY to operate. * sshd doesn't set the tty until too late in the auth process and * may not even set one (for tty-less connections) */ debug("PAM: setting PAM_TTY to \"ssh\""); sshpam_err = pam_set_item(sshpam_handle, PAM_TTY, "ssh"); if (sshpam_err != PAM_SUCCESS) { pam_end(sshpam_handle, sshpam_err); sshpam_handle = NULL; return (-1); } #endif return (0); } static void * sshpam_init_ctx(Authctxt *authctxt) { struct pam_ctxt *ctxt; int socks[2]; debug3("PAM: %s entering", __func__); /* * Refuse to start if we don't have PAM enabled or do_pam_account * has previously failed. */ if (!options.use_pam || sshpam_account_status == 0) return NULL; /* Initialize PAM */ if (sshpam_init(authctxt) == -1) { error("PAM: initialization failed"); return (NULL); } ctxt = xcalloc(1, sizeof *ctxt); /* Start the authentication thread */ if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, socks) == -1) { error("PAM: failed create sockets: %s", strerror(errno)); free(ctxt); return (NULL); } ctxt->pam_psock = socks[0]; ctxt->pam_csock = socks[1]; if (pthread_create(&ctxt->pam_thread, NULL, sshpam_thread, ctxt) == -1) { error("PAM: failed to start authentication thread: %s", strerror(errno)); close(socks[0]); close(socks[1]); free(ctxt); return (NULL); } cleanup_ctxt = ctxt; return (ctxt); } static int sshpam_query(void *ctx, char **name, char **info, u_int *num, char ***prompts, u_int **echo_on) { Buffer buffer; struct pam_ctxt *ctxt = ctx; size_t plen; u_char type; char *msg; size_t len, mlen; debug3("PAM: %s entering", __func__); buffer_init(&buffer); *name = xstrdup(""); *info = xstrdup(""); *prompts = xmalloc(sizeof(char *)); **prompts = NULL; plen = 0; *echo_on = xmalloc(sizeof(u_int)); while (ssh_msg_recv(ctxt->pam_psock, &buffer) == 0) { type = buffer_get_char(&buffer); msg = buffer_get_string(&buffer, NULL); mlen = strlen(msg); switch (type) { case PAM_PROMPT_ECHO_ON: case PAM_PROMPT_ECHO_OFF: *num = 1; len = plen + mlen + 1; **prompts = xreallocarray(**prompts, 1, len); strlcpy(**prompts + plen, msg, len - plen); plen += mlen; **echo_on = (type == PAM_PROMPT_ECHO_ON); free(msg); return (0); case PAM_ERROR_MSG: case PAM_TEXT_INFO: /* accumulate messages */ len = plen + mlen + 2; **prompts = xreallocarray(**prompts, 1, len); strlcpy(**prompts + plen, msg, len - plen); plen += mlen; strlcat(**prompts + plen, "\n", len - plen); plen++; free(msg); break; case PAM_ACCT_EXPIRED: sshpam_account_status = 0; /* FALLTHROUGH */ case PAM_AUTH_ERR: debug3("PAM: %s", pam_strerror(sshpam_handle, type)); if (**prompts != NULL && strlen(**prompts) != 0) { *info = **prompts; **prompts = NULL; *num = 0; **echo_on = 0; ctxt->pam_done = -1; free(msg); return 0; } /* FALLTHROUGH */ case PAM_SUCCESS: if (**prompts != NULL) { /* drain any accumulated messages */ debug("PAM: %s", **prompts); buffer_append(&loginmsg, **prompts, strlen(**prompts)); free(**prompts); **prompts = NULL; } if (type == PAM_SUCCESS) { if (!sshpam_authctxt->valid || (sshpam_authctxt->pw->pw_uid == 0 && options.permit_root_login != PERMIT_YES)) fatal("Internal error: PAM auth " "succeeded when it should have " "failed"); import_environments(&buffer); *num = 0; **echo_on = 0; ctxt->pam_done = 1; free(msg); return (0); } + BLACKLIST_NOTIFY(BLACKLIST_AUTH_FAIL); error("PAM: %s for %s%.100s from %.100s", msg, sshpam_authctxt->valid ? "" : "illegal user ", sshpam_authctxt->user, get_remote_name_or_ip(utmp_len, options.use_dns)); /* FALLTHROUGH */ default: *num = 0; **echo_on = 0; free(msg); ctxt->pam_done = -1; return (-1); } } return (-1); } /* XXX - see also comment in auth-chall.c:verify_response */ static int sshpam_respond(void *ctx, u_int num, char **resp) { Buffer buffer; struct pam_ctxt *ctxt = ctx; debug2("PAM: %s entering, %u responses", __func__, num); switch (ctxt->pam_done) { case 1: sshpam_authenticated = 1; return (0); case 0: break; default: return (-1); } if (num != 1) { error("PAM: expected one response, got %u", num); return (-1); } buffer_init(&buffer); if (sshpam_authctxt->valid && (sshpam_authctxt->pw->pw_uid != 0 || options.permit_root_login == PERMIT_YES)) buffer_put_cstring(&buffer, *resp); else buffer_put_cstring(&buffer, badpw); if (ssh_msg_send(ctxt->pam_psock, PAM_AUTHTOK, &buffer) == -1) { buffer_free(&buffer); return (-1); } buffer_free(&buffer); return (1); } static void sshpam_free_ctx(void *ctxtp) { struct pam_ctxt *ctxt = ctxtp; debug3("PAM: %s entering", __func__); sshpam_thread_cleanup(); free(ctxt); /* * We don't call sshpam_cleanup() here because we may need the PAM * handle at a later stage, e.g. when setting up a session. It's * still on the cleanup list, so pam_end() *will* be called before * the server process terminates. */ } KbdintDevice sshpam_device = { "pam", sshpam_init_ctx, sshpam_query, sshpam_respond, sshpam_free_ctx }; KbdintDevice mm_sshpam_device = { "pam", mm_sshpam_init_ctx, mm_sshpam_query, mm_sshpam_respond, mm_sshpam_free_ctx }; /* * This replaces auth-pam.c */ void start_pam(Authctxt *authctxt) { if (!options.use_pam) fatal("PAM: initialisation requested when UsePAM=no"); if (sshpam_init(authctxt) == -1) fatal("PAM: initialisation failed"); } void finish_pam(void) { sshpam_cleanup(); } u_int do_pam_account(void) { debug("%s: called", __func__); if (sshpam_account_status != -1) return (sshpam_account_status); sshpam_err = pam_acct_mgmt(sshpam_handle, 0); debug3("PAM: %s pam_acct_mgmt = %d (%s)", __func__, sshpam_err, pam_strerror(sshpam_handle, sshpam_err)); if (sshpam_err != PAM_SUCCESS && sshpam_err != PAM_NEW_AUTHTOK_REQD) { sshpam_account_status = 0; return (sshpam_account_status); } if (sshpam_err == PAM_NEW_AUTHTOK_REQD) sshpam_password_change_required(1); sshpam_account_status = 1; return (sshpam_account_status); } void do_pam_set_tty(const char *tty) { if (tty != NULL) { debug("PAM: setting PAM_TTY to \"%s\"", tty); sshpam_err = pam_set_item(sshpam_handle, PAM_TTY, tty); if (sshpam_err != PAM_SUCCESS) fatal("PAM: failed to set PAM_TTY: %s", pam_strerror(sshpam_handle, sshpam_err)); } } void do_pam_setcred(int init) { sshpam_err = pam_set_item(sshpam_handle, PAM_CONV, (const void *)&store_conv); if (sshpam_err != PAM_SUCCESS) fatal("PAM: failed to set PAM_CONV: %s", pam_strerror(sshpam_handle, sshpam_err)); if (init) { debug("PAM: establishing credentials"); sshpam_err = pam_setcred(sshpam_handle, PAM_ESTABLISH_CRED); } else { debug("PAM: reinitializing credentials"); sshpam_err = pam_setcred(sshpam_handle, PAM_REINITIALIZE_CRED); } if (sshpam_err == PAM_SUCCESS) { sshpam_cred_established = 1; return; } if (sshpam_authenticated) fatal("PAM: pam_setcred(): %s", pam_strerror(sshpam_handle, sshpam_err)); else debug("PAM: pam_setcred(): %s", pam_strerror(sshpam_handle, sshpam_err)); } static int sshpam_tty_conv(int n, sshpam_const struct pam_message **msg, struct pam_response **resp, void *data) { char input[PAM_MAX_MSG_SIZE]; struct pam_response *reply; int i; debug3("PAM: %s called with %d messages", __func__, n); *resp = NULL; if (n <= 0 || n > PAM_MAX_NUM_MSG || !isatty(STDIN_FILENO)) return (PAM_CONV_ERR); if ((reply = calloc(n, sizeof(*reply))) == NULL) return (PAM_CONV_ERR); for (i = 0; i < n; ++i) { switch (PAM_MSG_MEMBER(msg, i, msg_style)) { case PAM_PROMPT_ECHO_OFF: reply[i].resp = read_passphrase(PAM_MSG_MEMBER(msg, i, msg), RP_ALLOW_STDIN); reply[i].resp_retcode = PAM_SUCCESS; break; case PAM_PROMPT_ECHO_ON: fprintf(stderr, "%s\n", PAM_MSG_MEMBER(msg, i, msg)); if (fgets(input, sizeof input, stdin) == NULL) input[0] = '\0'; if ((reply[i].resp = strdup(input)) == NULL) goto fail; reply[i].resp_retcode = PAM_SUCCESS; break; case PAM_ERROR_MSG: case PAM_TEXT_INFO: fprintf(stderr, "%s\n", PAM_MSG_MEMBER(msg, i, msg)); reply[i].resp_retcode = PAM_SUCCESS; break; default: goto fail; } } *resp = reply; return (PAM_SUCCESS); fail: for(i = 0; i < n; i++) { free(reply[i].resp); } free(reply); return (PAM_CONV_ERR); } static struct pam_conv tty_conv = { sshpam_tty_conv, NULL }; /* * XXX this should be done in the authentication phase, but ssh1 doesn't * support that */ void do_pam_chauthtok(void) { if (use_privsep) fatal("Password expired (unable to change with privsep)"); sshpam_err = pam_set_item(sshpam_handle, PAM_CONV, (const void *)&tty_conv); if (sshpam_err != PAM_SUCCESS) fatal("PAM: failed to set PAM_CONV: %s", pam_strerror(sshpam_handle, sshpam_err)); debug("PAM: changing password"); sshpam_err = pam_chauthtok(sshpam_handle, PAM_CHANGE_EXPIRED_AUTHTOK); if (sshpam_err != PAM_SUCCESS) fatal("PAM: pam_chauthtok(): %s", pam_strerror(sshpam_handle, sshpam_err)); } void do_pam_session(void) { debug3("PAM: opening session"); sshpam_err = pam_set_item(sshpam_handle, PAM_CONV, (const void *)&store_conv); if (sshpam_err != PAM_SUCCESS) fatal("PAM: failed to set PAM_CONV: %s", pam_strerror(sshpam_handle, sshpam_err)); sshpam_err = pam_open_session(sshpam_handle, 0); if (sshpam_err == PAM_SUCCESS) sshpam_session_open = 1; else { sshpam_session_open = 0; disable_forwarding(); error("PAM: pam_open_session(): %s", pam_strerror(sshpam_handle, sshpam_err)); } } int is_pam_session_open(void) { return sshpam_session_open; } /* * Set a PAM environment string. We need to do this so that the session * modules can handle things like Kerberos/GSI credentials that appear * during the ssh authentication process. */ int do_pam_putenv(char *name, char *value) { int ret = 1; #ifdef HAVE_PAM_PUTENV char *compound; size_t len; len = strlen(name) + strlen(value) + 2; compound = xmalloc(len); snprintf(compound, len, "%s=%s", name, value); ret = pam_putenv(sshpam_handle, compound); free(compound); #endif return (ret); } char ** fetch_pam_child_environment(void) { return sshpam_env; } char ** fetch_pam_environment(void) { return (pam_getenvlist(sshpam_handle)); } void free_pam_environment(char **env) { char **envp; if (env == NULL) return; for (envp = env; *envp; envp++) free(*envp); free(env); } /* * "Blind" conversation function for password authentication. Assumes that * echo-off prompts are for the password and stores messages for later * display. */ static int sshpam_passwd_conv(int n, sshpam_const struct pam_message **msg, struct pam_response **resp, void *data) { struct pam_response *reply; int i; size_t len; debug3("PAM: %s called with %d messages", __func__, n); *resp = NULL; if (n <= 0 || n > PAM_MAX_NUM_MSG) return (PAM_CONV_ERR); if ((reply = calloc(n, sizeof(*reply))) == NULL) return (PAM_CONV_ERR); for (i = 0; i < n; ++i) { switch (PAM_MSG_MEMBER(msg, i, msg_style)) { case PAM_PROMPT_ECHO_OFF: if (sshpam_password == NULL) goto fail; if ((reply[i].resp = strdup(sshpam_password)) == NULL) goto fail; reply[i].resp_retcode = PAM_SUCCESS; break; case PAM_ERROR_MSG: case PAM_TEXT_INFO: len = strlen(PAM_MSG_MEMBER(msg, i, msg)); if (len > 0) { buffer_append(&loginmsg, PAM_MSG_MEMBER(msg, i, msg), len); buffer_append(&loginmsg, "\n", 1); } if ((reply[i].resp = strdup("")) == NULL) goto fail; reply[i].resp_retcode = PAM_SUCCESS; break; default: goto fail; } } *resp = reply; return (PAM_SUCCESS); fail: for(i = 0; i < n; i++) { free(reply[i].resp); } free(reply); return (PAM_CONV_ERR); } static struct pam_conv passwd_conv = { sshpam_passwd_conv, NULL }; /* * Attempt password authentication via PAM */ int sshpam_auth_passwd(Authctxt *authctxt, const char *password) { int flags = (options.permit_empty_passwd == 0 ? PAM_DISALLOW_NULL_AUTHTOK : 0); if (!options.use_pam || sshpam_handle == NULL) fatal("PAM: %s called when PAM disabled or failed to " "initialise.", __func__); sshpam_password = password; sshpam_authctxt = authctxt; /* * If the user logging in is invalid, or is root but is not permitted * by PermitRootLogin, use an invalid password to prevent leaking * information via timing (eg if the PAM config has a delay on fail). */ if (!authctxt->valid || (authctxt->pw->pw_uid == 0 && options.permit_root_login != PERMIT_YES)) sshpam_password = badpw; sshpam_err = pam_set_item(sshpam_handle, PAM_CONV, (const void *)&passwd_conv); if (sshpam_err != PAM_SUCCESS) fatal("PAM: %s: failed to set PAM_CONV: %s", __func__, pam_strerror(sshpam_handle, sshpam_err)); sshpam_err = pam_authenticate(sshpam_handle, flags); sshpam_password = NULL; if (sshpam_err == PAM_SUCCESS && authctxt->valid) { debug("PAM: password authentication accepted for %.100s", authctxt->user); return 1; } else { debug("PAM: password authentication failed for %.100s: %s", authctxt->valid ? authctxt->user : "an illegal user", pam_strerror(sshpam_handle, sshpam_err)); return 0; } } #endif /* USE_PAM */ Index: projects/clang390-import/crypto/openssh/auth.c =================================================================== --- projects/clang390-import/crypto/openssh/auth.c (revision 305080) +++ projects/clang390-import/crypto/openssh/auth.c (revision 305081) @@ -1,776 +1,781 @@ /* $OpenBSD: auth.c,v 1.113 2015/08/21 03:42:19 djm Exp $ */ /* * Copyright (c) 2000 Markus Friedl. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "includes.h" __RCSID("$FreeBSD$"); #include #include #include #include #include #ifdef HAVE_PATHS_H # include #endif #include #ifdef HAVE_LOGIN_H #include #endif #ifdef USE_SHADOW #include #endif #ifdef HAVE_LIBGEN_H #include #endif #include #include #include #include #include #include "xmalloc.h" #include "match.h" #include "groupaccess.h" #include "log.h" #include "buffer.h" #include "misc.h" #include "servconf.h" #include "key.h" #include "hostfile.h" #include "auth.h" #include "auth-options.h" #include "canohost.h" #include "uidswap.h" #include "packet.h" #include "loginrec.h" #ifdef GSSAPI #include "ssh-gss.h" #endif #include "authfile.h" #include "monitor_wrap.h" #include "authfile.h" #include "ssherr.h" #include "compat.h" +#include "blacklist_client.h" /* import */ extern ServerOptions options; extern int use_privsep; extern Buffer loginmsg; extern struct passwd *privsep_pw; /* Debugging messages */ Buffer auth_debug; int auth_debug_init; /* * Check if the user is allowed to log in via ssh. If user is listed * in DenyUsers or one of user's groups is listed in DenyGroups, false * will be returned. If AllowUsers isn't empty and user isn't listed * there, or if AllowGroups isn't empty and one of user's groups isn't * listed there, false will be returned. * If the user's shell is not executable, false will be returned. * Otherwise true is returned. */ int allowed_user(struct passwd * pw) { struct stat st; const char *hostname = NULL, *ipaddr = NULL, *passwd = NULL; u_int i; #ifdef USE_SHADOW struct spwd *spw = NULL; #endif /* Shouldn't be called if pw is NULL, but better safe than sorry... */ if (!pw || !pw->pw_name) return 0; #ifdef USE_SHADOW if (!options.use_pam) spw = getspnam(pw->pw_name); #ifdef HAS_SHADOW_EXPIRE if (!options.use_pam && spw != NULL && auth_shadow_acctexpired(spw)) return 0; #endif /* HAS_SHADOW_EXPIRE */ #endif /* USE_SHADOW */ /* grab passwd field for locked account check */ passwd = pw->pw_passwd; #ifdef USE_SHADOW if (spw != NULL) #ifdef USE_LIBIAF passwd = get_iaf_password(pw); #else passwd = spw->sp_pwdp; #endif /* USE_LIBIAF */ #endif /* check for locked account */ if (!options.use_pam && passwd && *passwd) { int locked = 0; #ifdef LOCKED_PASSWD_STRING if (strcmp(passwd, LOCKED_PASSWD_STRING) == 0) locked = 1; #endif #ifdef LOCKED_PASSWD_PREFIX if (strncmp(passwd, LOCKED_PASSWD_PREFIX, strlen(LOCKED_PASSWD_PREFIX)) == 0) locked = 1; #endif #ifdef LOCKED_PASSWD_SUBSTR if (strstr(passwd, LOCKED_PASSWD_SUBSTR)) locked = 1; #endif #ifdef USE_LIBIAF free((void *) passwd); #endif /* USE_LIBIAF */ if (locked) { logit("User %.100s not allowed because account is locked", pw->pw_name); return 0; } } /* * Deny if shell does not exist or is not executable unless we * are chrooting. */ if (options.chroot_directory == NULL || strcasecmp(options.chroot_directory, "none") == 0) { char *shell = xstrdup((pw->pw_shell[0] == '\0') ? _PATH_BSHELL : pw->pw_shell); /* empty = /bin/sh */ if (stat(shell, &st) != 0) { logit("User %.100s not allowed because shell %.100s " "does not exist", pw->pw_name, shell); free(shell); return 0; } if (S_ISREG(st.st_mode) == 0 || (st.st_mode & (S_IXOTH|S_IXUSR|S_IXGRP)) == 0) { logit("User %.100s not allowed because shell %.100s " "is not executable", pw->pw_name, shell); free(shell); return 0; } free(shell); } if (options.num_deny_users > 0 || options.num_allow_users > 0 || options.num_deny_groups > 0 || options.num_allow_groups > 0) { hostname = get_canonical_hostname(options.use_dns); ipaddr = get_remote_ipaddr(); } /* Return false if user is listed in DenyUsers */ if (options.num_deny_users > 0) { for (i = 0; i < options.num_deny_users; i++) if (match_user(pw->pw_name, hostname, ipaddr, options.deny_users[i])) { logit("User %.100s from %.100s not allowed " "because listed in DenyUsers", pw->pw_name, hostname); return 0; } } /* Return false if AllowUsers isn't empty and user isn't listed there */ if (options.num_allow_users > 0) { for (i = 0; i < options.num_allow_users; i++) if (match_user(pw->pw_name, hostname, ipaddr, options.allow_users[i])) break; /* i < options.num_allow_users iff we break for loop */ if (i >= options.num_allow_users) { logit("User %.100s from %.100s not allowed because " "not listed in AllowUsers", pw->pw_name, hostname); return 0; } } if (options.num_deny_groups > 0 || options.num_allow_groups > 0) { /* Get the user's group access list (primary and supplementary) */ if (ga_init(pw->pw_name, pw->pw_gid) == 0) { logit("User %.100s from %.100s not allowed because " "not in any group", pw->pw_name, hostname); return 0; } /* Return false if one of user's groups is listed in DenyGroups */ if (options.num_deny_groups > 0) if (ga_match(options.deny_groups, options.num_deny_groups)) { ga_free(); logit("User %.100s from %.100s not allowed " "because a group is listed in DenyGroups", pw->pw_name, hostname); return 0; } /* * Return false if AllowGroups isn't empty and one of user's groups * isn't listed there */ if (options.num_allow_groups > 0) if (!ga_match(options.allow_groups, options.num_allow_groups)) { ga_free(); logit("User %.100s from %.100s not allowed " "because none of user's groups are listed " "in AllowGroups", pw->pw_name, hostname); return 0; } ga_free(); } #ifdef CUSTOM_SYS_AUTH_ALLOWED_USER if (!sys_auth_allowed_user(pw, &loginmsg)) return 0; #endif /* We found no reason not to let this user try to log on... */ return 1; } void auth_info(Authctxt *authctxt, const char *fmt, ...) { va_list ap; int i; free(authctxt->info); authctxt->info = NULL; va_start(ap, fmt); i = vasprintf(&authctxt->info, fmt, ap); va_end(ap); if (i < 0 || authctxt->info == NULL) fatal("vasprintf failed"); } void auth_log(Authctxt *authctxt, int authenticated, int partial, const char *method, const char *submethod) { void (*authlog) (const char *fmt,...) = verbose; char *authmsg; if (use_privsep && !mm_is_monitor() && !authctxt->postponed) return; /* Raise logging level */ if (authenticated == 1 || !authctxt->valid || authctxt->failures >= options.max_authtries / 2 || strcmp(method, "password") == 0) authlog = logit; if (authctxt->postponed) authmsg = "Postponed"; else if (partial) authmsg = "Partial"; - else + else { authmsg = authenticated ? "Accepted" : "Failed"; + BLACKLIST_NOTIFY(authenticated ? + BLACKLIST_AUTH_OK : BLACKLIST_AUTH_FAIL); + } authlog("%s %s%s%s for %s%.100s from %.200s port %d %s%s%s", authmsg, method, submethod != NULL ? "/" : "", submethod == NULL ? "" : submethod, authctxt->valid ? "" : "invalid user ", authctxt->user, get_remote_ipaddr(), get_remote_port(), compat20 ? "ssh2" : "ssh1", authctxt->info != NULL ? ": " : "", authctxt->info != NULL ? authctxt->info : ""); free(authctxt->info); authctxt->info = NULL; #ifdef CUSTOM_FAILED_LOGIN if (authenticated == 0 && !authctxt->postponed && (strcmp(method, "password") == 0 || strncmp(method, "keyboard-interactive", 20) == 0 || strcmp(method, "challenge-response") == 0)) record_failed_login(authctxt->user, get_canonical_hostname(options.use_dns), "ssh"); # ifdef WITH_AIXAUTHENTICATE if (authenticated) sys_auth_record_login(authctxt->user, get_canonical_hostname(options.use_dns), "ssh", &loginmsg); # endif #endif #ifdef SSH_AUDIT_EVENTS if (authenticated == 0 && !authctxt->postponed) audit_event(audit_classify_auth(method)); #endif } void auth_maxtries_exceeded(Authctxt *authctxt) { error("maximum authentication attempts exceeded for " "%s%.100s from %.200s port %d %s", authctxt->valid ? "" : "invalid user ", authctxt->user, get_remote_ipaddr(), get_remote_port(), compat20 ? "ssh2" : "ssh1"); packet_disconnect("Too many authentication failures"); /* NOTREACHED */ } /* * Check whether root logins are disallowed. */ int auth_root_allowed(const char *method) { switch (options.permit_root_login) { case PERMIT_YES: return 1; case PERMIT_NO_PASSWD: if (strcmp(method, "publickey") == 0 || strcmp(method, "hostbased") == 0 || strcmp(method, "gssapi-with-mic") == 0) return 1; break; case PERMIT_FORCED_ONLY: if (forced_command) { logit("Root login accepted for forced command."); return 1; } break; } logit("ROOT LOGIN REFUSED FROM %.200s", get_remote_ipaddr()); return 0; } /* * Given a template and a passwd structure, build a filename * by substituting % tokenised options. Currently, %% becomes '%', * %h becomes the home directory and %u the username. * * This returns a buffer allocated by xmalloc. */ char * expand_authorized_keys(const char *filename, struct passwd *pw) { char *file, ret[PATH_MAX]; int i; file = percent_expand(filename, "h", pw->pw_dir, "u", pw->pw_name, (char *)NULL); /* * Ensure that filename starts anchored. If not, be backward * compatible and prepend the '%h/' */ if (*file == '/') return (file); i = snprintf(ret, sizeof(ret), "%s/%s", pw->pw_dir, file); if (i < 0 || (size_t)i >= sizeof(ret)) fatal("expand_authorized_keys: path too long"); free(file); return (xstrdup(ret)); } char * authorized_principals_file(struct passwd *pw) { if (options.authorized_principals_file == NULL) return NULL; return expand_authorized_keys(options.authorized_principals_file, pw); } /* return ok if key exists in sysfile or userfile */ HostStatus check_key_in_hostfiles(struct passwd *pw, Key *key, const char *host, const char *sysfile, const char *userfile) { char *user_hostfile; struct stat st; HostStatus host_status; struct hostkeys *hostkeys; const struct hostkey_entry *found; hostkeys = init_hostkeys(); load_hostkeys(hostkeys, host, sysfile); if (userfile != NULL) { user_hostfile = tilde_expand_filename(userfile, pw->pw_uid); if (options.strict_modes && (stat(user_hostfile, &st) == 0) && ((st.st_uid != 0 && st.st_uid != pw->pw_uid) || (st.st_mode & 022) != 0)) { logit("Authentication refused for %.100s: " "bad owner or modes for %.200s", pw->pw_name, user_hostfile); auth_debug_add("Ignored %.200s: bad ownership or modes", user_hostfile); } else { temporarily_use_uid(pw); load_hostkeys(hostkeys, host, user_hostfile); restore_uid(); } free(user_hostfile); } host_status = check_key_in_hostkeys(hostkeys, key, &found); if (host_status == HOST_REVOKED) error("WARNING: revoked key for %s attempted authentication", found->host); else if (host_status == HOST_OK) debug("%s: key for %s found at %s:%ld", __func__, found->host, found->file, found->line); else debug("%s: key for host %s not found", __func__, host); free_hostkeys(hostkeys); return host_status; } /* * Check a given path for security. This is defined as all components * of the path to the file must be owned by either the owner of * of the file or root and no directories must be group or world writable. * * XXX Should any specific check be done for sym links ? * * Takes a file name, its stat information (preferably from fstat() to * avoid races), the uid of the expected owner, their home directory and an * error buffer plus max size as arguments. * * Returns 0 on success and -1 on failure */ int auth_secure_path(const char *name, struct stat *stp, const char *pw_dir, uid_t uid, char *err, size_t errlen) { char buf[PATH_MAX], homedir[PATH_MAX]; char *cp; int comparehome = 0; struct stat st; if (realpath(name, buf) == NULL) { snprintf(err, errlen, "realpath %s failed: %s", name, strerror(errno)); return -1; } if (pw_dir != NULL && realpath(pw_dir, homedir) != NULL) comparehome = 1; if (!S_ISREG(stp->st_mode)) { snprintf(err, errlen, "%s is not a regular file", buf); return -1; } if ((!platform_sys_dir_uid(stp->st_uid) && stp->st_uid != uid) || (stp->st_mode & 022) != 0) { snprintf(err, errlen, "bad ownership or modes for file %s", buf); return -1; } /* for each component of the canonical path, walking upwards */ for (;;) { if ((cp = dirname(buf)) == NULL) { snprintf(err, errlen, "dirname() failed"); return -1; } strlcpy(buf, cp, sizeof(buf)); if (stat(buf, &st) < 0 || (!platform_sys_dir_uid(st.st_uid) && st.st_uid != uid) || (st.st_mode & 022) != 0) { snprintf(err, errlen, "bad ownership or modes for directory %s", buf); return -1; } /* If are past the homedir then we can stop */ if (comparehome && strcmp(homedir, buf) == 0) break; /* * dirname should always complete with a "/" path, * but we can be paranoid and check for "." too */ if ((strcmp("/", buf) == 0) || (strcmp(".", buf) == 0)) break; } return 0; } /* * Version of secure_path() that accepts an open file descriptor to * avoid races. * * Returns 0 on success and -1 on failure */ static int secure_filename(FILE *f, const char *file, struct passwd *pw, char *err, size_t errlen) { struct stat st; /* check the open file to avoid races */ if (fstat(fileno(f), &st) < 0) { snprintf(err, errlen, "cannot stat file %s: %s", file, strerror(errno)); return -1; } return auth_secure_path(file, &st, pw->pw_dir, pw->pw_uid, err, errlen); } static FILE * auth_openfile(const char *file, struct passwd *pw, int strict_modes, int log_missing, char *file_type) { char line[1024]; struct stat st; int fd; FILE *f; if ((fd = open(file, O_RDONLY|O_NONBLOCK)) == -1) { if (log_missing || errno != ENOENT) debug("Could not open %s '%s': %s", file_type, file, strerror(errno)); return NULL; } if (fstat(fd, &st) < 0) { close(fd); return NULL; } if (!S_ISREG(st.st_mode)) { logit("User %s %s %s is not a regular file", pw->pw_name, file_type, file); close(fd); return NULL; } unset_nonblock(fd); if ((f = fdopen(fd, "r")) == NULL) { close(fd); return NULL; } if (strict_modes && secure_filename(f, file, pw, line, sizeof(line)) != 0) { fclose(f); logit("Authentication refused: %s", line); auth_debug_add("Ignored %s: %s", file_type, line); return NULL; } return f; } FILE * auth_openkeyfile(const char *file, struct passwd *pw, int strict_modes) { return auth_openfile(file, pw, strict_modes, 1, "authorized keys"); } FILE * auth_openprincipals(const char *file, struct passwd *pw, int strict_modes) { return auth_openfile(file, pw, strict_modes, 0, "authorized principals"); } struct passwd * getpwnamallow(const char *user) { #ifdef HAVE_LOGIN_CAP extern login_cap_t *lc; #ifdef BSD_AUTH auth_session_t *as; #endif #endif struct passwd *pw; struct connection_info *ci = get_connection_info(1, options.use_dns); ci->user = user; parse_server_match_config(&options, ci); #if defined(_AIX) && defined(HAVE_SETAUTHDB) aix_setauthdb(user); #endif pw = getpwnam(user); #if defined(_AIX) && defined(HAVE_SETAUTHDB) aix_restoreauthdb(); #endif #ifdef HAVE_CYGWIN /* * Windows usernames are case-insensitive. To avoid later problems * when trying to match the username, the user is only allowed to * login if the username is given in the same case as stored in the * user database. */ if (pw != NULL && strcmp(user, pw->pw_name) != 0) { logit("Login name %.100s does not match stored username %.100s", user, pw->pw_name); pw = NULL; } #endif if (pw == NULL) { + BLACKLIST_NOTIFY(BLACKLIST_AUTH_FAIL); logit("Invalid user %.100s from %.100s", user, get_remote_ipaddr()); #ifdef CUSTOM_FAILED_LOGIN record_failed_login(user, get_canonical_hostname(options.use_dns), "ssh"); #endif #ifdef SSH_AUDIT_EVENTS audit_event(SSH_INVALID_USER); #endif /* SSH_AUDIT_EVENTS */ return (NULL); } if (!allowed_user(pw)) return (NULL); #ifdef HAVE_LOGIN_CAP if ((lc = login_getpwclass(pw)) == NULL) { debug("unable to get login class: %s", user); return (NULL); } #ifdef BSD_AUTH if ((as = auth_open()) == NULL || auth_setpwd(as, pw) != 0 || auth_approval(as, lc, pw->pw_name, "ssh") <= 0) { debug("Approval failure for %s", user); pw = NULL; } if (as != NULL) auth_close(as); #endif #endif if (pw != NULL) return (pwcopy(pw)); return (NULL); } /* Returns 1 if key is revoked by revoked_keys_file, 0 otherwise */ int auth_key_is_revoked(Key *key) { char *fp = NULL; int r; if (options.revoked_keys_file == NULL) return 0; if ((fp = sshkey_fingerprint(key, options.fingerprint_hash, SSH_FP_DEFAULT)) == NULL) { r = SSH_ERR_ALLOC_FAIL; error("%s: fingerprint key: %s", __func__, ssh_err(r)); goto out; } r = sshkey_check_revoked(key, options.revoked_keys_file); switch (r) { case 0: break; /* not revoked */ case SSH_ERR_KEY_REVOKED: error("Authentication key %s %s revoked by file %s", sshkey_type(key), fp, options.revoked_keys_file); goto out; default: error("Error checking authentication key %s %s in " "revoked keys file %s: %s", sshkey_type(key), fp, options.revoked_keys_file, ssh_err(r)); goto out; } /* Success */ r = 0; out: free(fp); return r == 0 ? 0 : 1; } void auth_debug_add(const char *fmt,...) { char buf[1024]; va_list args; if (!auth_debug_init) return; va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); buffer_put_cstring(&auth_debug, buf); } void auth_debug_send(void) { char *msg; if (!auth_debug_init) return; while (buffer_len(&auth_debug)) { msg = buffer_get_string(&auth_debug, NULL); packet_send_debug("%s", msg); free(msg); } } void auth_debug_reset(void) { if (auth_debug_init) buffer_clear(&auth_debug); else { buffer_init(&auth_debug); auth_debug_init = 1; } } struct passwd * fakepw(void) { static struct passwd fake; memset(&fake, 0, sizeof(fake)); fake.pw_name = "NOUSER"; fake.pw_passwd = "$2a$06$r3.juUaHZDlIbQaO2dS9FuYxL1W9M81R1Tc92PoSNmzvpEqLkLGrK"; #ifdef HAVE_STRUCT_PASSWD_PW_GECOS fake.pw_gecos = "NOUSER"; #endif fake.pw_uid = privsep_pw == NULL ? (uid_t)-1 : privsep_pw->pw_uid; fake.pw_gid = privsep_pw == NULL ? (gid_t)-1 : privsep_pw->pw_gid; #ifdef HAVE_STRUCT_PASSWD_PW_CLASS fake.pw_class = ""; #endif fake.pw_dir = "/nonexist"; fake.pw_shell = "/nonexist"; return (&fake); } Index: projects/clang390-import/crypto/openssh/auth1.c =================================================================== --- projects/clang390-import/crypto/openssh/auth1.c (revision 305080) +++ projects/clang390-import/crypto/openssh/auth1.c (revision 305081) @@ -1,444 +1,447 @@ /* $OpenBSD: auth1.c,v 1.82 2014/07/15 15:54:14 millert Exp $ */ /* * Copyright (c) 1995 Tatu Ylonen , Espoo, Finland * All rights reserved * * As far as I am concerned, the code I have written for this software * can be used freely for any purpose. Any derived versions of this * software must be clearly marked as such, and if the derived work is * incompatible with the protocol description in the RFC file, it must be * called by a name other than "ssh" or "Secure Shell". */ #include "includes.h" #ifdef WITH_SSH1 #include #include #include #include #include #include #include "openbsd-compat/sys-queue.h" #include "xmalloc.h" #include "rsa.h" #include "ssh1.h" #include "packet.h" #include "buffer.h" #include "log.h" #include "misc.h" #include "servconf.h" #include "compat.h" #include "key.h" #include "hostfile.h" #include "auth.h" #include "channels.h" #include "session.h" #include "uidswap.h" #ifdef GSSAPI #include "ssh-gss.h" #endif #include "monitor_wrap.h" #include "buffer.h" +#include "blacklist_client.h" /* import */ extern ServerOptions options; extern Buffer loginmsg; static int auth1_process_password(Authctxt *); static int auth1_process_rsa(Authctxt *); static int auth1_process_rhosts_rsa(Authctxt *); static int auth1_process_tis_challenge(Authctxt *); static int auth1_process_tis_response(Authctxt *); static char *client_user = NULL; /* Used to fill in remote user for PAM */ struct AuthMethod1 { int type; char *name; int *enabled; int (*method)(Authctxt *); }; const struct AuthMethod1 auth1_methods[] = { { SSH_CMSG_AUTH_PASSWORD, "password", &options.password_authentication, auth1_process_password }, { SSH_CMSG_AUTH_RSA, "rsa", &options.rsa_authentication, auth1_process_rsa }, { SSH_CMSG_AUTH_RHOSTS_RSA, "rhosts-rsa", &options.rhosts_rsa_authentication, auth1_process_rhosts_rsa }, { SSH_CMSG_AUTH_TIS, "challenge-response", &options.challenge_response_authentication, auth1_process_tis_challenge }, { SSH_CMSG_AUTH_TIS_RESPONSE, "challenge-response", &options.challenge_response_authentication, auth1_process_tis_response }, { -1, NULL, NULL, NULL} }; static const struct AuthMethod1 *lookup_authmethod1(int type) { int i; for (i = 0; auth1_methods[i].name != NULL; i++) if (auth1_methods[i].type == type) return (&(auth1_methods[i])); return (NULL); } static char * get_authname(int type) { const struct AuthMethod1 *a; static char buf[64]; if ((a = lookup_authmethod1(type)) != NULL) return (a->name); snprintf(buf, sizeof(buf), "bad-auth-msg-%d", type); return (buf); } /*ARGSUSED*/ static int auth1_process_password(Authctxt *authctxt) { int authenticated = 0; char *password; u_int dlen; /* * Read user password. It is in plain text, but was * transmitted over the encrypted channel so it is * not visible to an outside observer. */ password = packet_get_string(&dlen); packet_check_eom(); /* Try authentication with the password. */ authenticated = PRIVSEP(auth_password(authctxt, password)); explicit_bzero(password, dlen); free(password); return (authenticated); } /*ARGSUSED*/ static int auth1_process_rsa(Authctxt *authctxt) { int authenticated = 0; BIGNUM *n; /* RSA authentication requested. */ if ((n = BN_new()) == NULL) fatal("do_authloop: BN_new failed"); packet_get_bignum(n); packet_check_eom(); authenticated = auth_rsa(authctxt, n); BN_clear_free(n); return (authenticated); } /*ARGSUSED*/ static int auth1_process_rhosts_rsa(Authctxt *authctxt) { int keybits, authenticated = 0; u_int bits; Key *client_host_key; u_int ulen; /* * Get client user name. Note that we just have to * trust the client; root on the client machine can * claim to be any user. */ client_user = packet_get_cstring(&ulen); /* Get the client host key. */ client_host_key = key_new(KEY_RSA1); bits = packet_get_int(); packet_get_bignum(client_host_key->rsa->e); packet_get_bignum(client_host_key->rsa->n); keybits = BN_num_bits(client_host_key->rsa->n); if (keybits < 0 || bits != (u_int)keybits) { verbose("Warning: keysize mismatch for client_host_key: " "actual %d, announced %d", BN_num_bits(client_host_key->rsa->n), bits); } packet_check_eom(); authenticated = auth_rhosts_rsa(authctxt, client_user, client_host_key); key_free(client_host_key); auth_info(authctxt, "ruser %.100s", client_user); return (authenticated); } /*ARGSUSED*/ static int auth1_process_tis_challenge(Authctxt *authctxt) { char *challenge; if ((challenge = get_challenge(authctxt)) == NULL) return (0); debug("sending challenge '%s'", challenge); packet_start(SSH_SMSG_AUTH_TIS_CHALLENGE); packet_put_cstring(challenge); free(challenge); packet_send(); packet_write_wait(); return (-1); } /*ARGSUSED*/ static int auth1_process_tis_response(Authctxt *authctxt) { int authenticated = 0; char *response; u_int dlen; response = packet_get_string(&dlen); packet_check_eom(); authenticated = verify_response(authctxt, response); explicit_bzero(response, dlen); free(response); return (authenticated); } /* * read packets, try to authenticate the user and * return only if authentication is successful */ static void do_authloop(Authctxt *authctxt) { int authenticated = 0; int prev = 0, type = 0; const struct AuthMethod1 *meth; debug("Attempting authentication for %s%.100s.", authctxt->valid ? "" : "invalid user ", authctxt->user); /* If the user has no password, accept authentication immediately. */ if (options.permit_empty_passwd && options.password_authentication && #ifdef KRB5 (!options.kerberos_authentication || options.kerberos_or_local_passwd) && #endif PRIVSEP(auth_password(authctxt, ""))) { #ifdef USE_PAM if (options.use_pam && (PRIVSEP(do_pam_account()))) #endif { auth_log(authctxt, 1, 0, "without authentication", NULL); return; } } /* Indicate that authentication is needed. */ packet_start(SSH_SMSG_FAILURE); packet_send(); packet_write_wait(); for (;;) { /* default to fail */ authenticated = 0; /* Get a packet from the client. */ prev = type; type = packet_read(); /* * If we started challenge-response authentication but the * next packet is not a response to our challenge, release * the resources allocated by get_challenge() (which would * normally have been released by verify_response() had we * received such a response) */ if (prev == SSH_CMSG_AUTH_TIS && type != SSH_CMSG_AUTH_TIS_RESPONSE) abandon_challenge_response(authctxt); if (authctxt->failures >= options.max_authtries) goto skip; if ((meth = lookup_authmethod1(type)) == NULL) { logit("Unknown message during authentication: " "type %d", type); goto skip; } if (!*(meth->enabled)) { verbose("%s authentication disabled.", meth->name); goto skip; } authenticated = meth->method(authctxt); if (authenticated == -1) continue; /* "postponed" */ #ifdef BSD_AUTH if (authctxt->as) { auth_close(authctxt->as); authctxt->as = NULL; } #endif if (!authctxt->valid && authenticated) fatal("INTERNAL ERROR: authenticated invalid user %s", authctxt->user); #ifdef _UNICOS if (authenticated && cray_access_denied(authctxt->user)) { authenticated = 0; fatal("Access denied for user %s.",authctxt->user); } #endif /* _UNICOS */ #ifndef HAVE_CYGWIN /* Special handling for root */ if (authenticated && authctxt->pw->pw_uid == 0 && !auth_root_allowed(meth->name)) { authenticated = 0; # ifdef SSH_AUDIT_EVENTS PRIVSEP(audit_event(SSH_LOGIN_ROOT_DENIED)); # endif } #endif #ifdef USE_PAM if (options.use_pam && authenticated && !PRIVSEP(do_pam_account())) { char *msg; size_t len; + BLACKLIST_NOTIFY(BLACKLIST_AUTH_FAIL); error("Access denied for user %s by PAM account " "configuration", authctxt->user); len = buffer_len(&loginmsg); buffer_append(&loginmsg, "\0", 1); msg = buffer_ptr(&loginmsg); /* strip trailing newlines */ if (len > 0) while (len > 0 && msg[--len] == '\n') msg[len] = '\0'; else msg = "Access denied."; packet_disconnect("%s", msg); } #endif skip: /* Log before sending the reply */ auth_log(authctxt, authenticated, 0, get_authname(type), NULL); free(client_user); client_user = NULL; if (authenticated) return; if (++authctxt->failures >= options.max_authtries) { #ifdef SSH_AUDIT_EVENTS PRIVSEP(audit_event(SSH_LOGIN_EXCEED_MAXTRIES)); #endif auth_maxtries_exceeded(authctxt); } packet_start(SSH_SMSG_FAILURE); packet_send(); packet_write_wait(); } } /* * Performs authentication of an incoming connection. Session key has already * been exchanged and encryption is enabled. */ void do_authentication(Authctxt *authctxt) { u_int ulen; char *user, *style = NULL; /* Get the name of the user that we wish to log in as. */ packet_read_expect(SSH_CMSG_USER); /* Get the user name. */ user = packet_get_cstring(&ulen); packet_check_eom(); if ((style = strchr(user, ':')) != NULL) *style++ = '\0'; authctxt->user = user; authctxt->style = style; /* Verify that the user is a valid user. */ if ((authctxt->pw = PRIVSEP(getpwnamallow(user))) != NULL) authctxt->valid = 1; else { debug("do_authentication: invalid user %s", user); authctxt->pw = fakepw(); + BLACKLIST_NOTIFY(BLACKLIST_AUTH_FAIL); } /* Configuration may have changed as a result of Match */ if (options.num_auth_methods != 0) fatal("AuthenticationMethods is not supported with SSH " "protocol 1"); setproctitle("%s%s", authctxt->valid ? user : "unknown", use_privsep ? " [net]" : ""); #ifdef USE_PAM if (options.use_pam) PRIVSEP(start_pam(authctxt)); #endif /* * If we are not running as root, the user must have the same uid as * the server. */ #ifndef HAVE_CYGWIN if (!use_privsep && getuid() != 0 && authctxt->pw && authctxt->pw->pw_uid != getuid()) packet_disconnect("Cannot change user when server not running as root."); #endif /* * Loop until the user has been authenticated or the connection is * closed, do_authloop() returns only if authentication is successful */ do_authloop(authctxt); /* The user has been authenticated and accepted. */ packet_start(SSH_SMSG_SUCCESS); packet_send(); packet_write_wait(); } #endif /* WITH_SSH1 */ Index: projects/clang390-import/crypto/openssh/auth2.c =================================================================== --- projects/clang390-import/crypto/openssh/auth2.c (revision 305080) +++ projects/clang390-import/crypto/openssh/auth2.c (revision 305081) @@ -1,649 +1,651 @@ /* $OpenBSD: auth2.c,v 1.135 2015/01/19 20:07:45 markus Exp $ */ /* * Copyright (c) 2000 Markus Friedl. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "includes.h" __RCSID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include "atomicio.h" #include "xmalloc.h" #include "ssh2.h" #include "packet.h" #include "log.h" #include "buffer.h" #include "misc.h" #include "servconf.h" #include "compat.h" #include "key.h" #include "hostfile.h" #include "auth.h" #include "dispatch.h" #include "pathnames.h" #include "buffer.h" #include "canohost.h" +#include "blacklist_client.h" #ifdef GSSAPI #include "ssh-gss.h" #endif #include "monitor_wrap.h" /* import */ extern ServerOptions options; extern u_char *session_id2; extern u_int session_id2_len; extern Buffer loginmsg; /* methods */ extern Authmethod method_none; extern Authmethod method_pubkey; extern Authmethod method_passwd; extern Authmethod method_kbdint; extern Authmethod method_hostbased; #ifdef GSSAPI extern Authmethod method_gssapi; #endif Authmethod *authmethods[] = { &method_none, &method_pubkey, #ifdef GSSAPI &method_gssapi, #endif &method_passwd, &method_kbdint, &method_hostbased, NULL }; /* protocol */ static int input_service_request(int, u_int32_t, void *); static int input_userauth_request(int, u_int32_t, void *); /* helper */ static Authmethod *authmethod_lookup(Authctxt *, const char *); static char *authmethods_get(Authctxt *authctxt); #define MATCH_NONE 0 /* method or submethod mismatch */ #define MATCH_METHOD 1 /* method matches (no submethod specified) */ #define MATCH_BOTH 2 /* method and submethod match */ #define MATCH_PARTIAL 3 /* method matches, submethod can't be checked */ static int list_starts_with(const char *, const char *, const char *); char * auth2_read_banner(void) { struct stat st; char *banner = NULL; size_t len, n; int fd; if ((fd = open(options.banner, O_RDONLY)) == -1) return (NULL); if (fstat(fd, &st) == -1) { close(fd); return (NULL); } if (st.st_size <= 0 || st.st_size > 1*1024*1024) { close(fd); return (NULL); } len = (size_t)st.st_size; /* truncate */ banner = xmalloc(len + 1); n = atomicio(read, fd, banner, len); close(fd); if (n != len) { free(banner); return (NULL); } banner[n] = '\0'; return (banner); } void userauth_send_banner(const char *msg) { if (datafellows & SSH_BUG_BANNER) return; packet_start(SSH2_MSG_USERAUTH_BANNER); packet_put_cstring(msg); packet_put_cstring(""); /* language, unused */ packet_send(); debug("%s: sent", __func__); } static void userauth_banner(void) { char *banner = NULL; if (options.banner == NULL || (datafellows & SSH_BUG_BANNER) != 0) return; if ((banner = PRIVSEP(auth2_read_banner())) == NULL) goto done; userauth_send_banner(banner); done: free(banner); } /* * loop until authctxt->success == TRUE */ void do_authentication2(Authctxt *authctxt) { dispatch_init(&dispatch_protocol_error); dispatch_set(SSH2_MSG_SERVICE_REQUEST, &input_service_request); dispatch_run(DISPATCH_BLOCK, &authctxt->success, authctxt); } /*ARGSUSED*/ static int input_service_request(int type, u_int32_t seq, void *ctxt) { Authctxt *authctxt = ctxt; u_int len; int acceptit = 0; char *service = packet_get_cstring(&len); packet_check_eom(); if (authctxt == NULL) fatal("input_service_request: no authctxt"); if (strcmp(service, "ssh-userauth") == 0) { if (!authctxt->success) { acceptit = 1; /* now we can handle user-auth requests */ dispatch_set(SSH2_MSG_USERAUTH_REQUEST, &input_userauth_request); } } /* XXX all other service requests are denied */ if (acceptit) { packet_start(SSH2_MSG_SERVICE_ACCEPT); packet_put_cstring(service); packet_send(); packet_write_wait(); } else { debug("bad service request %s", service); packet_disconnect("bad service request %s", service); } free(service); return 0; } /*ARGSUSED*/ static int input_userauth_request(int type, u_int32_t seq, void *ctxt) { Authctxt *authctxt = ctxt; Authmethod *m = NULL; char *user, *service, *method, *style = NULL; int authenticated = 0; #ifdef HAVE_LOGIN_CAP login_cap_t *lc; const char *from_host, *from_ip; from_host = get_canonical_hostname(options.use_dns); from_ip = get_remote_ipaddr(); #endif if (authctxt == NULL) fatal("input_userauth_request: no authctxt"); user = packet_get_cstring(NULL); service = packet_get_cstring(NULL); method = packet_get_cstring(NULL); debug("userauth-request for user %s service %s method %s", user, service, method); debug("attempt %d failures %d", authctxt->attempt, authctxt->failures); if ((style = strchr(user, ':')) != NULL) *style++ = 0; if (authctxt->attempt++ == 0) { /* setup auth context */ authctxt->pw = PRIVSEP(getpwnamallow(user)); authctxt->user = xstrdup(user); if (authctxt->pw && strcmp(service, "ssh-connection")==0) { authctxt->valid = 1; debug2("input_userauth_request: setting up authctxt for %s", user); } else { logit("input_userauth_request: invalid user %s", user); authctxt->pw = fakepw(); + BLACKLIST_NOTIFY(BLACKLIST_AUTH_FAIL); #ifdef SSH_AUDIT_EVENTS PRIVSEP(audit_event(SSH_INVALID_USER)); #endif } #ifdef USE_PAM if (options.use_pam) PRIVSEP(start_pam(authctxt)); #endif setproctitle("%s%s", authctxt->valid ? user : "unknown", use_privsep ? " [net]" : ""); authctxt->service = xstrdup(service); authctxt->style = style ? xstrdup(style) : NULL; if (use_privsep) mm_inform_authserv(service, style); userauth_banner(); if (auth2_setup_methods_lists(authctxt) != 0) packet_disconnect("no authentication methods enabled"); } else if (strcmp(user, authctxt->user) != 0 || strcmp(service, authctxt->service) != 0) { packet_disconnect("Change of username or service not allowed: " "(%s,%s) -> (%s,%s)", authctxt->user, authctxt->service, user, service); } #ifdef HAVE_LOGIN_CAP if (authctxt->pw != NULL) { lc = login_getpwclass(authctxt->pw); if (lc == NULL) lc = login_getclassbyname(NULL, authctxt->pw); if (!auth_hostok(lc, from_host, from_ip)) { logit("Denied connection for %.200s from %.200s [%.200s].", authctxt->pw->pw_name, from_host, from_ip); packet_disconnect("Sorry, you are not allowed to connect."); } if (!auth_timeok(lc, time(NULL))) { logit("LOGIN %.200s REFUSED (TIME) FROM %.200s", authctxt->pw->pw_name, from_host); packet_disconnect("Logins not available right now."); } login_close(lc); lc = NULL; } #endif /* HAVE_LOGIN_CAP */ /* reset state */ auth2_challenge_stop(authctxt); #ifdef GSSAPI /* XXX move to auth2_gssapi_stop() */ dispatch_set(SSH2_MSG_USERAUTH_GSSAPI_TOKEN, NULL); dispatch_set(SSH2_MSG_USERAUTH_GSSAPI_EXCHANGE_COMPLETE, NULL); #endif authctxt->postponed = 0; authctxt->server_caused_failure = 0; /* try to authenticate user */ m = authmethod_lookup(authctxt, method); if (m != NULL && authctxt->failures < options.max_authtries) { debug2("input_userauth_request: try method %s", method); authenticated = m->userauth(authctxt); } userauth_finish(authctxt, authenticated, method, NULL); free(service); free(user); free(method); return 0; } void userauth_finish(Authctxt *authctxt, int authenticated, const char *method, const char *submethod) { char *methods; int partial = 0; if (!authctxt->valid && authenticated) fatal("INTERNAL ERROR: authenticated invalid user %s", authctxt->user); if (authenticated && authctxt->postponed) fatal("INTERNAL ERROR: authenticated and postponed"); /* Special handling for root */ if (authenticated && authctxt->pw->pw_uid == 0 && !auth_root_allowed(method)) { authenticated = 0; #ifdef SSH_AUDIT_EVENTS PRIVSEP(audit_event(SSH_LOGIN_ROOT_DENIED)); #endif } if (authenticated && options.num_auth_methods != 0) { if (!auth2_update_methods_lists(authctxt, method, submethod)) { authenticated = 0; partial = 1; } } /* Log before sending the reply */ auth_log(authctxt, authenticated, partial, method, submethod); if (authctxt->postponed) return; #ifdef USE_PAM if (options.use_pam && authenticated) { if (!PRIVSEP(do_pam_account())) { /* if PAM returned a message, send it to the user */ if (buffer_len(&loginmsg) > 0) { buffer_append(&loginmsg, "\0", 1); userauth_send_banner(buffer_ptr(&loginmsg)); packet_write_wait(); } fatal("Access denied for user %s by PAM account " "configuration", authctxt->user); } } #endif #ifdef _UNICOS if (authenticated && cray_access_denied(authctxt->user)) { authenticated = 0; fatal("Access denied for user %s.", authctxt->user); } #endif /* _UNICOS */ if (authenticated == 1) { /* turn off userauth */ dispatch_set(SSH2_MSG_USERAUTH_REQUEST, &dispatch_protocol_ignore); packet_start(SSH2_MSG_USERAUTH_SUCCESS); packet_send(); packet_write_wait(); /* now we can break out */ authctxt->success = 1; } else { /* Allow initial try of "none" auth without failure penalty */ if (!partial && !authctxt->server_caused_failure && (authctxt->attempt > 1 || strcmp(method, "none") != 0)) authctxt->failures++; if (authctxt->failures >= options.max_authtries) { #ifdef SSH_AUDIT_EVENTS PRIVSEP(audit_event(SSH_LOGIN_EXCEED_MAXTRIES)); #endif auth_maxtries_exceeded(authctxt); } methods = authmethods_get(authctxt); debug3("%s: failure partial=%d next methods=\"%s\"", __func__, partial, methods); packet_start(SSH2_MSG_USERAUTH_FAILURE); packet_put_cstring(methods); packet_put_char(partial); packet_send(); packet_write_wait(); free(methods); } } /* * Checks whether method is allowed by at least one AuthenticationMethods * methods list. Returns 1 if allowed, or no methods lists configured. * 0 otherwise. */ int auth2_method_allowed(Authctxt *authctxt, const char *method, const char *submethod) { u_int i; /* * NB. authctxt->num_auth_methods might be zero as a result of * auth2_setup_methods_lists(), so check the configuration. */ if (options.num_auth_methods == 0) return 1; for (i = 0; i < authctxt->num_auth_methods; i++) { if (list_starts_with(authctxt->auth_methods[i], method, submethod) != MATCH_NONE) return 1; } return 0; } static char * authmethods_get(Authctxt *authctxt) { Buffer b; char *list; u_int i; buffer_init(&b); for (i = 0; authmethods[i] != NULL; i++) { if (strcmp(authmethods[i]->name, "none") == 0) continue; if (authmethods[i]->enabled == NULL || *(authmethods[i]->enabled) == 0) continue; if (!auth2_method_allowed(authctxt, authmethods[i]->name, NULL)) continue; if (buffer_len(&b) > 0) buffer_append(&b, ",", 1); buffer_append(&b, authmethods[i]->name, strlen(authmethods[i]->name)); } buffer_append(&b, "\0", 1); list = xstrdup(buffer_ptr(&b)); buffer_free(&b); return list; } static Authmethod * authmethod_lookup(Authctxt *authctxt, const char *name) { int i; if (name != NULL) for (i = 0; authmethods[i] != NULL; i++) if (authmethods[i]->enabled != NULL && *(authmethods[i]->enabled) != 0 && strcmp(name, authmethods[i]->name) == 0 && auth2_method_allowed(authctxt, authmethods[i]->name, NULL)) return authmethods[i]; debug2("Unrecognized authentication method name: %s", name ? name : "NULL"); return NULL; } /* * Check a comma-separated list of methods for validity. Is need_enable is * non-zero, then also require that the methods are enabled. * Returns 0 on success or -1 if the methods list is invalid. */ int auth2_methods_valid(const char *_methods, int need_enable) { char *methods, *omethods, *method, *p; u_int i, found; int ret = -1; if (*_methods == '\0') { error("empty authentication method list"); return -1; } omethods = methods = xstrdup(_methods); while ((method = strsep(&methods, ",")) != NULL) { for (found = i = 0; !found && authmethods[i] != NULL; i++) { if ((p = strchr(method, ':')) != NULL) *p = '\0'; if (strcmp(method, authmethods[i]->name) != 0) continue; if (need_enable) { if (authmethods[i]->enabled == NULL || *(authmethods[i]->enabled) == 0) { error("Disabled method \"%s\" in " "AuthenticationMethods list \"%s\"", method, _methods); goto out; } } found = 1; break; } if (!found) { error("Unknown authentication method \"%s\" in list", method); goto out; } } ret = 0; out: free(omethods); return ret; } /* * Prune the AuthenticationMethods supplied in the configuration, removing * any methods lists that include disabled methods. Note that this might * leave authctxt->num_auth_methods == 0, even when multiple required auth * has been requested. For this reason, all tests for whether multiple is * enabled should consult options.num_auth_methods directly. */ int auth2_setup_methods_lists(Authctxt *authctxt) { u_int i; if (options.num_auth_methods == 0) return 0; debug3("%s: checking methods", __func__); authctxt->auth_methods = xcalloc(options.num_auth_methods, sizeof(*authctxt->auth_methods)); authctxt->num_auth_methods = 0; for (i = 0; i < options.num_auth_methods; i++) { if (auth2_methods_valid(options.auth_methods[i], 1) != 0) { logit("Authentication methods list \"%s\" contains " "disabled method, skipping", options.auth_methods[i]); continue; } debug("authentication methods list %d: %s", authctxt->num_auth_methods, options.auth_methods[i]); authctxt->auth_methods[authctxt->num_auth_methods++] = xstrdup(options.auth_methods[i]); } if (authctxt->num_auth_methods == 0) { error("No AuthenticationMethods left after eliminating " "disabled methods"); return -1; } return 0; } static int list_starts_with(const char *methods, const char *method, const char *submethod) { size_t l = strlen(method); int match; const char *p; if (strncmp(methods, method, l) != 0) return MATCH_NONE; p = methods + l; match = MATCH_METHOD; if (*p == ':') { if (!submethod) return MATCH_PARTIAL; l = strlen(submethod); p += 1; if (strncmp(submethod, p, l)) return MATCH_NONE; p += l; match = MATCH_BOTH; } if (*p != ',' && *p != '\0') return MATCH_NONE; return match; } /* * Remove method from the start of a comma-separated list of methods. * Returns 0 if the list of methods did not start with that method or 1 * if it did. */ static int remove_method(char **methods, const char *method, const char *submethod) { char *omethods = *methods, *p; size_t l = strlen(method); int match; match = list_starts_with(omethods, method, submethod); if (match != MATCH_METHOD && match != MATCH_BOTH) return 0; p = omethods + l; if (submethod && match == MATCH_BOTH) p += 1 + strlen(submethod); /* include colon */ if (*p == ',') p++; *methods = xstrdup(p); free(omethods); return 1; } /* * Called after successful authentication. Will remove the successful method * from the start of each list in which it occurs. If it was the last method * in any list, then authentication is deemed successful. * Returns 1 if the method completed any authentication list or 0 otherwise. */ int auth2_update_methods_lists(Authctxt *authctxt, const char *method, const char *submethod) { u_int i, found = 0; debug3("%s: updating methods list after \"%s\"", __func__, method); for (i = 0; i < authctxt->num_auth_methods; i++) { if (!remove_method(&(authctxt->auth_methods[i]), method, submethod)) continue; found = 1; if (*authctxt->auth_methods[i] == '\0') { debug2("authentication methods list %d complete", i); return 1; } debug3("authentication methods list %d remaining: \"%s\"", i, authctxt->auth_methods[i]); } /* This should not happen, but would be bad if it did */ if (!found) fatal("%s: method not in AuthenticationMethods", __func__); return 0; } Index: projects/clang390-import/crypto/openssh/blacklist.c =================================================================== --- projects/clang390-import/crypto/openssh/blacklist.c (nonexistent) +++ projects/clang390-import/crypto/openssh/blacklist.c (revision 305081) @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2015 The NetBSD Foundation, Inc. + * Copyright (c) 2016 The FreeBSD Foundation, Inc. + * All rights reserved. + * + * Portions of this software were developed by Kurt Lidl + * under sponsorship from the FreeBSD Foundation. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Christos Zoulas. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "includes.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "ssh.h" +#include "packet.h" +#include "log.h" +#include "misc.h" +#include "servconf.h" +#include "blacklist_client.h" +#include + +static struct blacklist *blstate = NULL; + +/* import */ +extern ServerOptions options; + +/* internal definition from bl.h */ +struct blacklist *bl_create(bool, char *, void (*)(int, const char *, va_list)); + +/* impedence match vsyslog() to sshd's internal logging levels */ +void +im_log(int priority, const char *message, va_list args) +{ + LogLevel imlevel; + + switch (priority) { + case LOG_ERR: + imlevel = SYSLOG_LEVEL_ERROR; + break; + case LOG_DEBUG: + imlevel = SYSLOG_LEVEL_DEBUG1; + break; + case LOG_INFO: + imlevel = SYSLOG_LEVEL_INFO; + break; + default: + imlevel = SYSLOG_LEVEL_DEBUG2; + } + do_log(imlevel, message, args); +} + +void +blacklist_init(void) +{ + + if (options.use_blacklist) + blstate = bl_create(false, NULL, im_log); +} + +void +blacklist_notify(int action) +{ + + if (blstate != NULL && packet_connection_is_on_socket()) + (void)blacklist_r(blstate, action, + packet_get_connection_in(), "ssh"); +} Property changes on: projects/clang390-import/crypto/openssh/blacklist.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/clang390-import/crypto/openssh/blacklist_client.h =================================================================== --- projects/clang390-import/crypto/openssh/blacklist_client.h (nonexistent) +++ projects/clang390-import/crypto/openssh/blacklist_client.h (revision 305081) @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2015 The NetBSD Foundation, Inc. + * Copyright (c) 2016 The FreeBSD Foundation, Inc. + * All rights reserved. + * + * Portions of this software were developed by Kurt Lidl + * under sponsorship from the FreeBSD Foundation. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Christos Zoulas. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef BLACKLIST_CLIENT_H +#define BLACKLIST_CLIENT_H + +enum { + BLACKLIST_AUTH_OK = 0, + BLACKLIST_AUTH_FAIL +}; + +#ifdef USE_BLACKLIST +void blacklist_init(void); +void blacklist_notify(int); + +#define BLACKLIST_INIT() blacklist_init() +#define BLACKLIST_NOTIFY(x) blacklist_notify(x) + +#else + +#define BLACKLIST_INIT() +#define BLACKLIST_NOTIFY(x) + +#endif + + +#endif /* BLACKLIST_CLIENT_H */ Property changes on: projects/clang390-import/crypto/openssh/blacklist_client.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/clang390-import/crypto/openssh/packet.c =================================================================== --- projects/clang390-import/crypto/openssh/packet.c (revision 305080) +++ projects/clang390-import/crypto/openssh/packet.c (revision 305081) @@ -1,2975 +1,2977 @@ /* $OpenBSD: packet.c,v 1.229 2016/02/17 22:20:14 djm Exp $ */ /* * Author: Tatu Ylonen * Copyright (c) 1995 Tatu Ylonen , Espoo, Finland * All rights reserved * This file contains code implementing the packet protocol and communication * with the other side. This same code is used both on client and server side. * * As far as I am concerned, the code I have written for this software * can be used freely for any purpose. Any derived versions of this * software must be clearly marked as such, and if the derived work is * incompatible with the protocol description in the RFC file, it must be * called by a name other than "ssh" or "Secure Shell". * * * SSH2 packet format added by Markus Friedl. * Copyright (c) 2000, 2001 Markus Friedl. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "includes.h" __RCSID("$FreeBSD$"); #include /* MIN roundup */ #include #include "openbsd-compat/sys-queue.h" #include #ifdef HAVE_SYS_TIME_H # include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "buffer.h" /* typedefs XXX */ #include "key.h" /* typedefs XXX */ #include "xmalloc.h" #include "crc32.h" #include "deattack.h" #include "compat.h" #include "ssh1.h" #include "ssh2.h" #include "cipher.h" #include "sshkey.h" #include "kex.h" #include "digest.h" #include "mac.h" #include "log.h" #include "canohost.h" #include "misc.h" #include "channels.h" #include "ssh.h" #include "packet.h" #include "ssherr.h" #include "sshbuf.h" +#include "blacklist_client.h" #ifdef PACKET_DEBUG #define DBG(x) x #else #define DBG(x) #endif #define PACKET_MAX_SIZE (256 * 1024) struct packet_state { u_int32_t seqnr; u_int32_t packets; u_int64_t blocks; u_int64_t bytes; }; struct packet { TAILQ_ENTRY(packet) next; u_char type; struct sshbuf *payload; }; struct session_state { /* * This variable contains the file descriptors used for * communicating with the other side. connection_in is used for * reading; connection_out for writing. These can be the same * descriptor, in which case it is assumed to be a socket. */ int connection_in; int connection_out; /* Protocol flags for the remote side. */ u_int remote_protocol_flags; /* Encryption context for receiving data. Only used for decryption. */ struct sshcipher_ctx receive_context; /* Encryption context for sending data. Only used for encryption. */ struct sshcipher_ctx send_context; /* Buffer for raw input data from the socket. */ struct sshbuf *input; /* Buffer for raw output data going to the socket. */ struct sshbuf *output; /* Buffer for the partial outgoing packet being constructed. */ struct sshbuf *outgoing_packet; /* Buffer for the incoming packet currently being processed. */ struct sshbuf *incoming_packet; /* Scratch buffer for packet compression/decompression. */ struct sshbuf *compression_buffer; /* Incoming/outgoing compression dictionaries */ z_stream compression_in_stream; z_stream compression_out_stream; int compression_in_started; int compression_out_started; int compression_in_failures; int compression_out_failures; /* * Flag indicating whether packet compression/decompression is * enabled. */ int packet_compression; /* default maximum packet size */ u_int max_packet_size; /* Flag indicating whether this module has been initialized. */ int initialized; /* Set to true if the connection is interactive. */ int interactive_mode; /* Set to true if we are the server side. */ int server_side; /* Set to true if we are authenticated. */ int after_authentication; int keep_alive_timeouts; /* The maximum time that we will wait to send or receive a packet */ int packet_timeout_ms; /* Session key information for Encryption and MAC */ struct newkeys *newkeys[MODE_MAX]; struct packet_state p_read, p_send; /* Volume-based rekeying */ u_int64_t max_blocks_in, max_blocks_out, rekey_limit; /* Time-based rekeying */ u_int32_t rekey_interval; /* how often in seconds */ time_t rekey_time; /* time of last rekeying */ /* Session key for protocol v1 */ u_char ssh1_key[SSH_SESSION_KEY_LENGTH]; u_int ssh1_keylen; /* roundup current message to extra_pad bytes */ u_char extra_pad; /* XXX discard incoming data after MAC error */ u_int packet_discard; struct sshmac *packet_discard_mac; /* Used in packet_read_poll2() */ u_int packlen; /* Used in packet_send2 */ int rekeying; /* Used in packet_set_interactive */ int set_interactive_called; /* Used in packet_set_maxsize */ int set_maxsize_called; /* One-off warning about weak ciphers */ int cipher_warning_done; /* SSH1 CRC compensation attack detector */ struct deattack_ctx deattack; TAILQ_HEAD(, packet) outgoing; }; struct ssh * ssh_alloc_session_state(void) { struct ssh *ssh = NULL; struct session_state *state = NULL; if ((ssh = calloc(1, sizeof(*ssh))) == NULL || (state = calloc(1, sizeof(*state))) == NULL || (state->input = sshbuf_new()) == NULL || (state->output = sshbuf_new()) == NULL || (state->outgoing_packet = sshbuf_new()) == NULL || (state->incoming_packet = sshbuf_new()) == NULL) goto fail; TAILQ_INIT(&state->outgoing); TAILQ_INIT(&ssh->private_keys); TAILQ_INIT(&ssh->public_keys); state->connection_in = -1; state->connection_out = -1; state->max_packet_size = 32768; state->packet_timeout_ms = -1; state->p_send.packets = state->p_read.packets = 0; state->initialized = 1; /* * ssh_packet_send2() needs to queue packets until * we've done the initial key exchange. */ state->rekeying = 1; ssh->state = state; return ssh; fail: if (state) { sshbuf_free(state->input); sshbuf_free(state->output); sshbuf_free(state->incoming_packet); sshbuf_free(state->outgoing_packet); free(state); } free(ssh); return NULL; } /* Returns nonzero if rekeying is in progress */ int ssh_packet_is_rekeying(struct ssh *ssh) { return compat20 && (ssh->state->rekeying || (ssh->kex != NULL && ssh->kex->done == 0)); } /* * Sets the descriptors used for communication. Disables encryption until * packet_set_encryption_key is called. */ struct ssh * ssh_packet_set_connection(struct ssh *ssh, int fd_in, int fd_out) { struct session_state *state; const struct sshcipher *none = cipher_by_name("none"); int r; if (none == NULL) { error("%s: cannot load cipher 'none'", __func__); return NULL; } if (ssh == NULL) ssh = ssh_alloc_session_state(); if (ssh == NULL) { error("%s: cound not allocate state", __func__); return NULL; } state = ssh->state; state->connection_in = fd_in; state->connection_out = fd_out; if ((r = cipher_init(&state->send_context, none, (const u_char *)"", 0, NULL, 0, CIPHER_ENCRYPT)) != 0 || (r = cipher_init(&state->receive_context, none, (const u_char *)"", 0, NULL, 0, CIPHER_DECRYPT)) != 0) { error("%s: cipher_init failed: %s", __func__, ssh_err(r)); free(ssh); return NULL; } state->newkeys[MODE_IN] = state->newkeys[MODE_OUT] = NULL; deattack_init(&state->deattack); /* * Cache the IP address of the remote connection for use in error * messages that might be generated after the connection has closed. */ (void)ssh_remote_ipaddr(ssh); return ssh; } void ssh_packet_set_timeout(struct ssh *ssh, int timeout, int count) { struct session_state *state = ssh->state; if (timeout <= 0 || count <= 0) { state->packet_timeout_ms = -1; return; } if ((INT_MAX / 1000) / count < timeout) state->packet_timeout_ms = INT_MAX; else state->packet_timeout_ms = timeout * count * 1000; } int ssh_packet_stop_discard(struct ssh *ssh) { struct session_state *state = ssh->state; int r; if (state->packet_discard_mac) { char buf[1024]; memset(buf, 'a', sizeof(buf)); while (sshbuf_len(state->incoming_packet) < PACKET_MAX_SIZE) if ((r = sshbuf_put(state->incoming_packet, buf, sizeof(buf))) != 0) return r; (void) mac_compute(state->packet_discard_mac, state->p_read.seqnr, sshbuf_ptr(state->incoming_packet), PACKET_MAX_SIZE, NULL, 0); } logit("Finished discarding for %.200s port %d", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh)); return SSH_ERR_MAC_INVALID; } static int ssh_packet_start_discard(struct ssh *ssh, struct sshenc *enc, struct sshmac *mac, u_int packet_length, u_int discard) { struct session_state *state = ssh->state; int r; if (enc == NULL || !cipher_is_cbc(enc->cipher) || (mac && mac->etm)) { if ((r = sshpkt_disconnect(ssh, "Packet corrupt")) != 0) return r; return SSH_ERR_MAC_INVALID; } if (packet_length != PACKET_MAX_SIZE && mac && mac->enabled) state->packet_discard_mac = mac; if (sshbuf_len(state->input) >= discard && (r = ssh_packet_stop_discard(ssh)) != 0) return r; state->packet_discard = discard - sshbuf_len(state->input); return 0; } /* Returns 1 if remote host is connected via socket, 0 if not. */ int ssh_packet_connection_is_on_socket(struct ssh *ssh) { struct session_state *state = ssh->state; struct sockaddr_storage from, to; socklen_t fromlen, tolen; /* filedescriptors in and out are the same, so it's a socket */ if (state->connection_in == state->connection_out) return 1; fromlen = sizeof(from); memset(&from, 0, sizeof(from)); if (getpeername(state->connection_in, (struct sockaddr *)&from, &fromlen) < 0) return 0; tolen = sizeof(to); memset(&to, 0, sizeof(to)); if (getpeername(state->connection_out, (struct sockaddr *)&to, &tolen) < 0) return 0; if (fromlen != tolen || memcmp(&from, &to, fromlen) != 0) return 0; if (from.ss_family != AF_INET && from.ss_family != AF_INET6) return 0; return 1; } void ssh_packet_get_bytes(struct ssh *ssh, u_int64_t *ibytes, u_int64_t *obytes) { if (ibytes) *ibytes = ssh->state->p_read.bytes; if (obytes) *obytes = ssh->state->p_send.bytes; } int ssh_packet_connection_af(struct ssh *ssh) { struct sockaddr_storage to; socklen_t tolen = sizeof(to); memset(&to, 0, sizeof(to)); if (getsockname(ssh->state->connection_out, (struct sockaddr *)&to, &tolen) < 0) return 0; #ifdef IPV4_IN_IPV6 if (to.ss_family == AF_INET6 && IN6_IS_ADDR_V4MAPPED(&((struct sockaddr_in6 *)&to)->sin6_addr)) return AF_INET; #endif return to.ss_family; } /* Sets the connection into non-blocking mode. */ void ssh_packet_set_nonblocking(struct ssh *ssh) { /* Set the socket into non-blocking mode. */ set_nonblock(ssh->state->connection_in); if (ssh->state->connection_out != ssh->state->connection_in) set_nonblock(ssh->state->connection_out); } /* Returns the socket used for reading. */ int ssh_packet_get_connection_in(struct ssh *ssh) { return ssh->state->connection_in; } /* Returns the descriptor used for writing. */ int ssh_packet_get_connection_out(struct ssh *ssh) { return ssh->state->connection_out; } /* * Returns the IP-address of the remote host as a string. The returned * string must not be freed. */ const char * ssh_remote_ipaddr(struct ssh *ssh) { const int sock = ssh->state->connection_in; /* Check whether we have cached the ipaddr. */ if (ssh->remote_ipaddr == NULL) { if (ssh_packet_connection_is_on_socket(ssh)) { ssh->remote_ipaddr = get_peer_ipaddr(sock); ssh->remote_port = get_sock_port(sock, 0); } else { ssh->remote_ipaddr = strdup("UNKNOWN"); ssh->remote_port = 0; } } return ssh->remote_ipaddr; } /* Returns the port number of the remote host. */ int ssh_remote_port(struct ssh *ssh) { (void)ssh_remote_ipaddr(ssh); /* Will lookup and cache. */ return ssh->remote_port; } /* Closes the connection and clears and frees internal data structures. */ void ssh_packet_close(struct ssh *ssh) { struct session_state *state = ssh->state; int r; u_int mode; if (!state->initialized) return; state->initialized = 0; if (state->connection_in == state->connection_out) { shutdown(state->connection_out, SHUT_RDWR); close(state->connection_out); } else { close(state->connection_in); close(state->connection_out); } sshbuf_free(state->input); sshbuf_free(state->output); sshbuf_free(state->outgoing_packet); sshbuf_free(state->incoming_packet); for (mode = 0; mode < MODE_MAX; mode++) kex_free_newkeys(state->newkeys[mode]); if (state->compression_buffer) { sshbuf_free(state->compression_buffer); if (state->compression_out_started) { z_streamp stream = &state->compression_out_stream; debug("compress outgoing: " "raw data %llu, compressed %llu, factor %.2f", (unsigned long long)stream->total_in, (unsigned long long)stream->total_out, stream->total_in == 0 ? 0.0 : (double) stream->total_out / stream->total_in); if (state->compression_out_failures == 0) deflateEnd(stream); } if (state->compression_in_started) { z_streamp stream = &state->compression_out_stream; debug("compress incoming: " "raw data %llu, compressed %llu, factor %.2f", (unsigned long long)stream->total_out, (unsigned long long)stream->total_in, stream->total_out == 0 ? 0.0 : (double) stream->total_in / stream->total_out); if (state->compression_in_failures == 0) inflateEnd(stream); } } if ((r = cipher_cleanup(&state->send_context)) != 0) error("%s: cipher_cleanup failed: %s", __func__, ssh_err(r)); if ((r = cipher_cleanup(&state->receive_context)) != 0) error("%s: cipher_cleanup failed: %s", __func__, ssh_err(r)); free(ssh->remote_ipaddr); ssh->remote_ipaddr = NULL; free(ssh->state); ssh->state = NULL; } /* Sets remote side protocol flags. */ void ssh_packet_set_protocol_flags(struct ssh *ssh, u_int protocol_flags) { ssh->state->remote_protocol_flags = protocol_flags; } /* Returns the remote protocol flags set earlier by the above function. */ u_int ssh_packet_get_protocol_flags(struct ssh *ssh) { return ssh->state->remote_protocol_flags; } /* * Starts packet compression from the next packet on in both directions. * Level is compression level 1 (fastest) - 9 (slow, best) as in gzip. */ static int ssh_packet_init_compression(struct ssh *ssh) { if (!ssh->state->compression_buffer && ((ssh->state->compression_buffer = sshbuf_new()) == NULL)) return SSH_ERR_ALLOC_FAIL; return 0; } static int start_compression_out(struct ssh *ssh, int level) { if (level < 1 || level > 9) return SSH_ERR_INVALID_ARGUMENT; debug("Enabling compression at level %d.", level); if (ssh->state->compression_out_started == 1) deflateEnd(&ssh->state->compression_out_stream); switch (deflateInit(&ssh->state->compression_out_stream, level)) { case Z_OK: ssh->state->compression_out_started = 1; break; case Z_MEM_ERROR: return SSH_ERR_ALLOC_FAIL; default: return SSH_ERR_INTERNAL_ERROR; } return 0; } static int start_compression_in(struct ssh *ssh) { if (ssh->state->compression_in_started == 1) inflateEnd(&ssh->state->compression_in_stream); switch (inflateInit(&ssh->state->compression_in_stream)) { case Z_OK: ssh->state->compression_in_started = 1; break; case Z_MEM_ERROR: return SSH_ERR_ALLOC_FAIL; default: return SSH_ERR_INTERNAL_ERROR; } return 0; } int ssh_packet_start_compression(struct ssh *ssh, int level) { int r; if (ssh->state->packet_compression && !compat20) return SSH_ERR_INTERNAL_ERROR; ssh->state->packet_compression = 1; if ((r = ssh_packet_init_compression(ssh)) != 0 || (r = start_compression_in(ssh)) != 0 || (r = start_compression_out(ssh, level)) != 0) return r; return 0; } /* XXX remove need for separate compression buffer */ static int compress_buffer(struct ssh *ssh, struct sshbuf *in, struct sshbuf *out) { u_char buf[4096]; int r, status; if (ssh->state->compression_out_started != 1) return SSH_ERR_INTERNAL_ERROR; /* This case is not handled below. */ if (sshbuf_len(in) == 0) return 0; /* Input is the contents of the input buffer. */ if ((ssh->state->compression_out_stream.next_in = sshbuf_mutable_ptr(in)) == NULL) return SSH_ERR_INTERNAL_ERROR; ssh->state->compression_out_stream.avail_in = sshbuf_len(in); /* Loop compressing until deflate() returns with avail_out != 0. */ do { /* Set up fixed-size output buffer. */ ssh->state->compression_out_stream.next_out = buf; ssh->state->compression_out_stream.avail_out = sizeof(buf); /* Compress as much data into the buffer as possible. */ status = deflate(&ssh->state->compression_out_stream, Z_PARTIAL_FLUSH); switch (status) { case Z_MEM_ERROR: return SSH_ERR_ALLOC_FAIL; case Z_OK: /* Append compressed data to output_buffer. */ if ((r = sshbuf_put(out, buf, sizeof(buf) - ssh->state->compression_out_stream.avail_out)) != 0) return r; break; case Z_STREAM_ERROR: default: ssh->state->compression_out_failures++; return SSH_ERR_INVALID_FORMAT; } } while (ssh->state->compression_out_stream.avail_out == 0); return 0; } static int uncompress_buffer(struct ssh *ssh, struct sshbuf *in, struct sshbuf *out) { u_char buf[4096]; int r, status; if (ssh->state->compression_in_started != 1) return SSH_ERR_INTERNAL_ERROR; if ((ssh->state->compression_in_stream.next_in = sshbuf_mutable_ptr(in)) == NULL) return SSH_ERR_INTERNAL_ERROR; ssh->state->compression_in_stream.avail_in = sshbuf_len(in); for (;;) { /* Set up fixed-size output buffer. */ ssh->state->compression_in_stream.next_out = buf; ssh->state->compression_in_stream.avail_out = sizeof(buf); status = inflate(&ssh->state->compression_in_stream, Z_PARTIAL_FLUSH); switch (status) { case Z_OK: if ((r = sshbuf_put(out, buf, sizeof(buf) - ssh->state->compression_in_stream.avail_out)) != 0) return r; break; case Z_BUF_ERROR: /* * Comments in zlib.h say that we should keep calling * inflate() until we get an error. This appears to * be the error that we get. */ return 0; case Z_DATA_ERROR: return SSH_ERR_INVALID_FORMAT; case Z_MEM_ERROR: return SSH_ERR_ALLOC_FAIL; case Z_STREAM_ERROR: default: ssh->state->compression_in_failures++; return SSH_ERR_INTERNAL_ERROR; } } /* NOTREACHED */ } /* Serialise compression state into a blob for privsep */ static int ssh_packet_get_compress_state(struct sshbuf *m, struct ssh *ssh) { struct session_state *state = ssh->state; struct sshbuf *b; int r; if ((b = sshbuf_new()) == NULL) return SSH_ERR_ALLOC_FAIL; if (state->compression_in_started) { if ((r = sshbuf_put_string(b, &state->compression_in_stream, sizeof(state->compression_in_stream))) != 0) goto out; } else if ((r = sshbuf_put_string(b, NULL, 0)) != 0) goto out; if (state->compression_out_started) { if ((r = sshbuf_put_string(b, &state->compression_out_stream, sizeof(state->compression_out_stream))) != 0) goto out; } else if ((r = sshbuf_put_string(b, NULL, 0)) != 0) goto out; r = sshbuf_put_stringb(m, b); out: sshbuf_free(b); return r; } /* Deserialise compression state from a blob for privsep */ static int ssh_packet_set_compress_state(struct ssh *ssh, struct sshbuf *m) { struct session_state *state = ssh->state; struct sshbuf *b = NULL; int r; const u_char *inblob, *outblob; size_t inl, outl; if ((r = sshbuf_froms(m, &b)) != 0) goto out; if ((r = sshbuf_get_string_direct(b, &inblob, &inl)) != 0 || (r = sshbuf_get_string_direct(b, &outblob, &outl)) != 0) goto out; if (inl == 0) state->compression_in_started = 0; else if (inl != sizeof(state->compression_in_stream)) { r = SSH_ERR_INTERNAL_ERROR; goto out; } else { state->compression_in_started = 1; memcpy(&state->compression_in_stream, inblob, inl); } if (outl == 0) state->compression_out_started = 0; else if (outl != sizeof(state->compression_out_stream)) { r = SSH_ERR_INTERNAL_ERROR; goto out; } else { state->compression_out_started = 1; memcpy(&state->compression_out_stream, outblob, outl); } r = 0; out: sshbuf_free(b); return r; } void ssh_packet_set_compress_hooks(struct ssh *ssh, void *ctx, void *(*allocfunc)(void *, u_int, u_int), void (*freefunc)(void *, void *)) { ssh->state->compression_out_stream.zalloc = (alloc_func)allocfunc; ssh->state->compression_out_stream.zfree = (free_func)freefunc; ssh->state->compression_out_stream.opaque = ctx; ssh->state->compression_in_stream.zalloc = (alloc_func)allocfunc; ssh->state->compression_in_stream.zfree = (free_func)freefunc; ssh->state->compression_in_stream.opaque = ctx; } /* * Causes any further packets to be encrypted using the given key. The same * key is used for both sending and reception. However, both directions are * encrypted independently of each other. */ void ssh_packet_set_encryption_key(struct ssh *ssh, const u_char *key, u_int keylen, int number) { #ifndef WITH_SSH1 fatal("no SSH protocol 1 support"); #else /* WITH_SSH1 */ struct session_state *state = ssh->state; const struct sshcipher *cipher = cipher_by_number(number); int r; const char *wmsg; if (cipher == NULL) fatal("%s: unknown cipher number %d", __func__, number); if (keylen < 20) fatal("%s: keylen too small: %d", __func__, keylen); if (keylen > SSH_SESSION_KEY_LENGTH) fatal("%s: keylen too big: %d", __func__, keylen); memcpy(state->ssh1_key, key, keylen); state->ssh1_keylen = keylen; if ((r = cipher_init(&state->send_context, cipher, key, keylen, NULL, 0, CIPHER_ENCRYPT)) != 0 || (r = cipher_init(&state->receive_context, cipher, key, keylen, NULL, 0, CIPHER_DECRYPT) != 0)) fatal("%s: cipher_init failed: %s", __func__, ssh_err(r)); if (!state->cipher_warning_done && ((wmsg = cipher_warning_message(&state->send_context)) != NULL || (wmsg = cipher_warning_message(&state->send_context)) != NULL)) { error("Warning: %s", wmsg); state->cipher_warning_done = 1; } #endif /* WITH_SSH1 */ } /* * Finalizes and sends the packet. If the encryption key has been set, * encrypts the packet before sending. */ int ssh_packet_send1(struct ssh *ssh) { struct session_state *state = ssh->state; u_char buf[8], *cp; int r, padding, len; u_int checksum; /* * If using packet compression, compress the payload of the outgoing * packet. */ if (state->packet_compression) { sshbuf_reset(state->compression_buffer); /* Skip padding. */ if ((r = sshbuf_consume(state->outgoing_packet, 8)) != 0) goto out; /* padding */ if ((r = sshbuf_put(state->compression_buffer, "\0\0\0\0\0\0\0\0", 8)) != 0) goto out; if ((r = compress_buffer(ssh, state->outgoing_packet, state->compression_buffer)) != 0) goto out; sshbuf_reset(state->outgoing_packet); if ((r = sshbuf_putb(state->outgoing_packet, state->compression_buffer)) != 0) goto out; } /* Compute packet length without padding (add checksum, remove padding). */ len = sshbuf_len(state->outgoing_packet) + 4 - 8; /* Insert padding. Initialized to zero in packet_start1() */ padding = 8 - len % 8; if (!state->send_context.plaintext) { cp = sshbuf_mutable_ptr(state->outgoing_packet); if (cp == NULL) { r = SSH_ERR_INTERNAL_ERROR; goto out; } arc4random_buf(cp + 8 - padding, padding); } if ((r = sshbuf_consume(state->outgoing_packet, 8 - padding)) != 0) goto out; /* Add check bytes. */ checksum = ssh_crc32(sshbuf_ptr(state->outgoing_packet), sshbuf_len(state->outgoing_packet)); POKE_U32(buf, checksum); if ((r = sshbuf_put(state->outgoing_packet, buf, 4)) != 0) goto out; #ifdef PACKET_DEBUG fprintf(stderr, "packet_send plain: "); sshbuf_dump(state->outgoing_packet, stderr); #endif /* Append to output. */ POKE_U32(buf, len); if ((r = sshbuf_put(state->output, buf, 4)) != 0) goto out; if ((r = sshbuf_reserve(state->output, sshbuf_len(state->outgoing_packet), &cp)) != 0) goto out; if ((r = cipher_crypt(&state->send_context, 0, cp, sshbuf_ptr(state->outgoing_packet), sshbuf_len(state->outgoing_packet), 0, 0)) != 0) goto out; #ifdef PACKET_DEBUG fprintf(stderr, "encrypted: "); sshbuf_dump(state->output, stderr); #endif state->p_send.packets++; state->p_send.bytes += len + sshbuf_len(state->outgoing_packet); sshbuf_reset(state->outgoing_packet); /* * Note that the packet is now only buffered in output. It won't be * actually sent until ssh_packet_write_wait or ssh_packet_write_poll * is called. */ r = 0; out: return r; } int ssh_set_newkeys(struct ssh *ssh, int mode) { struct session_state *state = ssh->state; struct sshenc *enc; struct sshmac *mac; struct sshcomp *comp; struct sshcipher_ctx *cc; u_int64_t *max_blocks; const char *wmsg; int r, crypt_type; debug2("set_newkeys: mode %d", mode); if (mode == MODE_OUT) { cc = &state->send_context; crypt_type = CIPHER_ENCRYPT; state->p_send.packets = state->p_send.blocks = 0; max_blocks = &state->max_blocks_out; } else { cc = &state->receive_context; crypt_type = CIPHER_DECRYPT; state->p_read.packets = state->p_read.blocks = 0; max_blocks = &state->max_blocks_in; } if (state->newkeys[mode] != NULL) { debug("set_newkeys: rekeying, input %llu bytes %llu blocks, " "output %llu bytes %llu blocks", (unsigned long long)state->p_read.bytes, (unsigned long long)state->p_read.blocks, (unsigned long long)state->p_send.bytes, (unsigned long long)state->p_send.blocks); if ((r = cipher_cleanup(cc)) != 0) return r; enc = &state->newkeys[mode]->enc; mac = &state->newkeys[mode]->mac; comp = &state->newkeys[mode]->comp; mac_clear(mac); explicit_bzero(enc->iv, enc->iv_len); explicit_bzero(enc->key, enc->key_len); explicit_bzero(mac->key, mac->key_len); free(enc->name); free(enc->iv); free(enc->key); free(mac->name); free(mac->key); free(comp->name); free(state->newkeys[mode]); } /* move newkeys from kex to state */ if ((state->newkeys[mode] = ssh->kex->newkeys[mode]) == NULL) return SSH_ERR_INTERNAL_ERROR; ssh->kex->newkeys[mode] = NULL; enc = &state->newkeys[mode]->enc; mac = &state->newkeys[mode]->mac; comp = &state->newkeys[mode]->comp; if (cipher_authlen(enc->cipher) == 0) { if ((r = mac_init(mac)) != 0) return r; } mac->enabled = 1; DBG(debug("cipher_init_context: %d", mode)); if ((r = cipher_init(cc, enc->cipher, enc->key, enc->key_len, enc->iv, enc->iv_len, crypt_type)) != 0) return r; if (!state->cipher_warning_done && (wmsg = cipher_warning_message(cc)) != NULL) { error("Warning: %s", wmsg); state->cipher_warning_done = 1; } /* Deleting the keys does not gain extra security */ /* explicit_bzero(enc->iv, enc->block_size); explicit_bzero(enc->key, enc->key_len); explicit_bzero(mac->key, mac->key_len); */ if ((comp->type == COMP_ZLIB || (comp->type == COMP_DELAYED && state->after_authentication)) && comp->enabled == 0) { if ((r = ssh_packet_init_compression(ssh)) < 0) return r; if (mode == MODE_OUT) { if ((r = start_compression_out(ssh, 6)) != 0) return r; } else { if ((r = start_compression_in(ssh)) != 0) return r; } comp->enabled = 1; } /* * The 2^(blocksize*2) limit is too expensive for 3DES, * blowfish, etc, so enforce a 1GB limit for small blocksizes. */ if (enc->block_size >= 16) *max_blocks = (u_int64_t)1 << (enc->block_size*2); else *max_blocks = ((u_int64_t)1 << 30) / enc->block_size; if (state->rekey_limit) *max_blocks = MIN(*max_blocks, state->rekey_limit / enc->block_size); debug("rekey after %llu blocks", (unsigned long long)*max_blocks); return 0; } #define MAX_PACKETS (1U<<31) static int ssh_packet_need_rekeying(struct ssh *ssh, u_int outbound_packet_len) { struct session_state *state = ssh->state; u_int32_t out_blocks; /* XXX client can't cope with rekeying pre-auth */ if (!state->after_authentication) return 0; /* Haven't keyed yet or KEX in progress. */ if (ssh->kex == NULL || ssh_packet_is_rekeying(ssh)) return 0; /* Peer can't rekey */ if (ssh->compat & SSH_BUG_NOREKEY) return 0; /* * Permit one packet in or out per rekey - this allows us to * make progress when rekey limits are very small. */ if (state->p_send.packets == 0 && state->p_read.packets == 0) return 0; /* Time-based rekeying */ if (state->rekey_interval != 0 && state->rekey_time + state->rekey_interval <= monotime()) return 1; /* Always rekey when MAX_PACKETS sent in either direction */ if (state->p_send.packets > MAX_PACKETS || state->p_read.packets > MAX_PACKETS) return 1; /* Rekey after (cipher-specific) maxiumum blocks */ out_blocks = roundup(outbound_packet_len, state->newkeys[MODE_OUT]->enc.block_size); return (state->max_blocks_out && (state->p_send.blocks + out_blocks > state->max_blocks_out)) || (state->max_blocks_in && (state->p_read.blocks > state->max_blocks_in)); } /* * Delayed compression for SSH2 is enabled after authentication: * This happens on the server side after a SSH2_MSG_USERAUTH_SUCCESS is sent, * and on the client side after a SSH2_MSG_USERAUTH_SUCCESS is received. */ static int ssh_packet_enable_delayed_compress(struct ssh *ssh) { struct session_state *state = ssh->state; struct sshcomp *comp = NULL; int r, mode; /* * Remember that we are past the authentication step, so rekeying * with COMP_DELAYED will turn on compression immediately. */ state->after_authentication = 1; for (mode = 0; mode < MODE_MAX; mode++) { /* protocol error: USERAUTH_SUCCESS received before NEWKEYS */ if (state->newkeys[mode] == NULL) continue; comp = &state->newkeys[mode]->comp; if (comp && !comp->enabled && comp->type == COMP_DELAYED) { if ((r = ssh_packet_init_compression(ssh)) != 0) return r; if (mode == MODE_OUT) { if ((r = start_compression_out(ssh, 6)) != 0) return r; } else { if ((r = start_compression_in(ssh)) != 0) return r; } comp->enabled = 1; } } return 0; } /* Used to mute debug logging for noisy packet types */ static int ssh_packet_log_type(u_char type) { switch (type) { case SSH2_MSG_CHANNEL_DATA: case SSH2_MSG_CHANNEL_EXTENDED_DATA: case SSH2_MSG_CHANNEL_WINDOW_ADJUST: return 0; default: return 1; } } /* * Finalize packet in SSH2 format (compress, mac, encrypt, enqueue) */ int ssh_packet_send2_wrapped(struct ssh *ssh) { struct session_state *state = ssh->state; u_char type, *cp, macbuf[SSH_DIGEST_MAX_LENGTH]; u_char padlen, pad = 0; u_int authlen = 0, aadlen = 0; u_int len; struct sshenc *enc = NULL; struct sshmac *mac = NULL; struct sshcomp *comp = NULL; int r, block_size; if (state->newkeys[MODE_OUT] != NULL) { enc = &state->newkeys[MODE_OUT]->enc; mac = &state->newkeys[MODE_OUT]->mac; comp = &state->newkeys[MODE_OUT]->comp; /* disable mac for authenticated encryption */ if ((authlen = cipher_authlen(enc->cipher)) != 0) mac = NULL; } block_size = enc ? enc->block_size : 8; aadlen = (mac && mac->enabled && mac->etm) || authlen ? 4 : 0; type = (sshbuf_ptr(state->outgoing_packet))[5]; if (ssh_packet_log_type(type)) debug3("send packet: type %u", type); #ifdef PACKET_DEBUG fprintf(stderr, "plain: "); sshbuf_dump(state->outgoing_packet, stderr); #endif if (comp && comp->enabled) { len = sshbuf_len(state->outgoing_packet); /* skip header, compress only payload */ if ((r = sshbuf_consume(state->outgoing_packet, 5)) != 0) goto out; sshbuf_reset(state->compression_buffer); if ((r = compress_buffer(ssh, state->outgoing_packet, state->compression_buffer)) != 0) goto out; sshbuf_reset(state->outgoing_packet); if ((r = sshbuf_put(state->outgoing_packet, "\0\0\0\0\0", 5)) != 0 || (r = sshbuf_putb(state->outgoing_packet, state->compression_buffer)) != 0) goto out; DBG(debug("compression: raw %d compressed %zd", len, sshbuf_len(state->outgoing_packet))); } /* sizeof (packet_len + pad_len + payload) */ len = sshbuf_len(state->outgoing_packet); /* * calc size of padding, alloc space, get random data, * minimum padding is 4 bytes */ len -= aadlen; /* packet length is not encrypted for EtM modes */ padlen = block_size - (len % block_size); if (padlen < 4) padlen += block_size; if (state->extra_pad) { /* will wrap if extra_pad+padlen > 255 */ state->extra_pad = roundup(state->extra_pad, block_size); pad = state->extra_pad - ((len + padlen) % state->extra_pad); DBG(debug3("%s: adding %d (len %d padlen %d extra_pad %d)", __func__, pad, len, padlen, state->extra_pad)); padlen += pad; state->extra_pad = 0; } if ((r = sshbuf_reserve(state->outgoing_packet, padlen, &cp)) != 0) goto out; if (enc && !state->send_context.plaintext) { /* random padding */ arc4random_buf(cp, padlen); } else { /* clear padding */ explicit_bzero(cp, padlen); } /* sizeof (packet_len + pad_len + payload + padding) */ len = sshbuf_len(state->outgoing_packet); cp = sshbuf_mutable_ptr(state->outgoing_packet); if (cp == NULL) { r = SSH_ERR_INTERNAL_ERROR; goto out; } /* packet_length includes payload, padding and padding length field */ POKE_U32(cp, len - 4); cp[4] = padlen; DBG(debug("send: len %d (includes padlen %d, aadlen %d)", len, padlen, aadlen)); /* compute MAC over seqnr and packet(length fields, payload, padding) */ if (mac && mac->enabled && !mac->etm) { if ((r = mac_compute(mac, state->p_send.seqnr, sshbuf_ptr(state->outgoing_packet), len, macbuf, sizeof(macbuf))) != 0) goto out; DBG(debug("done calc MAC out #%d", state->p_send.seqnr)); } /* encrypt packet and append to output buffer. */ if ((r = sshbuf_reserve(state->output, sshbuf_len(state->outgoing_packet) + authlen, &cp)) != 0) goto out; if ((r = cipher_crypt(&state->send_context, state->p_send.seqnr, cp, sshbuf_ptr(state->outgoing_packet), len - aadlen, aadlen, authlen)) != 0) goto out; /* append unencrypted MAC */ if (mac && mac->enabled) { if (mac->etm) { /* EtM: compute mac over aadlen + cipher text */ if ((r = mac_compute(mac, state->p_send.seqnr, cp, len, macbuf, sizeof(macbuf))) != 0) goto out; DBG(debug("done calc MAC(EtM) out #%d", state->p_send.seqnr)); } if ((r = sshbuf_put(state->output, macbuf, mac->mac_len)) != 0) goto out; } #ifdef PACKET_DEBUG fprintf(stderr, "encrypted: "); sshbuf_dump(state->output, stderr); #endif /* increment sequence number for outgoing packets */ if (++state->p_send.seqnr == 0) logit("outgoing seqnr wraps around"); if (++state->p_send.packets == 0) if (!(ssh->compat & SSH_BUG_NOREKEY)) return SSH_ERR_NEED_REKEY; state->p_send.blocks += len / block_size; state->p_send.bytes += len; sshbuf_reset(state->outgoing_packet); if (type == SSH2_MSG_NEWKEYS) r = ssh_set_newkeys(ssh, MODE_OUT); else if (type == SSH2_MSG_USERAUTH_SUCCESS && state->server_side) r = ssh_packet_enable_delayed_compress(ssh); else r = 0; out: return r; } /* returns non-zero if the specified packet type is usec by KEX */ static int ssh_packet_type_is_kex(u_char type) { return type >= SSH2_MSG_TRANSPORT_MIN && type <= SSH2_MSG_TRANSPORT_MAX && type != SSH2_MSG_SERVICE_REQUEST && type != SSH2_MSG_SERVICE_ACCEPT && type != SSH2_MSG_EXT_INFO; } int ssh_packet_send2(struct ssh *ssh) { struct session_state *state = ssh->state; struct packet *p; u_char type; int r, need_rekey; if (sshbuf_len(state->outgoing_packet) < 6) return SSH_ERR_INTERNAL_ERROR; type = sshbuf_ptr(state->outgoing_packet)[5]; need_rekey = !ssh_packet_type_is_kex(type) && ssh_packet_need_rekeying(ssh, sshbuf_len(state->outgoing_packet)); /* * During rekeying we can only send key exchange messages. * Queue everything else. */ if ((need_rekey || state->rekeying) && !ssh_packet_type_is_kex(type)) { if (need_rekey) debug3("%s: rekex triggered", __func__); debug("enqueue packet: %u", type); p = calloc(1, sizeof(*p)); if (p == NULL) return SSH_ERR_ALLOC_FAIL; p->type = type; p->payload = state->outgoing_packet; TAILQ_INSERT_TAIL(&state->outgoing, p, next); state->outgoing_packet = sshbuf_new(); if (state->outgoing_packet == NULL) return SSH_ERR_ALLOC_FAIL; if (need_rekey) { /* * This packet triggered a rekey, so send the * KEXINIT now. * NB. reenters this function via kex_start_rekex(). */ return kex_start_rekex(ssh); } return 0; } /* rekeying starts with sending KEXINIT */ if (type == SSH2_MSG_KEXINIT) state->rekeying = 1; if ((r = ssh_packet_send2_wrapped(ssh)) != 0) return r; /* after a NEWKEYS message we can send the complete queue */ if (type == SSH2_MSG_NEWKEYS) { state->rekeying = 0; state->rekey_time = monotime(); while ((p = TAILQ_FIRST(&state->outgoing))) { type = p->type; /* * If this packet triggers a rekex, then skip the * remaining packets in the queue for now. * NB. re-enters this function via kex_start_rekex. */ if (ssh_packet_need_rekeying(ssh, sshbuf_len(p->payload))) { debug3("%s: queued packet triggered rekex", __func__); return kex_start_rekex(ssh); } debug("dequeue packet: %u", type); sshbuf_free(state->outgoing_packet); state->outgoing_packet = p->payload; TAILQ_REMOVE(&state->outgoing, p, next); memset(p, 0, sizeof(*p)); free(p); if ((r = ssh_packet_send2_wrapped(ssh)) != 0) return r; } } return 0; } /* * Waits until a packet has been received, and returns its type. Note that * no other data is processed until this returns, so this function should not * be used during the interactive session. */ int ssh_packet_read_seqnr(struct ssh *ssh, u_char *typep, u_int32_t *seqnr_p) { struct session_state *state = ssh->state; int len, r, ms_remain; fd_set *setp; char buf[8192]; struct timeval timeout, start, *timeoutp = NULL; DBG(debug("packet_read()")); setp = calloc(howmany(state->connection_in + 1, NFDBITS), sizeof(fd_mask)); if (setp == NULL) return SSH_ERR_ALLOC_FAIL; /* * Since we are blocking, ensure that all written packets have * been sent. */ if ((r = ssh_packet_write_wait(ssh)) != 0) goto out; /* Stay in the loop until we have received a complete packet. */ for (;;) { /* Try to read a packet from the buffer. */ r = ssh_packet_read_poll_seqnr(ssh, typep, seqnr_p); if (r != 0) break; if (!compat20 && ( *typep == SSH_SMSG_SUCCESS || *typep == SSH_SMSG_FAILURE || *typep == SSH_CMSG_EOF || *typep == SSH_CMSG_EXIT_CONFIRMATION)) if ((r = sshpkt_get_end(ssh)) != 0) break; /* If we got a packet, return it. */ if (*typep != SSH_MSG_NONE) break; /* * Otherwise, wait for some data to arrive, add it to the * buffer, and try again. */ memset(setp, 0, howmany(state->connection_in + 1, NFDBITS) * sizeof(fd_mask)); FD_SET(state->connection_in, setp); if (state->packet_timeout_ms > 0) { ms_remain = state->packet_timeout_ms; timeoutp = &timeout; } /* Wait for some data to arrive. */ for (;;) { if (state->packet_timeout_ms != -1) { ms_to_timeval(&timeout, ms_remain); gettimeofday(&start, NULL); } if ((r = select(state->connection_in + 1, setp, NULL, NULL, timeoutp)) >= 0) break; if (errno != EAGAIN && errno != EINTR && errno != EWOULDBLOCK) break; if (state->packet_timeout_ms == -1) continue; ms_subtract_diff(&start, &ms_remain); if (ms_remain <= 0) { r = 0; break; } } if (r == 0) return SSH_ERR_CONN_TIMEOUT; /* Read data from the socket. */ len = read(state->connection_in, buf, sizeof(buf)); if (len == 0) { r = SSH_ERR_CONN_CLOSED; goto out; } if (len < 0) { r = SSH_ERR_SYSTEM_ERROR; goto out; } /* Append it to the buffer. */ if ((r = ssh_packet_process_incoming(ssh, buf, len)) != 0) goto out; } out: free(setp); return r; } int ssh_packet_read(struct ssh *ssh) { u_char type; int r; if ((r = ssh_packet_read_seqnr(ssh, &type, NULL)) != 0) fatal("%s: %s", __func__, ssh_err(r)); return type; } /* * Waits until a packet has been received, verifies that its type matches * that given, and gives a fatal error and exits if there is a mismatch. */ int ssh_packet_read_expect(struct ssh *ssh, u_int expected_type) { int r; u_char type; if ((r = ssh_packet_read_seqnr(ssh, &type, NULL)) != 0) return r; if (type != expected_type) { if ((r = sshpkt_disconnect(ssh, "Protocol error: expected packet type %d, got %d", expected_type, type)) != 0) return r; return SSH_ERR_PROTOCOL_ERROR; } return 0; } /* Checks if a full packet is available in the data received so far via * packet_process_incoming. If so, reads the packet; otherwise returns * SSH_MSG_NONE. This does not wait for data from the connection. * * SSH_MSG_DISCONNECT is handled specially here. Also, * SSH_MSG_IGNORE messages are skipped by this function and are never returned * to higher levels. */ int ssh_packet_read_poll1(struct ssh *ssh, u_char *typep) { struct session_state *state = ssh->state; u_int len, padded_len; const char *emsg; const u_char *cp; u_char *p; u_int checksum, stored_checksum; int r; *typep = SSH_MSG_NONE; /* Check if input size is less than minimum packet size. */ if (sshbuf_len(state->input) < 4 + 8) return 0; /* Get length of incoming packet. */ len = PEEK_U32(sshbuf_ptr(state->input)); if (len < 1 + 2 + 2 || len > 256 * 1024) { if ((r = sshpkt_disconnect(ssh, "Bad packet length %u", len)) != 0) return r; return SSH_ERR_CONN_CORRUPT; } padded_len = (len + 8) & ~7; /* Check if the packet has been entirely received. */ if (sshbuf_len(state->input) < 4 + padded_len) return 0; /* The entire packet is in buffer. */ /* Consume packet length. */ if ((r = sshbuf_consume(state->input, 4)) != 0) goto out; /* * Cryptographic attack detector for ssh * (C)1998 CORE-SDI, Buenos Aires Argentina * Ariel Futoransky(futo@core-sdi.com) */ if (!state->receive_context.plaintext) { emsg = NULL; switch (detect_attack(&state->deattack, sshbuf_ptr(state->input), padded_len)) { case DEATTACK_OK: break; case DEATTACK_DETECTED: emsg = "crc32 compensation attack detected"; break; case DEATTACK_DOS_DETECTED: emsg = "deattack denial of service detected"; break; default: emsg = "deattack error"; break; } if (emsg != NULL) { error("%s", emsg); if ((r = sshpkt_disconnect(ssh, "%s", emsg)) != 0 || (r = ssh_packet_write_wait(ssh)) != 0) return r; return SSH_ERR_CONN_CORRUPT; } } /* Decrypt data to incoming_packet. */ sshbuf_reset(state->incoming_packet); if ((r = sshbuf_reserve(state->incoming_packet, padded_len, &p)) != 0) goto out; if ((r = cipher_crypt(&state->receive_context, 0, p, sshbuf_ptr(state->input), padded_len, 0, 0)) != 0) goto out; if ((r = sshbuf_consume(state->input, padded_len)) != 0) goto out; #ifdef PACKET_DEBUG fprintf(stderr, "read_poll plain: "); sshbuf_dump(state->incoming_packet, stderr); #endif /* Compute packet checksum. */ checksum = ssh_crc32(sshbuf_ptr(state->incoming_packet), sshbuf_len(state->incoming_packet) - 4); /* Skip padding. */ if ((r = sshbuf_consume(state->incoming_packet, 8 - len % 8)) != 0) goto out; /* Test check bytes. */ if (len != sshbuf_len(state->incoming_packet)) { error("%s: len %d != sshbuf_len %zd", __func__, len, sshbuf_len(state->incoming_packet)); if ((r = sshpkt_disconnect(ssh, "invalid packet length")) != 0 || (r = ssh_packet_write_wait(ssh)) != 0) return r; return SSH_ERR_CONN_CORRUPT; } cp = sshbuf_ptr(state->incoming_packet) + len - 4; stored_checksum = PEEK_U32(cp); if (checksum != stored_checksum) { error("Corrupted check bytes on input"); if ((r = sshpkt_disconnect(ssh, "connection corrupted")) != 0 || (r = ssh_packet_write_wait(ssh)) != 0) return r; return SSH_ERR_CONN_CORRUPT; } if ((r = sshbuf_consume_end(state->incoming_packet, 4)) < 0) goto out; if (state->packet_compression) { sshbuf_reset(state->compression_buffer); if ((r = uncompress_buffer(ssh, state->incoming_packet, state->compression_buffer)) != 0) goto out; sshbuf_reset(state->incoming_packet); if ((r = sshbuf_putb(state->incoming_packet, state->compression_buffer)) != 0) goto out; } state->p_read.packets++; state->p_read.bytes += padded_len + 4; if ((r = sshbuf_get_u8(state->incoming_packet, typep)) != 0) goto out; if (*typep < SSH_MSG_MIN || *typep > SSH_MSG_MAX) { error("Invalid ssh1 packet type: %d", *typep); if ((r = sshpkt_disconnect(ssh, "invalid packet type")) != 0 || (r = ssh_packet_write_wait(ssh)) != 0) return r; return SSH_ERR_PROTOCOL_ERROR; } r = 0; out: return r; } int ssh_packet_read_poll2(struct ssh *ssh, u_char *typep, u_int32_t *seqnr_p) { struct session_state *state = ssh->state; u_int padlen, need; u_char *cp, macbuf[SSH_DIGEST_MAX_LENGTH]; u_int maclen, aadlen = 0, authlen = 0, block_size; struct sshenc *enc = NULL; struct sshmac *mac = NULL; struct sshcomp *comp = NULL; int r; *typep = SSH_MSG_NONE; if (state->packet_discard) return 0; if (state->newkeys[MODE_IN] != NULL) { enc = &state->newkeys[MODE_IN]->enc; mac = &state->newkeys[MODE_IN]->mac; comp = &state->newkeys[MODE_IN]->comp; /* disable mac for authenticated encryption */ if ((authlen = cipher_authlen(enc->cipher)) != 0) mac = NULL; } maclen = mac && mac->enabled ? mac->mac_len : 0; block_size = enc ? enc->block_size : 8; aadlen = (mac && mac->enabled && mac->etm) || authlen ? 4 : 0; if (aadlen && state->packlen == 0) { if (cipher_get_length(&state->receive_context, &state->packlen, state->p_read.seqnr, sshbuf_ptr(state->input), sshbuf_len(state->input)) != 0) return 0; if (state->packlen < 1 + 4 || state->packlen > PACKET_MAX_SIZE) { #ifdef PACKET_DEBUG sshbuf_dump(state->input, stderr); #endif logit("Bad packet length %u.", state->packlen); if ((r = sshpkt_disconnect(ssh, "Packet corrupt")) != 0) return r; return SSH_ERR_CONN_CORRUPT; } sshbuf_reset(state->incoming_packet); } else if (state->packlen == 0) { /* * check if input size is less than the cipher block size, * decrypt first block and extract length of incoming packet */ if (sshbuf_len(state->input) < block_size) return 0; sshbuf_reset(state->incoming_packet); if ((r = sshbuf_reserve(state->incoming_packet, block_size, &cp)) != 0) goto out; if ((r = cipher_crypt(&state->receive_context, state->p_send.seqnr, cp, sshbuf_ptr(state->input), block_size, 0, 0)) != 0) goto out; state->packlen = PEEK_U32(sshbuf_ptr(state->incoming_packet)); if (state->packlen < 1 + 4 || state->packlen > PACKET_MAX_SIZE) { #ifdef PACKET_DEBUG fprintf(stderr, "input: \n"); sshbuf_dump(state->input, stderr); fprintf(stderr, "incoming_packet: \n"); sshbuf_dump(state->incoming_packet, stderr); #endif logit("Bad packet length %u.", state->packlen); return ssh_packet_start_discard(ssh, enc, mac, state->packlen, PACKET_MAX_SIZE); } if ((r = sshbuf_consume(state->input, block_size)) != 0) goto out; } DBG(debug("input: packet len %u", state->packlen+4)); if (aadlen) { /* only the payload is encrypted */ need = state->packlen; } else { /* * the payload size and the payload are encrypted, but we * have a partial packet of block_size bytes */ need = 4 + state->packlen - block_size; } DBG(debug("partial packet: block %d, need %d, maclen %d, authlen %d," " aadlen %d", block_size, need, maclen, authlen, aadlen)); if (need % block_size != 0) { logit("padding error: need %d block %d mod %d", need, block_size, need % block_size); return ssh_packet_start_discard(ssh, enc, mac, state->packlen, PACKET_MAX_SIZE - block_size); } /* * check if the entire packet has been received and * decrypt into incoming_packet: * 'aadlen' bytes are unencrypted, but authenticated. * 'need' bytes are encrypted, followed by either * 'authlen' bytes of authentication tag or * 'maclen' bytes of message authentication code. */ if (sshbuf_len(state->input) < aadlen + need + authlen + maclen) return 0; #ifdef PACKET_DEBUG fprintf(stderr, "read_poll enc/full: "); sshbuf_dump(state->input, stderr); #endif /* EtM: compute mac over encrypted input */ if (mac && mac->enabled && mac->etm) { if ((r = mac_compute(mac, state->p_read.seqnr, sshbuf_ptr(state->input), aadlen + need, macbuf, sizeof(macbuf))) != 0) goto out; } if ((r = sshbuf_reserve(state->incoming_packet, aadlen + need, &cp)) != 0) goto out; if ((r = cipher_crypt(&state->receive_context, state->p_read.seqnr, cp, sshbuf_ptr(state->input), need, aadlen, authlen)) != 0) goto out; if ((r = sshbuf_consume(state->input, aadlen + need + authlen)) != 0) goto out; /* * compute MAC over seqnr and packet, * increment sequence number for incoming packet */ if (mac && mac->enabled) { if (!mac->etm) if ((r = mac_compute(mac, state->p_read.seqnr, sshbuf_ptr(state->incoming_packet), sshbuf_len(state->incoming_packet), macbuf, sizeof(macbuf))) != 0) goto out; if (timingsafe_bcmp(macbuf, sshbuf_ptr(state->input), mac->mac_len) != 0) { logit("Corrupted MAC on input."); if (need > PACKET_MAX_SIZE) return SSH_ERR_INTERNAL_ERROR; return ssh_packet_start_discard(ssh, enc, mac, state->packlen, PACKET_MAX_SIZE - need); } DBG(debug("MAC #%d ok", state->p_read.seqnr)); if ((r = sshbuf_consume(state->input, mac->mac_len)) != 0) goto out; } if (seqnr_p != NULL) *seqnr_p = state->p_read.seqnr; if (++state->p_read.seqnr == 0) logit("incoming seqnr wraps around"); if (++state->p_read.packets == 0) if (!(ssh->compat & SSH_BUG_NOREKEY)) return SSH_ERR_NEED_REKEY; state->p_read.blocks += (state->packlen + 4) / block_size; state->p_read.bytes += state->packlen + 4; /* get padlen */ padlen = sshbuf_ptr(state->incoming_packet)[4]; DBG(debug("input: padlen %d", padlen)); if (padlen < 4) { if ((r = sshpkt_disconnect(ssh, "Corrupted padlen %d on input.", padlen)) != 0 || (r = ssh_packet_write_wait(ssh)) != 0) return r; return SSH_ERR_CONN_CORRUPT; } /* skip packet size + padlen, discard padding */ if ((r = sshbuf_consume(state->incoming_packet, 4 + 1)) != 0 || ((r = sshbuf_consume_end(state->incoming_packet, padlen)) != 0)) goto out; DBG(debug("input: len before de-compress %zd", sshbuf_len(state->incoming_packet))); if (comp && comp->enabled) { sshbuf_reset(state->compression_buffer); if ((r = uncompress_buffer(ssh, state->incoming_packet, state->compression_buffer)) != 0) goto out; sshbuf_reset(state->incoming_packet); if ((r = sshbuf_putb(state->incoming_packet, state->compression_buffer)) != 0) goto out; DBG(debug("input: len after de-compress %zd", sshbuf_len(state->incoming_packet))); } /* * get packet type, implies consume. * return length of payload (without type field) */ if ((r = sshbuf_get_u8(state->incoming_packet, typep)) != 0) goto out; if (ssh_packet_log_type(*typep)) debug3("receive packet: type %u", *typep); if (*typep < SSH2_MSG_MIN || *typep >= SSH2_MSG_LOCAL_MIN) { if ((r = sshpkt_disconnect(ssh, "Invalid ssh2 packet type: %d", *typep)) != 0 || (r = ssh_packet_write_wait(ssh)) != 0) return r; return SSH_ERR_PROTOCOL_ERROR; } if (*typep == SSH2_MSG_NEWKEYS) r = ssh_set_newkeys(ssh, MODE_IN); else if (*typep == SSH2_MSG_USERAUTH_SUCCESS && !state->server_side) r = ssh_packet_enable_delayed_compress(ssh); else r = 0; #ifdef PACKET_DEBUG fprintf(stderr, "read/plain[%d]:\r\n", *typep); sshbuf_dump(state->incoming_packet, stderr); #endif /* reset for next packet */ state->packlen = 0; /* do we need to rekey? */ if (ssh_packet_need_rekeying(ssh, 0)) { debug3("%s: rekex triggered", __func__); if ((r = kex_start_rekex(ssh)) != 0) return r; } out: return r; } int ssh_packet_read_poll_seqnr(struct ssh *ssh, u_char *typep, u_int32_t *seqnr_p) { struct session_state *state = ssh->state; u_int reason, seqnr; int r; u_char *msg; for (;;) { msg = NULL; if (compat20) { r = ssh_packet_read_poll2(ssh, typep, seqnr_p); if (r != 0) return r; if (*typep) { state->keep_alive_timeouts = 0; DBG(debug("received packet type %d", *typep)); } switch (*typep) { case SSH2_MSG_IGNORE: debug3("Received SSH2_MSG_IGNORE"); break; case SSH2_MSG_DEBUG: if ((r = sshpkt_get_u8(ssh, NULL)) != 0 || (r = sshpkt_get_string(ssh, &msg, NULL)) != 0 || (r = sshpkt_get_string(ssh, NULL, NULL)) != 0) { free(msg); return r; } debug("Remote: %.900s", msg); free(msg); break; case SSH2_MSG_DISCONNECT: if ((r = sshpkt_get_u32(ssh, &reason)) != 0 || (r = sshpkt_get_string(ssh, &msg, NULL)) != 0) return r; /* Ignore normal client exit notifications */ do_log2(ssh->state->server_side && reason == SSH2_DISCONNECT_BY_APPLICATION ? SYSLOG_LEVEL_INFO : SYSLOG_LEVEL_ERROR, "Received disconnect from %s port %d:" "%u: %.400s", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh), reason, msg); free(msg); return SSH_ERR_DISCONNECTED; case SSH2_MSG_UNIMPLEMENTED: if ((r = sshpkt_get_u32(ssh, &seqnr)) != 0) return r; debug("Received SSH2_MSG_UNIMPLEMENTED for %u", seqnr); break; default: return 0; } } else { r = ssh_packet_read_poll1(ssh, typep); switch (*typep) { case SSH_MSG_NONE: return SSH_MSG_NONE; case SSH_MSG_IGNORE: break; case SSH_MSG_DEBUG: if ((r = sshpkt_get_string(ssh, &msg, NULL)) != 0) return r; debug("Remote: %.900s", msg); free(msg); break; case SSH_MSG_DISCONNECT: if ((r = sshpkt_get_string(ssh, &msg, NULL)) != 0) return r; logit("Received disconnect from %s port %d: " "%.400s", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh), msg); free(msg); return SSH_ERR_DISCONNECTED; default: DBG(debug("received packet type %d", *typep)); return 0; } } } } /* * Buffers the given amount of input characters. This is intended to be used * together with packet_read_poll. */ int ssh_packet_process_incoming(struct ssh *ssh, const char *buf, u_int len) { struct session_state *state = ssh->state; int r; if (state->packet_discard) { state->keep_alive_timeouts = 0; /* ?? */ if (len >= state->packet_discard) { if ((r = ssh_packet_stop_discard(ssh)) != 0) return r; } state->packet_discard -= len; return 0; } if ((r = sshbuf_put(ssh->state->input, buf, len)) != 0) return r; return 0; } int ssh_packet_remaining(struct ssh *ssh) { return sshbuf_len(ssh->state->incoming_packet); } /* * Sends a diagnostic message from the server to the client. This message * can be sent at any time (but not while constructing another message). The * message is printed immediately, but only if the client is being executed * in verbose mode. These messages are primarily intended to ease debugging * authentication problems. The length of the formatted message must not * exceed 1024 bytes. This will automatically call ssh_packet_write_wait. */ void ssh_packet_send_debug(struct ssh *ssh, const char *fmt,...) { char buf[1024]; va_list args; int r; if (compat20 && (ssh->compat & SSH_BUG_DEBUG)) return; va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); if (compat20) { if ((r = sshpkt_start(ssh, SSH2_MSG_DEBUG)) != 0 || (r = sshpkt_put_u8(ssh, 0)) != 0 || /* always display */ (r = sshpkt_put_cstring(ssh, buf)) != 0 || (r = sshpkt_put_cstring(ssh, "")) != 0 || (r = sshpkt_send(ssh)) != 0) fatal("%s: %s", __func__, ssh_err(r)); } else { if ((r = sshpkt_start(ssh, SSH_MSG_DEBUG)) != 0 || (r = sshpkt_put_cstring(ssh, buf)) != 0 || (r = sshpkt_send(ssh)) != 0) fatal("%s: %s", __func__, ssh_err(r)); } if ((r = ssh_packet_write_wait(ssh)) != 0) fatal("%s: %s", __func__, ssh_err(r)); } /* * Pretty-print connection-terminating errors and exit. */ void sshpkt_fatal(struct ssh *ssh, const char *tag, int r) { switch (r) { case SSH_ERR_CONN_CLOSED: logit("Connection closed by %.200s port %d", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh)); cleanup_exit(255); case SSH_ERR_CONN_TIMEOUT: logit("Connection %s %.200s port %d timed out", ssh->state->server_side ? "from" : "to", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh)); cleanup_exit(255); case SSH_ERR_DISCONNECTED: logit("Disconnected from %.200s port %d", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh)); cleanup_exit(255); case SSH_ERR_SYSTEM_ERROR: if (errno == ECONNRESET) { logit("Connection reset by %.200s port %d", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh)); cleanup_exit(255); } /* FALLTHROUGH */ case SSH_ERR_NO_CIPHER_ALG_MATCH: case SSH_ERR_NO_MAC_ALG_MATCH: case SSH_ERR_NO_COMPRESS_ALG_MATCH: case SSH_ERR_NO_KEX_ALG_MATCH: case SSH_ERR_NO_HOSTKEY_ALG_MATCH: if (ssh && ssh->kex && ssh->kex->failed_choice) { + BLACKLIST_NOTIFY(BLACKLIST_AUTH_FAIL); fatal("Unable to negotiate with %.200s port %d: %s. " "Their offer: %s", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh), ssh_err(r), ssh->kex->failed_choice); } /* FALLTHROUGH */ default: fatal("%s%sConnection %s %.200s port %d: %s", tag != NULL ? tag : "", tag != NULL ? ": " : "", ssh->state->server_side ? "from" : "to", ssh_remote_ipaddr(ssh), ssh_remote_port(ssh), ssh_err(r)); } } /* * Logs the error plus constructs and sends a disconnect packet, closes the * connection, and exits. This function never returns. The error message * should not contain a newline. The length of the formatted message must * not exceed 1024 bytes. */ void ssh_packet_disconnect(struct ssh *ssh, const char *fmt,...) { char buf[1024]; va_list args; static int disconnecting = 0; int r; if (disconnecting) /* Guard against recursive invocations. */ fatal("packet_disconnect called recursively."); disconnecting = 1; /* * Format the message. Note that the caller must make sure the * message is of limited size. */ va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); /* Display the error locally */ logit("Disconnecting: %.100s", buf); /* * Send the disconnect message to the other side, and wait * for it to get sent. */ if ((r = sshpkt_disconnect(ssh, "%s", buf)) != 0) sshpkt_fatal(ssh, __func__, r); if ((r = ssh_packet_write_wait(ssh)) != 0) sshpkt_fatal(ssh, __func__, r); /* Close the connection. */ ssh_packet_close(ssh); cleanup_exit(255); } /* * Checks if there is any buffered output, and tries to write some of * the output. */ int ssh_packet_write_poll(struct ssh *ssh) { struct session_state *state = ssh->state; int len = sshbuf_len(state->output); int r; if (len > 0) { len = write(state->connection_out, sshbuf_ptr(state->output), len); if (len == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) return 0; return SSH_ERR_SYSTEM_ERROR; } if (len == 0) return SSH_ERR_CONN_CLOSED; if ((r = sshbuf_consume(state->output, len)) != 0) return r; } return 0; } /* * Calls packet_write_poll repeatedly until all pending output data has been * written. */ int ssh_packet_write_wait(struct ssh *ssh) { fd_set *setp; int ret, r, ms_remain = 0; struct timeval start, timeout, *timeoutp = NULL; struct session_state *state = ssh->state; setp = calloc(howmany(state->connection_out + 1, NFDBITS), sizeof(fd_mask)); if (setp == NULL) return SSH_ERR_ALLOC_FAIL; if ((r = ssh_packet_write_poll(ssh)) != 0) { free(setp); return r; } while (ssh_packet_have_data_to_write(ssh)) { memset(setp, 0, howmany(state->connection_out + 1, NFDBITS) * sizeof(fd_mask)); FD_SET(state->connection_out, setp); if (state->packet_timeout_ms > 0) { ms_remain = state->packet_timeout_ms; timeoutp = &timeout; } for (;;) { if (state->packet_timeout_ms != -1) { ms_to_timeval(&timeout, ms_remain); gettimeofday(&start, NULL); } if ((ret = select(state->connection_out + 1, NULL, setp, NULL, timeoutp)) >= 0) break; if (errno != EAGAIN && errno != EINTR && errno != EWOULDBLOCK) break; if (state->packet_timeout_ms == -1) continue; ms_subtract_diff(&start, &ms_remain); if (ms_remain <= 0) { ret = 0; break; } } if (ret == 0) { free(setp); return SSH_ERR_CONN_TIMEOUT; } if ((r = ssh_packet_write_poll(ssh)) != 0) { free(setp); return r; } } free(setp); return 0; } /* Returns true if there is buffered data to write to the connection. */ int ssh_packet_have_data_to_write(struct ssh *ssh) { return sshbuf_len(ssh->state->output) != 0; } /* Returns true if there is not too much data to write to the connection. */ int ssh_packet_not_very_much_data_to_write(struct ssh *ssh) { if (ssh->state->interactive_mode) return sshbuf_len(ssh->state->output) < 16384; else return sshbuf_len(ssh->state->output) < 128 * 1024; } void ssh_packet_set_tos(struct ssh *ssh, int tos) { #ifndef IP_TOS_IS_BROKEN if (!ssh_packet_connection_is_on_socket(ssh)) return; switch (ssh_packet_connection_af(ssh)) { # ifdef IP_TOS case AF_INET: debug3("%s: set IP_TOS 0x%02x", __func__, tos); if (setsockopt(ssh->state->connection_in, IPPROTO_IP, IP_TOS, &tos, sizeof(tos)) < 0) error("setsockopt IP_TOS %d: %.100s:", tos, strerror(errno)); break; # endif /* IP_TOS */ # ifdef IPV6_TCLASS case AF_INET6: debug3("%s: set IPV6_TCLASS 0x%02x", __func__, tos); if (setsockopt(ssh->state->connection_in, IPPROTO_IPV6, IPV6_TCLASS, &tos, sizeof(tos)) < 0) error("setsockopt IPV6_TCLASS %d: %.100s:", tos, strerror(errno)); break; # endif /* IPV6_TCLASS */ } #endif /* IP_TOS_IS_BROKEN */ } /* Informs that the current session is interactive. Sets IP flags for that. */ void ssh_packet_set_interactive(struct ssh *ssh, int interactive, int qos_interactive, int qos_bulk) { struct session_state *state = ssh->state; if (state->set_interactive_called) return; state->set_interactive_called = 1; /* Record that we are in interactive mode. */ state->interactive_mode = interactive; /* Only set socket options if using a socket. */ if (!ssh_packet_connection_is_on_socket(ssh)) return; set_nodelay(state->connection_in); ssh_packet_set_tos(ssh, interactive ? qos_interactive : qos_bulk); } /* Returns true if the current connection is interactive. */ int ssh_packet_is_interactive(struct ssh *ssh) { return ssh->state->interactive_mode; } int ssh_packet_set_maxsize(struct ssh *ssh, u_int s) { struct session_state *state = ssh->state; if (state->set_maxsize_called) { logit("packet_set_maxsize: called twice: old %d new %d", state->max_packet_size, s); return -1; } if (s < 4 * 1024 || s > 1024 * 1024) { logit("packet_set_maxsize: bad size %d", s); return -1; } state->set_maxsize_called = 1; debug("packet_set_maxsize: setting to %d", s); state->max_packet_size = s; return s; } int ssh_packet_inc_alive_timeouts(struct ssh *ssh) { return ++ssh->state->keep_alive_timeouts; } void ssh_packet_set_alive_timeouts(struct ssh *ssh, int ka) { ssh->state->keep_alive_timeouts = ka; } u_int ssh_packet_get_maxsize(struct ssh *ssh) { return ssh->state->max_packet_size; } /* * 9.2. Ignored Data Message * * byte SSH_MSG_IGNORE * string data * * All implementations MUST understand (and ignore) this message at any * time (after receiving the protocol version). No implementation is * required to send them. This message can be used as an additional * protection measure against advanced traffic analysis techniques. */ void ssh_packet_send_ignore(struct ssh *ssh, int nbytes) { u_int32_t rnd = 0; int r, i; if ((r = sshpkt_start(ssh, compat20 ? SSH2_MSG_IGNORE : SSH_MSG_IGNORE)) != 0 || (r = sshpkt_put_u32(ssh, nbytes)) != 0) fatal("%s: %s", __func__, ssh_err(r)); for (i = 0; i < nbytes; i++) { if (i % 4 == 0) rnd = arc4random(); if ((r = sshpkt_put_u8(ssh, (u_char)rnd & 0xff)) != 0) fatal("%s: %s", __func__, ssh_err(r)); rnd >>= 8; } } void ssh_packet_set_rekey_limits(struct ssh *ssh, u_int64_t bytes, time_t seconds) { debug3("rekey after %llu bytes, %d seconds", (unsigned long long)bytes, (int)seconds); ssh->state->rekey_limit = bytes; ssh->state->rekey_interval = seconds; } time_t ssh_packet_get_rekey_timeout(struct ssh *ssh) { time_t seconds; seconds = ssh->state->rekey_time + ssh->state->rekey_interval - monotime(); return (seconds <= 0 ? 1 : seconds); } void ssh_packet_set_server(struct ssh *ssh) { ssh->state->server_side = 1; } void ssh_packet_set_authenticated(struct ssh *ssh) { ssh->state->after_authentication = 1; } void * ssh_packet_get_input(struct ssh *ssh) { return (void *)ssh->state->input; } void * ssh_packet_get_output(struct ssh *ssh) { return (void *)ssh->state->output; } /* Reset after_authentication and reset compression in post-auth privsep */ static int ssh_packet_set_postauth(struct ssh *ssh) { struct sshcomp *comp; int r, mode; debug("%s: called", __func__); /* This was set in net child, but is not visible in user child */ ssh->state->after_authentication = 1; ssh->state->rekeying = 0; for (mode = 0; mode < MODE_MAX; mode++) { if (ssh->state->newkeys[mode] == NULL) continue; comp = &ssh->state->newkeys[mode]->comp; if (comp && comp->enabled && (r = ssh_packet_init_compression(ssh)) != 0) return r; } return 0; } /* Packet state (de-)serialization for privsep */ /* turn kex into a blob for packet state serialization */ static int kex_to_blob(struct sshbuf *m, struct kex *kex) { int r; if ((r = sshbuf_put_string(m, kex->session_id, kex->session_id_len)) != 0 || (r = sshbuf_put_u32(m, kex->we_need)) != 0 || (r = sshbuf_put_u32(m, kex->hostkey_type)) != 0 || (r = sshbuf_put_u32(m, kex->kex_type)) != 0 || (r = sshbuf_put_stringb(m, kex->my)) != 0 || (r = sshbuf_put_stringb(m, kex->peer)) != 0 || (r = sshbuf_put_u32(m, kex->flags)) != 0 || (r = sshbuf_put_cstring(m, kex->client_version_string)) != 0 || (r = sshbuf_put_cstring(m, kex->server_version_string)) != 0) return r; return 0; } /* turn key exchange results into a blob for packet state serialization */ static int newkeys_to_blob(struct sshbuf *m, struct ssh *ssh, int mode) { struct sshbuf *b; struct sshcipher_ctx *cc; struct sshcomp *comp; struct sshenc *enc; struct sshmac *mac; struct newkeys *newkey; int r; if ((newkey = ssh->state->newkeys[mode]) == NULL) return SSH_ERR_INTERNAL_ERROR; enc = &newkey->enc; mac = &newkey->mac; comp = &newkey->comp; cc = (mode == MODE_OUT) ? &ssh->state->send_context : &ssh->state->receive_context; if ((r = cipher_get_keyiv(cc, enc->iv, enc->iv_len)) != 0) return r; if ((b = sshbuf_new()) == NULL) return SSH_ERR_ALLOC_FAIL; /* The cipher struct is constant and shared, you export pointer */ if ((r = sshbuf_put_cstring(b, enc->name)) != 0 || (r = sshbuf_put(b, &enc->cipher, sizeof(enc->cipher))) != 0 || (r = sshbuf_put_u32(b, enc->enabled)) != 0 || (r = sshbuf_put_u32(b, enc->block_size)) != 0 || (r = sshbuf_put_string(b, enc->key, enc->key_len)) != 0 || (r = sshbuf_put_string(b, enc->iv, enc->iv_len)) != 0) goto out; if (cipher_authlen(enc->cipher) == 0) { if ((r = sshbuf_put_cstring(b, mac->name)) != 0 || (r = sshbuf_put_u32(b, mac->enabled)) != 0 || (r = sshbuf_put_string(b, mac->key, mac->key_len)) != 0) goto out; } if ((r = sshbuf_put_u32(b, comp->type)) != 0 || (r = sshbuf_put_u32(b, comp->enabled)) != 0 || (r = sshbuf_put_cstring(b, comp->name)) != 0) goto out; r = sshbuf_put_stringb(m, b); out: sshbuf_free(b); return r; } /* serialize packet state into a blob */ int ssh_packet_get_state(struct ssh *ssh, struct sshbuf *m) { struct session_state *state = ssh->state; u_char *p; size_t slen, rlen; int r, ssh1cipher; if (!compat20) { ssh1cipher = cipher_get_number(state->receive_context.cipher); slen = cipher_get_keyiv_len(&state->send_context); rlen = cipher_get_keyiv_len(&state->receive_context); if ((r = sshbuf_put_u32(m, state->remote_protocol_flags)) != 0 || (r = sshbuf_put_u32(m, ssh1cipher)) != 0 || (r = sshbuf_put_string(m, state->ssh1_key, state->ssh1_keylen)) != 0 || (r = sshbuf_put_u32(m, slen)) != 0 || (r = sshbuf_reserve(m, slen, &p)) != 0 || (r = cipher_get_keyiv(&state->send_context, p, slen)) != 0 || (r = sshbuf_put_u32(m, rlen)) != 0 || (r = sshbuf_reserve(m, rlen, &p)) != 0 || (r = cipher_get_keyiv(&state->receive_context, p, rlen)) != 0) return r; } else { if ((r = kex_to_blob(m, ssh->kex)) != 0 || (r = newkeys_to_blob(m, ssh, MODE_OUT)) != 0 || (r = newkeys_to_blob(m, ssh, MODE_IN)) != 0 || (r = sshbuf_put_u64(m, state->rekey_limit)) != 0 || (r = sshbuf_put_u32(m, state->rekey_interval)) != 0 || (r = sshbuf_put_u32(m, state->p_send.seqnr)) != 0 || (r = sshbuf_put_u64(m, state->p_send.blocks)) != 0 || (r = sshbuf_put_u32(m, state->p_send.packets)) != 0 || (r = sshbuf_put_u64(m, state->p_send.bytes)) != 0 || (r = sshbuf_put_u32(m, state->p_read.seqnr)) != 0 || (r = sshbuf_put_u64(m, state->p_read.blocks)) != 0 || (r = sshbuf_put_u32(m, state->p_read.packets)) != 0 || (r = sshbuf_put_u64(m, state->p_read.bytes)) != 0) return r; } slen = cipher_get_keycontext(&state->send_context, NULL); rlen = cipher_get_keycontext(&state->receive_context, NULL); if ((r = sshbuf_put_u32(m, slen)) != 0 || (r = sshbuf_reserve(m, slen, &p)) != 0) return r; if (cipher_get_keycontext(&state->send_context, p) != (int)slen) return SSH_ERR_INTERNAL_ERROR; if ((r = sshbuf_put_u32(m, rlen)) != 0 || (r = sshbuf_reserve(m, rlen, &p)) != 0) return r; if (cipher_get_keycontext(&state->receive_context, p) != (int)rlen) return SSH_ERR_INTERNAL_ERROR; if ((r = ssh_packet_get_compress_state(m, ssh)) != 0 || (r = sshbuf_put_stringb(m, state->input)) != 0 || (r = sshbuf_put_stringb(m, state->output)) != 0) return r; return 0; } /* restore key exchange results from blob for packet state de-serialization */ static int newkeys_from_blob(struct sshbuf *m, struct ssh *ssh, int mode) { struct sshbuf *b = NULL; struct sshcomp *comp; struct sshenc *enc; struct sshmac *mac; struct newkeys *newkey = NULL; size_t keylen, ivlen, maclen; int r; if ((newkey = calloc(1, sizeof(*newkey))) == NULL) { r = SSH_ERR_ALLOC_FAIL; goto out; } if ((r = sshbuf_froms(m, &b)) != 0) goto out; #ifdef DEBUG_PK sshbuf_dump(b, stderr); #endif enc = &newkey->enc; mac = &newkey->mac; comp = &newkey->comp; if ((r = sshbuf_get_cstring(b, &enc->name, NULL)) != 0 || (r = sshbuf_get(b, &enc->cipher, sizeof(enc->cipher))) != 0 || (r = sshbuf_get_u32(b, (u_int *)&enc->enabled)) != 0 || (r = sshbuf_get_u32(b, &enc->block_size)) != 0 || (r = sshbuf_get_string(b, &enc->key, &keylen)) != 0 || (r = sshbuf_get_string(b, &enc->iv, &ivlen)) != 0) goto out; if (cipher_authlen(enc->cipher) == 0) { if ((r = sshbuf_get_cstring(b, &mac->name, NULL)) != 0) goto out; if ((r = mac_setup(mac, mac->name)) != 0) goto out; if ((r = sshbuf_get_u32(b, (u_int *)&mac->enabled)) != 0 || (r = sshbuf_get_string(b, &mac->key, &maclen)) != 0) goto out; if (maclen > mac->key_len) { r = SSH_ERR_INVALID_FORMAT; goto out; } mac->key_len = maclen; } if ((r = sshbuf_get_u32(b, &comp->type)) != 0 || (r = sshbuf_get_u32(b, (u_int *)&comp->enabled)) != 0 || (r = sshbuf_get_cstring(b, &comp->name, NULL)) != 0) goto out; if (enc->name == NULL || cipher_by_name(enc->name) != enc->cipher) { r = SSH_ERR_INVALID_FORMAT; goto out; } if (sshbuf_len(b) != 0) { r = SSH_ERR_INVALID_FORMAT; goto out; } enc->key_len = keylen; enc->iv_len = ivlen; ssh->kex->newkeys[mode] = newkey; newkey = NULL; r = 0; out: free(newkey); sshbuf_free(b); return r; } /* restore kex from blob for packet state de-serialization */ static int kex_from_blob(struct sshbuf *m, struct kex **kexp) { struct kex *kex; int r; if ((kex = calloc(1, sizeof(struct kex))) == NULL || (kex->my = sshbuf_new()) == NULL || (kex->peer = sshbuf_new()) == NULL) { r = SSH_ERR_ALLOC_FAIL; goto out; } if ((r = sshbuf_get_string(m, &kex->session_id, &kex->session_id_len)) != 0 || (r = sshbuf_get_u32(m, &kex->we_need)) != 0 || (r = sshbuf_get_u32(m, (u_int *)&kex->hostkey_type)) != 0 || (r = sshbuf_get_u32(m, &kex->kex_type)) != 0 || (r = sshbuf_get_stringb(m, kex->my)) != 0 || (r = sshbuf_get_stringb(m, kex->peer)) != 0 || (r = sshbuf_get_u32(m, &kex->flags)) != 0 || (r = sshbuf_get_cstring(m, &kex->client_version_string, NULL)) != 0 || (r = sshbuf_get_cstring(m, &kex->server_version_string, NULL)) != 0) goto out; kex->server = 1; kex->done = 1; r = 0; out: if (r != 0 || kexp == NULL) { if (kex != NULL) { sshbuf_free(kex->my); sshbuf_free(kex->peer); free(kex); } if (kexp != NULL) *kexp = NULL; } else { *kexp = kex; } return r; } /* * Restore packet state from content of blob 'm' (de-serialization). * Note that 'm' will be partially consumed on parsing or any other errors. */ int ssh_packet_set_state(struct ssh *ssh, struct sshbuf *m) { struct session_state *state = ssh->state; const u_char *ssh1key, *ivin, *ivout, *keyin, *keyout, *input, *output; size_t ssh1keylen, rlen, slen, ilen, olen; int r; u_int ssh1cipher = 0; if (!compat20) { if ((r = sshbuf_get_u32(m, &state->remote_protocol_flags)) != 0 || (r = sshbuf_get_u32(m, &ssh1cipher)) != 0 || (r = sshbuf_get_string_direct(m, &ssh1key, &ssh1keylen)) != 0 || (r = sshbuf_get_string_direct(m, &ivout, &slen)) != 0 || (r = sshbuf_get_string_direct(m, &ivin, &rlen)) != 0) return r; if (ssh1cipher > INT_MAX) return SSH_ERR_KEY_UNKNOWN_CIPHER; ssh_packet_set_encryption_key(ssh, ssh1key, ssh1keylen, (int)ssh1cipher); if (cipher_get_keyiv_len(&state->send_context) != (int)slen || cipher_get_keyiv_len(&state->receive_context) != (int)rlen) return SSH_ERR_INVALID_FORMAT; if ((r = cipher_set_keyiv(&state->send_context, ivout)) != 0 || (r = cipher_set_keyiv(&state->receive_context, ivin)) != 0) return r; } else { if ((r = kex_from_blob(m, &ssh->kex)) != 0 || (r = newkeys_from_blob(m, ssh, MODE_OUT)) != 0 || (r = newkeys_from_blob(m, ssh, MODE_IN)) != 0 || (r = sshbuf_get_u64(m, &state->rekey_limit)) != 0 || (r = sshbuf_get_u32(m, &state->rekey_interval)) != 0 || (r = sshbuf_get_u32(m, &state->p_send.seqnr)) != 0 || (r = sshbuf_get_u64(m, &state->p_send.blocks)) != 0 || (r = sshbuf_get_u32(m, &state->p_send.packets)) != 0 || (r = sshbuf_get_u64(m, &state->p_send.bytes)) != 0 || (r = sshbuf_get_u32(m, &state->p_read.seqnr)) != 0 || (r = sshbuf_get_u64(m, &state->p_read.blocks)) != 0 || (r = sshbuf_get_u32(m, &state->p_read.packets)) != 0 || (r = sshbuf_get_u64(m, &state->p_read.bytes)) != 0) return r; /* * We set the time here so that in post-auth privsep slave we * count from the completion of the authentication. */ state->rekey_time = monotime(); /* XXX ssh_set_newkeys overrides p_read.packets? XXX */ if ((r = ssh_set_newkeys(ssh, MODE_IN)) != 0 || (r = ssh_set_newkeys(ssh, MODE_OUT)) != 0) return r; } if ((r = sshbuf_get_string_direct(m, &keyout, &slen)) != 0 || (r = sshbuf_get_string_direct(m, &keyin, &rlen)) != 0) return r; if (cipher_get_keycontext(&state->send_context, NULL) != (int)slen || cipher_get_keycontext(&state->receive_context, NULL) != (int)rlen) return SSH_ERR_INVALID_FORMAT; cipher_set_keycontext(&state->send_context, keyout); cipher_set_keycontext(&state->receive_context, keyin); if ((r = ssh_packet_set_compress_state(ssh, m)) != 0 || (r = ssh_packet_set_postauth(ssh)) != 0) return r; sshbuf_reset(state->input); sshbuf_reset(state->output); if ((r = sshbuf_get_string_direct(m, &input, &ilen)) != 0 || (r = sshbuf_get_string_direct(m, &output, &olen)) != 0 || (r = sshbuf_put(state->input, input, ilen)) != 0 || (r = sshbuf_put(state->output, output, olen)) != 0) return r; if (sshbuf_len(m)) return SSH_ERR_INVALID_FORMAT; debug3("%s: done", __func__); return 0; } /* NEW API */ /* put data to the outgoing packet */ int sshpkt_put(struct ssh *ssh, const void *v, size_t len) { return sshbuf_put(ssh->state->outgoing_packet, v, len); } int sshpkt_putb(struct ssh *ssh, const struct sshbuf *b) { return sshbuf_putb(ssh->state->outgoing_packet, b); } int sshpkt_put_u8(struct ssh *ssh, u_char val) { return sshbuf_put_u8(ssh->state->outgoing_packet, val); } int sshpkt_put_u32(struct ssh *ssh, u_int32_t val) { return sshbuf_put_u32(ssh->state->outgoing_packet, val); } int sshpkt_put_u64(struct ssh *ssh, u_int64_t val) { return sshbuf_put_u64(ssh->state->outgoing_packet, val); } int sshpkt_put_string(struct ssh *ssh, const void *v, size_t len) { return sshbuf_put_string(ssh->state->outgoing_packet, v, len); } int sshpkt_put_cstring(struct ssh *ssh, const void *v) { return sshbuf_put_cstring(ssh->state->outgoing_packet, v); } int sshpkt_put_stringb(struct ssh *ssh, const struct sshbuf *v) { return sshbuf_put_stringb(ssh->state->outgoing_packet, v); } #ifdef WITH_OPENSSL #ifdef OPENSSL_HAS_ECC int sshpkt_put_ec(struct ssh *ssh, const EC_POINT *v, const EC_GROUP *g) { return sshbuf_put_ec(ssh->state->outgoing_packet, v, g); } #endif /* OPENSSL_HAS_ECC */ #ifdef WITH_SSH1 int sshpkt_put_bignum1(struct ssh *ssh, const BIGNUM *v) { return sshbuf_put_bignum1(ssh->state->outgoing_packet, v); } #endif /* WITH_SSH1 */ int sshpkt_put_bignum2(struct ssh *ssh, const BIGNUM *v) { return sshbuf_put_bignum2(ssh->state->outgoing_packet, v); } #endif /* WITH_OPENSSL */ /* fetch data from the incoming packet */ int sshpkt_get(struct ssh *ssh, void *valp, size_t len) { return sshbuf_get(ssh->state->incoming_packet, valp, len); } int sshpkt_get_u8(struct ssh *ssh, u_char *valp) { return sshbuf_get_u8(ssh->state->incoming_packet, valp); } int sshpkt_get_u32(struct ssh *ssh, u_int32_t *valp) { return sshbuf_get_u32(ssh->state->incoming_packet, valp); } int sshpkt_get_u64(struct ssh *ssh, u_int64_t *valp) { return sshbuf_get_u64(ssh->state->incoming_packet, valp); } int sshpkt_get_string(struct ssh *ssh, u_char **valp, size_t *lenp) { return sshbuf_get_string(ssh->state->incoming_packet, valp, lenp); } int sshpkt_get_string_direct(struct ssh *ssh, const u_char **valp, size_t *lenp) { return sshbuf_get_string_direct(ssh->state->incoming_packet, valp, lenp); } int sshpkt_get_cstring(struct ssh *ssh, char **valp, size_t *lenp) { return sshbuf_get_cstring(ssh->state->incoming_packet, valp, lenp); } #ifdef WITH_OPENSSL #ifdef OPENSSL_HAS_ECC int sshpkt_get_ec(struct ssh *ssh, EC_POINT *v, const EC_GROUP *g) { return sshbuf_get_ec(ssh->state->incoming_packet, v, g); } #endif /* OPENSSL_HAS_ECC */ #ifdef WITH_SSH1 int sshpkt_get_bignum1(struct ssh *ssh, BIGNUM *v) { return sshbuf_get_bignum1(ssh->state->incoming_packet, v); } #endif /* WITH_SSH1 */ int sshpkt_get_bignum2(struct ssh *ssh, BIGNUM *v) { return sshbuf_get_bignum2(ssh->state->incoming_packet, v); } #endif /* WITH_OPENSSL */ int sshpkt_get_end(struct ssh *ssh) { if (sshbuf_len(ssh->state->incoming_packet) > 0) return SSH_ERR_UNEXPECTED_TRAILING_DATA; return 0; } const u_char * sshpkt_ptr(struct ssh *ssh, size_t *lenp) { if (lenp != NULL) *lenp = sshbuf_len(ssh->state->incoming_packet); return sshbuf_ptr(ssh->state->incoming_packet); } /* start a new packet */ int sshpkt_start(struct ssh *ssh, u_char type) { u_char buf[9]; int len; DBG(debug("packet_start[%d]", type)); len = compat20 ? 6 : 9; memset(buf, 0, len - 1); buf[len - 1] = type; sshbuf_reset(ssh->state->outgoing_packet); return sshbuf_put(ssh->state->outgoing_packet, buf, len); } /* send it */ int sshpkt_send(struct ssh *ssh) { if (compat20) return ssh_packet_send2(ssh); else return ssh_packet_send1(ssh); } int sshpkt_disconnect(struct ssh *ssh, const char *fmt,...) { char buf[1024]; va_list args; int r; va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); if (compat20) { if ((r = sshpkt_start(ssh, SSH2_MSG_DISCONNECT)) != 0 || (r = sshpkt_put_u32(ssh, SSH2_DISCONNECT_PROTOCOL_ERROR)) != 0 || (r = sshpkt_put_cstring(ssh, buf)) != 0 || (r = sshpkt_put_cstring(ssh, "")) != 0 || (r = sshpkt_send(ssh)) != 0) return r; } else { if ((r = sshpkt_start(ssh, SSH_MSG_DISCONNECT)) != 0 || (r = sshpkt_put_cstring(ssh, buf)) != 0 || (r = sshpkt_send(ssh)) != 0) return r; } return 0; } /* roundup current message to pad bytes */ int sshpkt_add_padding(struct ssh *ssh, u_char pad) { ssh->state->extra_pad = pad; return 0; } Index: projects/clang390-import/crypto/openssh/servconf.c =================================================================== --- projects/clang390-import/crypto/openssh/servconf.c (revision 305080) +++ projects/clang390-import/crypto/openssh/servconf.c (revision 305081) @@ -1,2377 +1,2387 @@ /* $OpenBSD: servconf.c,v 1.285 2016/02/17 05:29:04 djm Exp $ */ /* * Copyright (c) 1995 Tatu Ylonen , Espoo, Finland * All rights reserved * * As far as I am concerned, the code I have written for this software * can be used freely for any purpose. Any derived versions of this * software must be clearly marked as such, and if the derived work is * incompatible with the protocol description in the RFC file, it must be * called by a name other than "ssh" or "Secure Shell". */ #include "includes.h" __RCSID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_UTIL_H #include #endif #include "openbsd-compat/sys-queue.h" #include "xmalloc.h" #include "ssh.h" #include "log.h" #include "buffer.h" #include "misc.h" #include "servconf.h" #include "compat.h" #include "pathnames.h" #include "cipher.h" #include "key.h" #include "kex.h" #include "mac.h" #include "match.h" #include "channels.h" #include "groupaccess.h" #include "canohost.h" #include "packet.h" #include "hostfile.h" #include "auth.h" #include "myproposal.h" #include "digest.h" #include "version.h" static void add_listen_addr(ServerOptions *, char *, int); static void add_one_listen_addr(ServerOptions *, char *, int); /* Use of privilege separation or not */ extern int use_privsep; extern Buffer cfg; /* Initializes the server options to their default values. */ void initialize_server_options(ServerOptions *options) { memset(options, 0, sizeof(*options)); /* Portable-specific options */ options->use_pam = -1; /* Standard Options */ options->num_ports = 0; options->ports_from_cmdline = 0; options->queued_listen_addrs = NULL; options->num_queued_listens = 0; options->listen_addrs = NULL; options->address_family = -1; options->num_host_key_files = 0; options->num_host_cert_files = 0; options->host_key_agent = NULL; options->pid_file = NULL; options->server_key_bits = -1; options->login_grace_time = -1; options->key_regeneration_time = -1; options->permit_root_login = PERMIT_NOT_SET; options->ignore_rhosts = -1; options->ignore_user_known_hosts = -1; options->print_motd = -1; options->print_lastlog = -1; options->x11_forwarding = -1; options->x11_display_offset = -1; options->x11_use_localhost = -1; options->permit_tty = -1; options->permit_user_rc = -1; options->xauth_location = NULL; options->strict_modes = -1; options->tcp_keep_alive = -1; options->log_facility = SYSLOG_FACILITY_NOT_SET; options->log_level = SYSLOG_LEVEL_NOT_SET; options->rhosts_rsa_authentication = -1; options->hostbased_authentication = -1; options->hostbased_uses_name_from_packet_only = -1; options->hostbased_key_types = NULL; options->hostkeyalgorithms = NULL; options->rsa_authentication = -1; options->pubkey_authentication = -1; options->pubkey_key_types = NULL; options->kerberos_authentication = -1; options->kerberos_or_local_passwd = -1; options->kerberos_ticket_cleanup = -1; options->kerberos_get_afs_token = -1; options->gss_authentication=-1; options->gss_cleanup_creds = -1; options->gss_strict_acceptor = -1; options->password_authentication = -1; options->kbd_interactive_authentication = -1; options->challenge_response_authentication = -1; options->permit_empty_passwd = -1; options->permit_user_env = -1; options->use_login = -1; options->compression = -1; options->rekey_limit = -1; options->rekey_interval = -1; options->allow_tcp_forwarding = -1; options->allow_streamlocal_forwarding = -1; options->allow_agent_forwarding = -1; options->num_allow_users = 0; options->num_deny_users = 0; options->num_allow_groups = 0; options->num_deny_groups = 0; options->ciphers = NULL; options->macs = NULL; options->kex_algorithms = NULL; options->protocol = SSH_PROTO_UNKNOWN; options->fwd_opts.gateway_ports = -1; options->fwd_opts.streamlocal_bind_mask = (mode_t)-1; options->fwd_opts.streamlocal_bind_unlink = -1; options->num_subsystems = 0; options->max_startups_begin = -1; options->max_startups_rate = -1; options->max_startups = -1; options->max_authtries = -1; options->max_sessions = -1; options->banner = NULL; options->use_dns = -1; options->client_alive_interval = -1; options->client_alive_count_max = -1; options->num_authkeys_files = 0; options->num_accept_env = 0; options->permit_tun = -1; options->num_permitted_opens = -1; options->adm_forced_command = NULL; options->chroot_directory = NULL; options->authorized_keys_command = NULL; options->authorized_keys_command_user = NULL; options->revoked_keys_file = NULL; options->trusted_user_ca_keys = NULL; options->authorized_principals_file = NULL; options->authorized_principals_command = NULL; options->authorized_principals_command_user = NULL; options->ip_qos_interactive = -1; options->ip_qos_bulk = -1; options->version_addendum = NULL; options->fingerprint_hash = -1; + options->use_blacklist = -1; } /* Returns 1 if a string option is unset or set to "none" or 0 otherwise. */ static int option_clear_or_none(const char *o) { return o == NULL || strcasecmp(o, "none") == 0; } static void assemble_algorithms(ServerOptions *o) { if (kex_assemble_names(KEX_SERVER_ENCRYPT, &o->ciphers) != 0 || kex_assemble_names(KEX_SERVER_MAC, &o->macs) != 0 || kex_assemble_names(KEX_SERVER_KEX, &o->kex_algorithms) != 0 || kex_assemble_names(KEX_DEFAULT_PK_ALG, &o->hostkeyalgorithms) != 0 || kex_assemble_names(KEX_DEFAULT_PK_ALG, &o->hostbased_key_types) != 0 || kex_assemble_names(KEX_DEFAULT_PK_ALG, &o->pubkey_key_types) != 0) fatal("kex_assemble_names failed"); } void fill_default_server_options(ServerOptions *options) { int i; /* Portable-specific options */ if (options->use_pam == -1) options->use_pam = 1; /* Standard Options */ if (options->protocol == SSH_PROTO_UNKNOWN) options->protocol = SSH_PROTO_2; #define add_host_key_file(path) \ do { \ if (access((path), O_RDONLY) == 0) \ options->host_key_files \ [options->num_host_key_files++] = (path); \ } while (0) if (options->num_host_key_files == 0) { /* fill default hostkeys for protocols */ if (options->protocol & SSH_PROTO_1) add_host_key_file(_PATH_HOST_KEY_FILE); if (options->protocol & SSH_PROTO_2) { add_host_key_file(_PATH_HOST_RSA_KEY_FILE); add_host_key_file(_PATH_HOST_DSA_KEY_FILE); #ifdef OPENSSL_HAS_ECC add_host_key_file(_PATH_HOST_ECDSA_KEY_FILE); #endif add_host_key_file(_PATH_HOST_ED25519_KEY_FILE); } } #undef add_host_key_file if (options->num_host_key_files == 0) fatal("No host key files found"); /* No certificates by default */ if (options->num_ports == 0) options->ports[options->num_ports++] = SSH_DEFAULT_PORT; if (options->address_family == -1) options->address_family = AF_UNSPEC; if (options->listen_addrs == NULL) add_listen_addr(options, NULL, 0); if (options->pid_file == NULL) options->pid_file = xstrdup(_PATH_SSH_DAEMON_PID_FILE); if (options->server_key_bits == -1) options->server_key_bits = 1024; if (options->login_grace_time == -1) options->login_grace_time = 120; if (options->key_regeneration_time == -1) options->key_regeneration_time = 3600; if (options->permit_root_login == PERMIT_NOT_SET) options->permit_root_login = PERMIT_NO; if (options->ignore_rhosts == -1) options->ignore_rhosts = 1; if (options->ignore_user_known_hosts == -1) options->ignore_user_known_hosts = 0; if (options->print_motd == -1) options->print_motd = 1; if (options->print_lastlog == -1) options->print_lastlog = 1; if (options->x11_forwarding == -1) options->x11_forwarding = 1; if (options->x11_display_offset == -1) options->x11_display_offset = 10; if (options->x11_use_localhost == -1) options->x11_use_localhost = 1; if (options->xauth_location == NULL) options->xauth_location = xstrdup(_PATH_XAUTH); if (options->permit_tty == -1) options->permit_tty = 1; if (options->permit_user_rc == -1) options->permit_user_rc = 1; if (options->strict_modes == -1) options->strict_modes = 1; if (options->tcp_keep_alive == -1) options->tcp_keep_alive = 1; if (options->log_facility == SYSLOG_FACILITY_NOT_SET) options->log_facility = SYSLOG_FACILITY_AUTH; if (options->log_level == SYSLOG_LEVEL_NOT_SET) options->log_level = SYSLOG_LEVEL_INFO; if (options->rhosts_rsa_authentication == -1) options->rhosts_rsa_authentication = 0; if (options->hostbased_authentication == -1) options->hostbased_authentication = 0; if (options->hostbased_uses_name_from_packet_only == -1) options->hostbased_uses_name_from_packet_only = 0; if (options->rsa_authentication == -1) options->rsa_authentication = 1; if (options->pubkey_authentication == -1) options->pubkey_authentication = 1; if (options->kerberos_authentication == -1) options->kerberos_authentication = 0; if (options->kerberos_or_local_passwd == -1) options->kerberos_or_local_passwd = 1; if (options->kerberos_ticket_cleanup == -1) options->kerberos_ticket_cleanup = 1; if (options->kerberos_get_afs_token == -1) options->kerberos_get_afs_token = 0; if (options->gss_authentication == -1) options->gss_authentication = 0; if (options->gss_cleanup_creds == -1) options->gss_cleanup_creds = 1; if (options->gss_strict_acceptor == -1) options->gss_strict_acceptor = 0; if (options->password_authentication == -1) options->password_authentication = 0; if (options->kbd_interactive_authentication == -1) options->kbd_interactive_authentication = 0; if (options->challenge_response_authentication == -1) options->challenge_response_authentication = 1; if (options->permit_empty_passwd == -1) options->permit_empty_passwd = 0; if (options->permit_user_env == -1) options->permit_user_env = 0; if (options->use_login == -1) options->use_login = 0; if (options->compression == -1) options->compression = COMP_DELAYED; if (options->rekey_limit == -1) options->rekey_limit = 0; if (options->rekey_interval == -1) options->rekey_interval = 0; if (options->allow_tcp_forwarding == -1) options->allow_tcp_forwarding = FORWARD_ALLOW; if (options->allow_streamlocal_forwarding == -1) options->allow_streamlocal_forwarding = FORWARD_ALLOW; if (options->allow_agent_forwarding == -1) options->allow_agent_forwarding = 1; if (options->fwd_opts.gateway_ports == -1) options->fwd_opts.gateway_ports = 0; if (options->max_startups == -1) options->max_startups = 100; if (options->max_startups_rate == -1) options->max_startups_rate = 30; /* 30% */ if (options->max_startups_begin == -1) options->max_startups_begin = 10; if (options->max_authtries == -1) options->max_authtries = DEFAULT_AUTH_FAIL_MAX; if (options->max_sessions == -1) options->max_sessions = DEFAULT_SESSIONS_MAX; if (options->use_dns == -1) options->use_dns = 1; if (options->client_alive_interval == -1) options->client_alive_interval = 0; if (options->client_alive_count_max == -1) options->client_alive_count_max = 3; if (options->num_authkeys_files == 0) { options->authorized_keys_files[options->num_authkeys_files++] = xstrdup(_PATH_SSH_USER_PERMITTED_KEYS); options->authorized_keys_files[options->num_authkeys_files++] = xstrdup(_PATH_SSH_USER_PERMITTED_KEYS2); } if (options->permit_tun == -1) options->permit_tun = SSH_TUNMODE_NO; if (options->ip_qos_interactive == -1) options->ip_qos_interactive = IPTOS_LOWDELAY; if (options->ip_qos_bulk == -1) options->ip_qos_bulk = IPTOS_THROUGHPUT; if (options->version_addendum == NULL) options->version_addendum = xstrdup(SSH_VERSION_FREEBSD); if (options->fwd_opts.streamlocal_bind_mask == (mode_t)-1) options->fwd_opts.streamlocal_bind_mask = 0177; if (options->fwd_opts.streamlocal_bind_unlink == -1) options->fwd_opts.streamlocal_bind_unlink = 0; if (options->fingerprint_hash == -1) options->fingerprint_hash = SSH_FP_HASH_DEFAULT; + if (options->use_blacklist == -1) + options->use_blacklist = 0; assemble_algorithms(options); /* Turn privilege separation and sandboxing on by default */ if (use_privsep == -1) use_privsep = PRIVSEP_ON; #define CLEAR_ON_NONE(v) \ do { \ if (option_clear_or_none(v)) { \ free(v); \ v = NULL; \ } \ } while(0) CLEAR_ON_NONE(options->pid_file); CLEAR_ON_NONE(options->xauth_location); CLEAR_ON_NONE(options->banner); CLEAR_ON_NONE(options->trusted_user_ca_keys); CLEAR_ON_NONE(options->revoked_keys_file); CLEAR_ON_NONE(options->authorized_principals_file); CLEAR_ON_NONE(options->adm_forced_command); CLEAR_ON_NONE(options->chroot_directory); for (i = 0; i < options->num_host_key_files; i++) CLEAR_ON_NONE(options->host_key_files[i]); for (i = 0; i < options->num_host_cert_files; i++) CLEAR_ON_NONE(options->host_cert_files[i]); #undef CLEAR_ON_NONE #ifndef HAVE_MMAP if (use_privsep && options->compression == 1) { error("This platform does not support both privilege " "separation and compression"); error("Compression disabled"); options->compression = 0; } #endif } /* Keyword tokens. */ typedef enum { sBadOption, /* == unknown option */ /* Portable-specific options */ sUsePAM, /* Standard Options */ sPort, sHostKeyFile, sServerKeyBits, sLoginGraceTime, sKeyRegenerationTime, sPermitRootLogin, sLogFacility, sLogLevel, sRhostsRSAAuthentication, sRSAAuthentication, sKerberosAuthentication, sKerberosOrLocalPasswd, sKerberosTicketCleanup, sKerberosGetAFSToken, sKerberosTgtPassing, sChallengeResponseAuthentication, sPasswordAuthentication, sKbdInteractiveAuthentication, sListenAddress, sAddressFamily, sPrintMotd, sPrintLastLog, sIgnoreRhosts, sX11Forwarding, sX11DisplayOffset, sX11UseLocalhost, sPermitTTY, sStrictModes, sEmptyPasswd, sTCPKeepAlive, sPermitUserEnvironment, sUseLogin, sAllowTcpForwarding, sCompression, sRekeyLimit, sAllowUsers, sDenyUsers, sAllowGroups, sDenyGroups, sIgnoreUserKnownHosts, sCiphers, sMacs, sProtocol, sPidFile, sGatewayPorts, sPubkeyAuthentication, sPubkeyAcceptedKeyTypes, sXAuthLocation, sSubsystem, sMaxStartups, sMaxAuthTries, sMaxSessions, sBanner, sUseDNS, sHostbasedAuthentication, sHostbasedUsesNameFromPacketOnly, sHostbasedAcceptedKeyTypes, sHostKeyAlgorithms, sClientAliveInterval, sClientAliveCountMax, sAuthorizedKeysFile, sGssAuthentication, sGssCleanupCreds, sGssStrictAcceptor, sAcceptEnv, sPermitTunnel, sMatch, sPermitOpen, sForceCommand, sChrootDirectory, sUsePrivilegeSeparation, sAllowAgentForwarding, sHostCertificate, sRevokedKeys, sTrustedUserCAKeys, sAuthorizedPrincipalsFile, sAuthorizedPrincipalsCommand, sAuthorizedPrincipalsCommandUser, sKexAlgorithms, sIPQoS, sVersionAddendum, sAuthorizedKeysCommand, sAuthorizedKeysCommandUser, sAuthenticationMethods, sHostKeyAgent, sPermitUserRC, sStreamLocalBindMask, sStreamLocalBindUnlink, sAllowStreamLocalForwarding, sFingerprintHash, + sUseBlacklist, sDeprecated, sUnsupported } ServerOpCodes; #define SSHCFG_GLOBAL 0x01 /* allowed in main section of sshd_config */ #define SSHCFG_MATCH 0x02 /* allowed inside a Match section */ #define SSHCFG_ALL (SSHCFG_GLOBAL|SSHCFG_MATCH) /* Textual representation of the tokens. */ static struct { const char *name; ServerOpCodes opcode; u_int flags; } keywords[] = { /* Portable-specific options */ #ifdef USE_PAM { "usepam", sUsePAM, SSHCFG_GLOBAL }, #else { "usepam", sUnsupported, SSHCFG_GLOBAL }, #endif { "pamauthenticationviakbdint", sDeprecated, SSHCFG_GLOBAL }, /* Standard Options */ { "port", sPort, SSHCFG_GLOBAL }, { "hostkey", sHostKeyFile, SSHCFG_GLOBAL }, { "hostdsakey", sHostKeyFile, SSHCFG_GLOBAL }, /* alias */ { "hostkeyagent", sHostKeyAgent, SSHCFG_GLOBAL }, { "pidfile", sPidFile, SSHCFG_GLOBAL }, { "serverkeybits", sServerKeyBits, SSHCFG_GLOBAL }, { "logingracetime", sLoginGraceTime, SSHCFG_GLOBAL }, { "keyregenerationinterval", sKeyRegenerationTime, SSHCFG_GLOBAL }, { "permitrootlogin", sPermitRootLogin, SSHCFG_ALL }, { "syslogfacility", sLogFacility, SSHCFG_GLOBAL }, { "loglevel", sLogLevel, SSHCFG_GLOBAL }, { "rhostsauthentication", sDeprecated, SSHCFG_GLOBAL }, { "rhostsrsaauthentication", sRhostsRSAAuthentication, SSHCFG_ALL }, { "hostbasedauthentication", sHostbasedAuthentication, SSHCFG_ALL }, { "hostbasedusesnamefrompacketonly", sHostbasedUsesNameFromPacketOnly, SSHCFG_ALL }, { "hostbasedacceptedkeytypes", sHostbasedAcceptedKeyTypes, SSHCFG_ALL }, { "hostkeyalgorithms", sHostKeyAlgorithms, SSHCFG_GLOBAL }, { "rsaauthentication", sRSAAuthentication, SSHCFG_ALL }, { "pubkeyauthentication", sPubkeyAuthentication, SSHCFG_ALL }, { "pubkeyacceptedkeytypes", sPubkeyAcceptedKeyTypes, SSHCFG_ALL }, { "dsaauthentication", sPubkeyAuthentication, SSHCFG_GLOBAL }, /* alias */ #ifdef KRB5 { "kerberosauthentication", sKerberosAuthentication, SSHCFG_ALL }, { "kerberosorlocalpasswd", sKerberosOrLocalPasswd, SSHCFG_GLOBAL }, { "kerberosticketcleanup", sKerberosTicketCleanup, SSHCFG_GLOBAL }, #ifdef USE_AFS { "kerberosgetafstoken", sKerberosGetAFSToken, SSHCFG_GLOBAL }, #else { "kerberosgetafstoken", sUnsupported, SSHCFG_GLOBAL }, #endif #else { "kerberosauthentication", sUnsupported, SSHCFG_ALL }, { "kerberosorlocalpasswd", sUnsupported, SSHCFG_GLOBAL }, { "kerberosticketcleanup", sUnsupported, SSHCFG_GLOBAL }, { "kerberosgetafstoken", sUnsupported, SSHCFG_GLOBAL }, #endif { "kerberostgtpassing", sUnsupported, SSHCFG_GLOBAL }, { "afstokenpassing", sUnsupported, SSHCFG_GLOBAL }, #ifdef GSSAPI { "gssapiauthentication", sGssAuthentication, SSHCFG_ALL }, { "gssapicleanupcredentials", sGssCleanupCreds, SSHCFG_GLOBAL }, { "gssapistrictacceptorcheck", sGssStrictAcceptor, SSHCFG_GLOBAL }, #else { "gssapiauthentication", sUnsupported, SSHCFG_ALL }, { "gssapicleanupcredentials", sUnsupported, SSHCFG_GLOBAL }, { "gssapistrictacceptorcheck", sUnsupported, SSHCFG_GLOBAL }, #endif { "passwordauthentication", sPasswordAuthentication, SSHCFG_ALL }, { "kbdinteractiveauthentication", sKbdInteractiveAuthentication, SSHCFG_ALL }, { "challengeresponseauthentication", sChallengeResponseAuthentication, SSHCFG_GLOBAL }, { "skeyauthentication", sChallengeResponseAuthentication, SSHCFG_GLOBAL }, /* alias */ { "checkmail", sDeprecated, SSHCFG_GLOBAL }, { "listenaddress", sListenAddress, SSHCFG_GLOBAL }, { "addressfamily", sAddressFamily, SSHCFG_GLOBAL }, { "printmotd", sPrintMotd, SSHCFG_GLOBAL }, #ifdef DISABLE_LASTLOG { "printlastlog", sUnsupported, SSHCFG_GLOBAL }, #else { "printlastlog", sPrintLastLog, SSHCFG_GLOBAL }, #endif { "ignorerhosts", sIgnoreRhosts, SSHCFG_GLOBAL }, { "ignoreuserknownhosts", sIgnoreUserKnownHosts, SSHCFG_GLOBAL }, { "x11forwarding", sX11Forwarding, SSHCFG_ALL }, { "x11displayoffset", sX11DisplayOffset, SSHCFG_ALL }, { "x11uselocalhost", sX11UseLocalhost, SSHCFG_ALL }, { "xauthlocation", sXAuthLocation, SSHCFG_GLOBAL }, { "strictmodes", sStrictModes, SSHCFG_GLOBAL }, { "permitemptypasswords", sEmptyPasswd, SSHCFG_ALL }, { "permituserenvironment", sPermitUserEnvironment, SSHCFG_GLOBAL }, { "uselogin", sUseLogin, SSHCFG_GLOBAL }, { "compression", sCompression, SSHCFG_GLOBAL }, { "rekeylimit", sRekeyLimit, SSHCFG_ALL }, { "tcpkeepalive", sTCPKeepAlive, SSHCFG_GLOBAL }, { "keepalive", sTCPKeepAlive, SSHCFG_GLOBAL }, /* obsolete alias */ { "allowtcpforwarding", sAllowTcpForwarding, SSHCFG_ALL }, { "allowagentforwarding", sAllowAgentForwarding, SSHCFG_ALL }, { "allowusers", sAllowUsers, SSHCFG_ALL }, { "denyusers", sDenyUsers, SSHCFG_ALL }, { "allowgroups", sAllowGroups, SSHCFG_ALL }, { "denygroups", sDenyGroups, SSHCFG_ALL }, { "ciphers", sCiphers, SSHCFG_GLOBAL }, { "macs", sMacs, SSHCFG_GLOBAL }, { "protocol", sProtocol, SSHCFG_GLOBAL }, { "gatewayports", sGatewayPorts, SSHCFG_ALL }, { "subsystem", sSubsystem, SSHCFG_GLOBAL }, { "maxstartups", sMaxStartups, SSHCFG_GLOBAL }, { "maxauthtries", sMaxAuthTries, SSHCFG_ALL }, { "maxsessions", sMaxSessions, SSHCFG_ALL }, { "banner", sBanner, SSHCFG_ALL }, { "usedns", sUseDNS, SSHCFG_GLOBAL }, { "verifyreversemapping", sDeprecated, SSHCFG_GLOBAL }, { "reversemappingcheck", sDeprecated, SSHCFG_GLOBAL }, { "clientaliveinterval", sClientAliveInterval, SSHCFG_GLOBAL }, { "clientalivecountmax", sClientAliveCountMax, SSHCFG_GLOBAL }, { "authorizedkeysfile", sAuthorizedKeysFile, SSHCFG_ALL }, { "authorizedkeysfile2", sDeprecated, SSHCFG_ALL }, { "useprivilegeseparation", sUsePrivilegeSeparation, SSHCFG_GLOBAL}, { "acceptenv", sAcceptEnv, SSHCFG_ALL }, { "permittunnel", sPermitTunnel, SSHCFG_ALL }, { "permittty", sPermitTTY, SSHCFG_ALL }, { "permituserrc", sPermitUserRC, SSHCFG_ALL }, { "match", sMatch, SSHCFG_ALL }, { "permitopen", sPermitOpen, SSHCFG_ALL }, { "forcecommand", sForceCommand, SSHCFG_ALL }, { "chrootdirectory", sChrootDirectory, SSHCFG_ALL }, { "hostcertificate", sHostCertificate, SSHCFG_GLOBAL }, { "revokedkeys", sRevokedKeys, SSHCFG_ALL }, { "trustedusercakeys", sTrustedUserCAKeys, SSHCFG_ALL }, { "authorizedprincipalsfile", sAuthorizedPrincipalsFile, SSHCFG_ALL }, { "kexalgorithms", sKexAlgorithms, SSHCFG_GLOBAL }, { "ipqos", sIPQoS, SSHCFG_ALL }, { "authorizedkeyscommand", sAuthorizedKeysCommand, SSHCFG_ALL }, { "authorizedkeyscommanduser", sAuthorizedKeysCommandUser, SSHCFG_ALL }, { "authorizedprincipalscommand", sAuthorizedPrincipalsCommand, SSHCFG_ALL }, { "authorizedprincipalscommanduser", sAuthorizedPrincipalsCommandUser, SSHCFG_ALL }, { "versionaddendum", sVersionAddendum, SSHCFG_GLOBAL }, { "authenticationmethods", sAuthenticationMethods, SSHCFG_ALL }, { "streamlocalbindmask", sStreamLocalBindMask, SSHCFG_ALL }, { "streamlocalbindunlink", sStreamLocalBindUnlink, SSHCFG_ALL }, { "allowstreamlocalforwarding", sAllowStreamLocalForwarding, SSHCFG_ALL }, { "fingerprinthash", sFingerprintHash, SSHCFG_GLOBAL }, + { "useblacklist", sUseBlacklist, SSHCFG_GLOBAL }, { "noneenabled", sUnsupported, SSHCFG_ALL }, { "hpndisabled", sDeprecated, SSHCFG_ALL }, { "hpnbuffersize", sDeprecated, SSHCFG_ALL }, { "tcprcvbufpoll", sDeprecated, SSHCFG_ALL }, { NULL, sBadOption, 0 } }; static struct { int val; char *text; } tunmode_desc[] = { { SSH_TUNMODE_NO, "no" }, { SSH_TUNMODE_POINTOPOINT, "point-to-point" }, { SSH_TUNMODE_ETHERNET, "ethernet" }, { SSH_TUNMODE_YES, "yes" }, { -1, NULL } }; /* * Returns the number of the token pointed to by cp or sBadOption. */ static ServerOpCodes parse_token(const char *cp, const char *filename, int linenum, u_int *flags) { u_int i; for (i = 0; keywords[i].name; i++) if (strcasecmp(cp, keywords[i].name) == 0) { *flags = keywords[i].flags; return keywords[i].opcode; } error("%s: line %d: Bad configuration option: %s", filename, linenum, cp); return sBadOption; } char * derelativise_path(const char *path) { char *expanded, *ret, cwd[PATH_MAX]; if (strcasecmp(path, "none") == 0) return xstrdup("none"); expanded = tilde_expand_filename(path, getuid()); if (*expanded == '/') return expanded; if (getcwd(cwd, sizeof(cwd)) == NULL) fatal("%s: getcwd: %s", __func__, strerror(errno)); xasprintf(&ret, "%s/%s", cwd, expanded); free(expanded); return ret; } static void add_listen_addr(ServerOptions *options, char *addr, int port) { u_int i; if (port == 0) for (i = 0; i < options->num_ports; i++) add_one_listen_addr(options, addr, options->ports[i]); else add_one_listen_addr(options, addr, port); } static void add_one_listen_addr(ServerOptions *options, char *addr, int port) { struct addrinfo hints, *ai, *aitop; char strport[NI_MAXSERV]; int gaierr; memset(&hints, 0, sizeof(hints)); hints.ai_family = options->address_family; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = (addr == NULL) ? AI_PASSIVE : 0; snprintf(strport, sizeof strport, "%d", port); if ((gaierr = getaddrinfo(addr, strport, &hints, &aitop)) != 0) fatal("bad addr or host: %s (%s)", addr ? addr : "", ssh_gai_strerror(gaierr)); for (ai = aitop; ai->ai_next; ai = ai->ai_next) ; ai->ai_next = options->listen_addrs; options->listen_addrs = aitop; } /* * Queue a ListenAddress to be processed once we have all of the Ports * and AddressFamily options. */ static void queue_listen_addr(ServerOptions *options, char *addr, int port) { options->queued_listen_addrs = xreallocarray( options->queued_listen_addrs, options->num_queued_listens + 1, sizeof(addr)); options->queued_listen_ports = xreallocarray( options->queued_listen_ports, options->num_queued_listens + 1, sizeof(port)); options->queued_listen_addrs[options->num_queued_listens] = xstrdup(addr); options->queued_listen_ports[options->num_queued_listens] = port; options->num_queued_listens++; } /* * Process queued (text) ListenAddress entries. */ static void process_queued_listen_addrs(ServerOptions *options) { u_int i; if (options->num_ports == 0) options->ports[options->num_ports++] = SSH_DEFAULT_PORT; if (options->address_family == -1) options->address_family = AF_UNSPEC; for (i = 0; i < options->num_queued_listens; i++) { add_listen_addr(options, options->queued_listen_addrs[i], options->queued_listen_ports[i]); free(options->queued_listen_addrs[i]); options->queued_listen_addrs[i] = NULL; } free(options->queued_listen_addrs); options->queued_listen_addrs = NULL; free(options->queued_listen_ports); options->queued_listen_ports = NULL; options->num_queued_listens = 0; } struct connection_info * get_connection_info(int populate, int use_dns) { static struct connection_info ci; if (!populate) return &ci; ci.host = get_canonical_hostname(use_dns); ci.address = get_remote_ipaddr(); ci.laddress = get_local_ipaddr(packet_get_connection_in()); ci.lport = get_local_port(); return &ci; } /* * The strategy for the Match blocks is that the config file is parsed twice. * * The first time is at startup. activep is initialized to 1 and the * directives in the global context are processed and acted on. Hitting a * Match directive unsets activep and the directives inside the block are * checked for syntax only. * * The second time is after a connection has been established but before * authentication. activep is initialized to 2 and global config directives * are ignored since they have already been processed. If the criteria in a * Match block is met, activep is set and the subsequent directives * processed and actioned until EOF or another Match block unsets it. Any * options set are copied into the main server config. * * Potential additions/improvements: * - Add Match support for pre-kex directives, eg Protocol, Ciphers. * * - Add a Tag directive (idea from David Leonard) ala pf, eg: * Match Address 192.168.0.* * Tag trusted * Match Group wheel * Tag trusted * Match Tag trusted * AllowTcpForwarding yes * GatewayPorts clientspecified * [...] * * - Add a PermittedChannelRequests directive * Match Group shell * PermittedChannelRequests session,forwarded-tcpip */ static int match_cfg_line_group(const char *grps, int line, const char *user) { int result = 0; struct passwd *pw; if (user == NULL) goto out; if ((pw = getpwnam(user)) == NULL) { debug("Can't match group at line %d because user %.100s does " "not exist", line, user); } else if (ga_init(pw->pw_name, pw->pw_gid) == 0) { debug("Can't Match group because user %.100s not in any group " "at line %d", user, line); } else if (ga_match_pattern_list(grps) != 1) { debug("user %.100s does not match group list %.100s at line %d", user, grps, line); } else { debug("user %.100s matched group list %.100s at line %d", user, grps, line); result = 1; } out: ga_free(); return result; } /* * All of the attributes on a single Match line are ANDed together, so we need * to check every attribute and set the result to zero if any attribute does * not match. */ static int match_cfg_line(char **condition, int line, struct connection_info *ci) { int result = 1, attributes = 0, port; char *arg, *attrib, *cp = *condition; if (ci == NULL) debug3("checking syntax for 'Match %s'", cp); else debug3("checking match for '%s' user %s host %s addr %s " "laddr %s lport %d", cp, ci->user ? ci->user : "(null)", ci->host ? ci->host : "(null)", ci->address ? ci->address : "(null)", ci->laddress ? ci->laddress : "(null)", ci->lport); while ((attrib = strdelim(&cp)) && *attrib != '\0') { attributes++; if (strcasecmp(attrib, "all") == 0) { if (attributes != 1 || ((arg = strdelim(&cp)) != NULL && *arg != '\0')) { error("'all' cannot be combined with other " "Match attributes"); return -1; } *condition = cp; return 1; } if ((arg = strdelim(&cp)) == NULL || *arg == '\0') { error("Missing Match criteria for %s", attrib); return -1; } if (strcasecmp(attrib, "user") == 0) { if (ci == NULL || ci->user == NULL) { result = 0; continue; } if (match_pattern_list(ci->user, arg, 0) != 1) result = 0; else debug("user %.100s matched 'User %.100s' at " "line %d", ci->user, arg, line); } else if (strcasecmp(attrib, "group") == 0) { if (ci == NULL || ci->user == NULL) { result = 0; continue; } switch (match_cfg_line_group(arg, line, ci->user)) { case -1: return -1; case 0: result = 0; } } else if (strcasecmp(attrib, "host") == 0) { if (ci == NULL || ci->host == NULL) { result = 0; continue; } if (match_hostname(ci->host, arg) != 1) result = 0; else debug("connection from %.100s matched 'Host " "%.100s' at line %d", ci->host, arg, line); } else if (strcasecmp(attrib, "address") == 0) { if (ci == NULL || ci->address == NULL) { result = 0; continue; } switch (addr_match_list(ci->address, arg)) { case 1: debug("connection from %.100s matched 'Address " "%.100s' at line %d", ci->address, arg, line); break; case 0: case -1: result = 0; break; case -2: return -1; } } else if (strcasecmp(attrib, "localaddress") == 0){ if (ci == NULL || ci->laddress == NULL) { result = 0; continue; } switch (addr_match_list(ci->laddress, arg)) { case 1: debug("connection from %.100s matched " "'LocalAddress %.100s' at line %d", ci->laddress, arg, line); break; case 0: case -1: result = 0; break; case -2: return -1; } } else if (strcasecmp(attrib, "localport") == 0) { if ((port = a2port(arg)) == -1) { error("Invalid LocalPort '%s' on Match line", arg); return -1; } if (ci == NULL || ci->lport == 0) { result = 0; continue; } /* TODO support port lists */ if (port == ci->lport) debug("connection from %.100s matched " "'LocalPort %d' at line %d", ci->laddress, port, line); else result = 0; } else { error("Unsupported Match attribute %s", attrib); return -1; } } if (attributes == 0) { error("One or more attributes required for Match"); return -1; } if (ci != NULL) debug3("match %sfound", result ? "" : "not "); *condition = cp; return result; } #define WHITESPACE " \t\r\n" /* Multistate option parsing */ struct multistate { char *key; int value; }; static const struct multistate multistate_addressfamily[] = { { "inet", AF_INET }, { "inet6", AF_INET6 }, { "any", AF_UNSPEC }, { NULL, -1 } }; static const struct multistate multistate_permitrootlogin[] = { { "without-password", PERMIT_NO_PASSWD }, { "prohibit-password", PERMIT_NO_PASSWD }, { "forced-commands-only", PERMIT_FORCED_ONLY }, { "yes", PERMIT_YES }, { "no", PERMIT_NO }, { NULL, -1 } }; static const struct multistate multistate_compression[] = { { "delayed", COMP_DELAYED }, { "yes", COMP_ZLIB }, { "no", COMP_NONE }, { NULL, -1 } }; static const struct multistate multistate_gatewayports[] = { { "clientspecified", 2 }, { "yes", 1 }, { "no", 0 }, { NULL, -1 } }; static const struct multistate multistate_privsep[] = { { "yes", PRIVSEP_NOSANDBOX }, { "sandbox", PRIVSEP_ON }, { "nosandbox", PRIVSEP_NOSANDBOX }, { "no", PRIVSEP_OFF }, { NULL, -1 } }; static const struct multistate multistate_tcpfwd[] = { { "yes", FORWARD_ALLOW }, { "all", FORWARD_ALLOW }, { "no", FORWARD_DENY }, { "remote", FORWARD_REMOTE }, { "local", FORWARD_LOCAL }, { NULL, -1 } }; int process_server_config_line(ServerOptions *options, char *line, const char *filename, int linenum, int *activep, struct connection_info *connectinfo) { char *cp, **charptr, *arg, *p; int cmdline = 0, *intptr, value, value2, n, port; SyslogFacility *log_facility_ptr; LogLevel *log_level_ptr; ServerOpCodes opcode; u_int i, flags = 0; size_t len; long long val64; const struct multistate *multistate_ptr; cp = line; if ((arg = strdelim(&cp)) == NULL) return 0; /* Ignore leading whitespace */ if (*arg == '\0') arg = strdelim(&cp); if (!arg || !*arg || *arg == '#') return 0; intptr = NULL; charptr = NULL; opcode = parse_token(arg, filename, linenum, &flags); if (activep == NULL) { /* We are processing a command line directive */ cmdline = 1; activep = &cmdline; } if (*activep && opcode != sMatch) debug3("%s:%d setting %s %s", filename, linenum, arg, cp); if (*activep == 0 && !(flags & SSHCFG_MATCH)) { if (connectinfo == NULL) { fatal("%s line %d: Directive '%s' is not allowed " "within a Match block", filename, linenum, arg); } else { /* this is a directive we have already processed */ while (arg) arg = strdelim(&cp); return 0; } } switch (opcode) { /* Portable-specific options */ case sUsePAM: intptr = &options->use_pam; goto parse_flag; /* Standard Options */ case sBadOption: return -1; case sPort: /* ignore ports from configfile if cmdline specifies ports */ if (options->ports_from_cmdline) return 0; if (options->num_ports >= MAX_PORTS) fatal("%s line %d: too many ports.", filename, linenum); arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing port number.", filename, linenum); options->ports[options->num_ports++] = a2port(arg); if (options->ports[options->num_ports-1] <= 0) fatal("%s line %d: Badly formatted port number.", filename, linenum); break; case sServerKeyBits: intptr = &options->server_key_bits; parse_int: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing integer value.", filename, linenum); value = atoi(arg); if (*activep && *intptr == -1) *intptr = value; break; case sLoginGraceTime: intptr = &options->login_grace_time; parse_time: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing time value.", filename, linenum); if ((value = convtime(arg)) == -1) fatal("%s line %d: invalid time value.", filename, linenum); if (*activep && *intptr == -1) *intptr = value; break; case sKeyRegenerationTime: intptr = &options->key_regeneration_time; goto parse_time; case sListenAddress: arg = strdelim(&cp); if (arg == NULL || *arg == '\0') fatal("%s line %d: missing address", filename, linenum); /* check for bare IPv6 address: no "[]" and 2 or more ":" */ if (strchr(arg, '[') == NULL && (p = strchr(arg, ':')) != NULL && strchr(p+1, ':') != NULL) { queue_listen_addr(options, arg, 0); break; } p = hpdelim(&arg); if (p == NULL) fatal("%s line %d: bad address:port usage", filename, linenum); p = cleanhostname(p); if (arg == NULL) port = 0; else if ((port = a2port(arg)) <= 0) fatal("%s line %d: bad port number", filename, linenum); queue_listen_addr(options, p, port); break; case sAddressFamily: intptr = &options->address_family; multistate_ptr = multistate_addressfamily; parse_multistate: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing argument.", filename, linenum); value = -1; for (i = 0; multistate_ptr[i].key != NULL; i++) { if (strcasecmp(arg, multistate_ptr[i].key) == 0) { value = multistate_ptr[i].value; break; } } if (value == -1) fatal("%s line %d: unsupported option \"%s\".", filename, linenum, arg); if (*activep && *intptr == -1) *intptr = value; break; case sHostKeyFile: intptr = &options->num_host_key_files; if (*intptr >= MAX_HOSTKEYS) fatal("%s line %d: too many host keys specified (max %d).", filename, linenum, MAX_HOSTKEYS); charptr = &options->host_key_files[*intptr]; parse_filename: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing file name.", filename, linenum); if (*activep && *charptr == NULL) { *charptr = derelativise_path(arg); /* increase optional counter */ if (intptr != NULL) *intptr = *intptr + 1; } break; case sHostKeyAgent: charptr = &options->host_key_agent; arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing socket name.", filename, linenum); if (*activep && *charptr == NULL) *charptr = !strcmp(arg, SSH_AUTHSOCKET_ENV_NAME) ? xstrdup(arg) : derelativise_path(arg); break; case sHostCertificate: intptr = &options->num_host_cert_files; if (*intptr >= MAX_HOSTKEYS) fatal("%s line %d: too many host certificates " "specified (max %d).", filename, linenum, MAX_HOSTCERTS); charptr = &options->host_cert_files[*intptr]; goto parse_filename; break; case sPidFile: charptr = &options->pid_file; goto parse_filename; case sPermitRootLogin: intptr = &options->permit_root_login; multistate_ptr = multistate_permitrootlogin; goto parse_multistate; case sIgnoreRhosts: intptr = &options->ignore_rhosts; parse_flag: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing yes/no argument.", filename, linenum); value = 0; /* silence compiler */ if (strcmp(arg, "yes") == 0) value = 1; else if (strcmp(arg, "no") == 0) value = 0; else fatal("%s line %d: Bad yes/no argument: %s", filename, linenum, arg); if (*activep && *intptr == -1) *intptr = value; break; case sIgnoreUserKnownHosts: intptr = &options->ignore_user_known_hosts; goto parse_flag; case sRhostsRSAAuthentication: intptr = &options->rhosts_rsa_authentication; goto parse_flag; case sHostbasedAuthentication: intptr = &options->hostbased_authentication; goto parse_flag; case sHostbasedUsesNameFromPacketOnly: intptr = &options->hostbased_uses_name_from_packet_only; goto parse_flag; case sHostbasedAcceptedKeyTypes: charptr = &options->hostbased_key_types; parse_keytypes: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing argument.", filename, linenum); if (!sshkey_names_valid2(*arg == '+' ? arg + 1 : arg, 1)) fatal("%s line %d: Bad key types '%s'.", filename, linenum, arg ? arg : ""); if (*activep && *charptr == NULL) *charptr = xstrdup(arg); break; case sHostKeyAlgorithms: charptr = &options->hostkeyalgorithms; goto parse_keytypes; case sRSAAuthentication: intptr = &options->rsa_authentication; goto parse_flag; case sPubkeyAuthentication: intptr = &options->pubkey_authentication; goto parse_flag; case sPubkeyAcceptedKeyTypes: charptr = &options->pubkey_key_types; goto parse_keytypes; case sKerberosAuthentication: intptr = &options->kerberos_authentication; goto parse_flag; case sKerberosOrLocalPasswd: intptr = &options->kerberos_or_local_passwd; goto parse_flag; case sKerberosTicketCleanup: intptr = &options->kerberos_ticket_cleanup; goto parse_flag; case sKerberosGetAFSToken: intptr = &options->kerberos_get_afs_token; goto parse_flag; case sGssAuthentication: intptr = &options->gss_authentication; goto parse_flag; case sGssCleanupCreds: intptr = &options->gss_cleanup_creds; goto parse_flag; case sGssStrictAcceptor: intptr = &options->gss_strict_acceptor; goto parse_flag; case sPasswordAuthentication: intptr = &options->password_authentication; goto parse_flag; case sKbdInteractiveAuthentication: intptr = &options->kbd_interactive_authentication; goto parse_flag; case sChallengeResponseAuthentication: intptr = &options->challenge_response_authentication; goto parse_flag; case sPrintMotd: intptr = &options->print_motd; goto parse_flag; case sPrintLastLog: intptr = &options->print_lastlog; goto parse_flag; case sX11Forwarding: intptr = &options->x11_forwarding; goto parse_flag; case sX11DisplayOffset: intptr = &options->x11_display_offset; goto parse_int; case sX11UseLocalhost: intptr = &options->x11_use_localhost; goto parse_flag; case sXAuthLocation: charptr = &options->xauth_location; goto parse_filename; case sPermitTTY: intptr = &options->permit_tty; goto parse_flag; case sPermitUserRC: intptr = &options->permit_user_rc; goto parse_flag; case sStrictModes: intptr = &options->strict_modes; goto parse_flag; case sTCPKeepAlive: intptr = &options->tcp_keep_alive; goto parse_flag; case sEmptyPasswd: intptr = &options->permit_empty_passwd; goto parse_flag; case sPermitUserEnvironment: intptr = &options->permit_user_env; goto parse_flag; case sUseLogin: intptr = &options->use_login; goto parse_flag; case sCompression: intptr = &options->compression; multistate_ptr = multistate_compression; goto parse_multistate; case sRekeyLimit: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%.200s line %d: Missing argument.", filename, linenum); if (strcmp(arg, "default") == 0) { val64 = 0; } else { if (scan_scaled(arg, &val64) == -1) fatal("%.200s line %d: Bad number '%s': %s", filename, linenum, arg, strerror(errno)); if (val64 != 0 && val64 < 16) fatal("%.200s line %d: RekeyLimit too small", filename, linenum); } if (*activep && options->rekey_limit == -1) options->rekey_limit = val64; if (cp != NULL) { /* optional rekey interval present */ if (strcmp(cp, "none") == 0) { (void)strdelim(&cp); /* discard */ break; } intptr = &options->rekey_interval; goto parse_time; } break; case sGatewayPorts: intptr = &options->fwd_opts.gateway_ports; multistate_ptr = multistate_gatewayports; goto parse_multistate; case sUseDNS: intptr = &options->use_dns; goto parse_flag; case sLogFacility: log_facility_ptr = &options->log_facility; arg = strdelim(&cp); value = log_facility_number(arg); if (value == SYSLOG_FACILITY_NOT_SET) fatal("%.200s line %d: unsupported log facility '%s'", filename, linenum, arg ? arg : ""); if (*log_facility_ptr == -1) *log_facility_ptr = (SyslogFacility) value; break; case sLogLevel: log_level_ptr = &options->log_level; arg = strdelim(&cp); value = log_level_number(arg); if (value == SYSLOG_LEVEL_NOT_SET) fatal("%.200s line %d: unsupported log level '%s'", filename, linenum, arg ? arg : ""); if (*log_level_ptr == -1) *log_level_ptr = (LogLevel) value; break; case sAllowTcpForwarding: intptr = &options->allow_tcp_forwarding; multistate_ptr = multistate_tcpfwd; goto parse_multistate; case sAllowStreamLocalForwarding: intptr = &options->allow_streamlocal_forwarding; multistate_ptr = multistate_tcpfwd; goto parse_multistate; case sAllowAgentForwarding: intptr = &options->allow_agent_forwarding; goto parse_flag; case sUsePrivilegeSeparation: intptr = &use_privsep; multistate_ptr = multistate_privsep; goto parse_multistate; case sAllowUsers: while ((arg = strdelim(&cp)) && *arg != '\0') { if (options->num_allow_users >= MAX_ALLOW_USERS) fatal("%s line %d: too many allow users.", filename, linenum); if (!*activep) continue; options->allow_users[options->num_allow_users++] = xstrdup(arg); } break; case sDenyUsers: while ((arg = strdelim(&cp)) && *arg != '\0') { if (options->num_deny_users >= MAX_DENY_USERS) fatal("%s line %d: too many deny users.", filename, linenum); if (!*activep) continue; options->deny_users[options->num_deny_users++] = xstrdup(arg); } break; case sAllowGroups: while ((arg = strdelim(&cp)) && *arg != '\0') { if (options->num_allow_groups >= MAX_ALLOW_GROUPS) fatal("%s line %d: too many allow groups.", filename, linenum); if (!*activep) continue; options->allow_groups[options->num_allow_groups++] = xstrdup(arg); } break; case sDenyGroups: while ((arg = strdelim(&cp)) && *arg != '\0') { if (options->num_deny_groups >= MAX_DENY_GROUPS) fatal("%s line %d: too many deny groups.", filename, linenum); if (!*activep) continue; options->deny_groups[options->num_deny_groups++] = xstrdup(arg); } break; case sCiphers: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing argument.", filename, linenum); if (!ciphers_valid(*arg == '+' ? arg + 1 : arg)) fatal("%s line %d: Bad SSH2 cipher spec '%s'.", filename, linenum, arg ? arg : ""); if (options->ciphers == NULL) options->ciphers = xstrdup(arg); break; case sMacs: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing argument.", filename, linenum); if (!mac_valid(*arg == '+' ? arg + 1 : arg)) fatal("%s line %d: Bad SSH2 mac spec '%s'.", filename, linenum, arg ? arg : ""); if (options->macs == NULL) options->macs = xstrdup(arg); break; case sKexAlgorithms: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing argument.", filename, linenum); if (!kex_names_valid(*arg == '+' ? arg + 1 : arg)) fatal("%s line %d: Bad SSH2 KexAlgorithms '%s'.", filename, linenum, arg ? arg : ""); if (options->kex_algorithms == NULL) options->kex_algorithms = xstrdup(arg); break; case sProtocol: intptr = &options->protocol; arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing argument.", filename, linenum); value = proto_spec(arg); if (value == SSH_PROTO_UNKNOWN) fatal("%s line %d: Bad protocol spec '%s'.", filename, linenum, arg ? arg : ""); if (*intptr == SSH_PROTO_UNKNOWN) *intptr = value; break; case sSubsystem: if (options->num_subsystems >= MAX_SUBSYSTEMS) { fatal("%s line %d: too many subsystems defined.", filename, linenum); } arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing subsystem name.", filename, linenum); if (!*activep) { arg = strdelim(&cp); break; } for (i = 0; i < options->num_subsystems; i++) if (strcmp(arg, options->subsystem_name[i]) == 0) fatal("%s line %d: Subsystem '%s' already defined.", filename, linenum, arg); options->subsystem_name[options->num_subsystems] = xstrdup(arg); arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing subsystem command.", filename, linenum); options->subsystem_command[options->num_subsystems] = xstrdup(arg); /* Collect arguments (separate to executable) */ p = xstrdup(arg); len = strlen(p) + 1; while ((arg = strdelim(&cp)) != NULL && *arg != '\0') { len += 1 + strlen(arg); p = xreallocarray(p, 1, len); strlcat(p, " ", len); strlcat(p, arg, len); } options->subsystem_args[options->num_subsystems] = p; options->num_subsystems++; break; case sMaxStartups: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing MaxStartups spec.", filename, linenum); if ((n = sscanf(arg, "%d:%d:%d", &options->max_startups_begin, &options->max_startups_rate, &options->max_startups)) == 3) { if (options->max_startups_begin > options->max_startups || options->max_startups_rate > 100 || options->max_startups_rate < 1) fatal("%s line %d: Illegal MaxStartups spec.", filename, linenum); } else if (n != 1) fatal("%s line %d: Illegal MaxStartups spec.", filename, linenum); else options->max_startups = options->max_startups_begin; break; case sMaxAuthTries: intptr = &options->max_authtries; goto parse_int; case sMaxSessions: intptr = &options->max_sessions; goto parse_int; case sBanner: charptr = &options->banner; goto parse_filename; /* * These options can contain %X options expanded at * connect time, so that you can specify paths like: * * AuthorizedKeysFile /etc/ssh_keys/%u */ case sAuthorizedKeysFile: if (*activep && options->num_authkeys_files == 0) { while ((arg = strdelim(&cp)) && *arg != '\0') { if (options->num_authkeys_files >= MAX_AUTHKEYS_FILES) fatal("%s line %d: " "too many authorized keys files.", filename, linenum); options->authorized_keys_files[ options->num_authkeys_files++] = tilde_expand_filename(arg, getuid()); } } return 0; case sAuthorizedPrincipalsFile: charptr = &options->authorized_principals_file; arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing file name.", filename, linenum); if (*activep && *charptr == NULL) { *charptr = tilde_expand_filename(arg, getuid()); /* increase optional counter */ if (intptr != NULL) *intptr = *intptr + 1; } break; case sClientAliveInterval: intptr = &options->client_alive_interval; goto parse_time; case sClientAliveCountMax: intptr = &options->client_alive_count_max; goto parse_int; case sAcceptEnv: while ((arg = strdelim(&cp)) && *arg != '\0') { if (strchr(arg, '=') != NULL) fatal("%s line %d: Invalid environment name.", filename, linenum); if (options->num_accept_env >= MAX_ACCEPT_ENV) fatal("%s line %d: too many allow env.", filename, linenum); if (!*activep) continue; options->accept_env[options->num_accept_env++] = xstrdup(arg); } break; case sPermitTunnel: intptr = &options->permit_tun; arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: Missing yes/point-to-point/" "ethernet/no argument.", filename, linenum); value = -1; for (i = 0; tunmode_desc[i].val != -1; i++) if (strcmp(tunmode_desc[i].text, arg) == 0) { value = tunmode_desc[i].val; break; } if (value == -1) fatal("%s line %d: Bad yes/point-to-point/ethernet/" "no argument: %s", filename, linenum, arg); if (*activep && *intptr == -1) *intptr = value; break; case sMatch: if (cmdline) fatal("Match directive not supported as a command-line " "option"); value = match_cfg_line(&cp, linenum, connectinfo); if (value < 0) fatal("%s line %d: Bad Match condition", filename, linenum); *activep = value; break; case sPermitOpen: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing PermitOpen specification", filename, linenum); n = options->num_permitted_opens; /* modified later */ if (strcmp(arg, "any") == 0) { if (*activep && n == -1) { channel_clear_adm_permitted_opens(); options->num_permitted_opens = 0; } break; } if (strcmp(arg, "none") == 0) { if (*activep && n == -1) { options->num_permitted_opens = 1; channel_disable_adm_local_opens(); } break; } if (*activep && n == -1) channel_clear_adm_permitted_opens(); for (; arg != NULL && *arg != '\0'; arg = strdelim(&cp)) { p = hpdelim(&arg); if (p == NULL) fatal("%s line %d: missing host in PermitOpen", filename, linenum); p = cleanhostname(p); if (arg == NULL || ((port = permitopen_port(arg)) < 0)) fatal("%s line %d: bad port number in " "PermitOpen", filename, linenum); if (*activep && n == -1) options->num_permitted_opens = channel_add_adm_permitted_opens(p, port); } break; case sForceCommand: if (cp == NULL || *cp == '\0') fatal("%.200s line %d: Missing argument.", filename, linenum); len = strspn(cp, WHITESPACE); if (*activep && options->adm_forced_command == NULL) options->adm_forced_command = xstrdup(cp + len); return 0; case sChrootDirectory: charptr = &options->chroot_directory; arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing file name.", filename, linenum); if (*activep && *charptr == NULL) *charptr = xstrdup(arg); break; case sTrustedUserCAKeys: charptr = &options->trusted_user_ca_keys; goto parse_filename; case sRevokedKeys: charptr = &options->revoked_keys_file; goto parse_filename; case sIPQoS: arg = strdelim(&cp); if ((value = parse_ipqos(arg)) == -1) fatal("%s line %d: Bad IPQoS value: %s", filename, linenum, arg); arg = strdelim(&cp); if (arg == NULL) value2 = value; else if ((value2 = parse_ipqos(arg)) == -1) fatal("%s line %d: Bad IPQoS value: %s", filename, linenum, arg); if (*activep) { options->ip_qos_interactive = value; options->ip_qos_bulk = value2; } break; case sVersionAddendum: if (cp == NULL || *cp == '\0') fatal("%.200s line %d: Missing argument.", filename, linenum); len = strspn(cp, WHITESPACE); if (*activep && options->version_addendum == NULL) { if (strcasecmp(cp + len, "none") == 0) options->version_addendum = xstrdup(""); else if (strchr(cp + len, '\r') != NULL) fatal("%.200s line %d: Invalid argument", filename, linenum); else options->version_addendum = xstrdup(cp + len); } return 0; case sAuthorizedKeysCommand: if (cp == NULL) fatal("%.200s line %d: Missing argument.", filename, linenum); len = strspn(cp, WHITESPACE); if (*activep && options->authorized_keys_command == NULL) { if (cp[len] != '/' && strcasecmp(cp + len, "none") != 0) fatal("%.200s line %d: AuthorizedKeysCommand " "must be an absolute path", filename, linenum); options->authorized_keys_command = xstrdup(cp + len); } return 0; case sAuthorizedKeysCommandUser: charptr = &options->authorized_keys_command_user; arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing AuthorizedKeysCommandUser " "argument.", filename, linenum); if (*activep && *charptr == NULL) *charptr = xstrdup(arg); break; case sAuthorizedPrincipalsCommand: if (cp == NULL) fatal("%.200s line %d: Missing argument.", filename, linenum); len = strspn(cp, WHITESPACE); if (*activep && options->authorized_principals_command == NULL) { if (cp[len] != '/' && strcasecmp(cp + len, "none") != 0) fatal("%.200s line %d: " "AuthorizedPrincipalsCommand must be " "an absolute path", filename, linenum); options->authorized_principals_command = xstrdup(cp + len); } return 0; case sAuthorizedPrincipalsCommandUser: charptr = &options->authorized_principals_command_user; arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing " "AuthorizedPrincipalsCommandUser argument.", filename, linenum); if (*activep && *charptr == NULL) *charptr = xstrdup(arg); break; case sAuthenticationMethods: if (options->num_auth_methods == 0) { while ((arg = strdelim(&cp)) && *arg != '\0') { if (options->num_auth_methods >= MAX_AUTH_METHODS) fatal("%s line %d: " "too many authentication methods.", filename, linenum); if (auth2_methods_valid(arg, 0) != 0) fatal("%s line %d: invalid " "authentication method list.", filename, linenum); if (!*activep) continue; options->auth_methods[ options->num_auth_methods++] = xstrdup(arg); } } return 0; case sStreamLocalBindMask: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%s line %d: missing StreamLocalBindMask " "argument.", filename, linenum); /* Parse mode in octal format */ value = strtol(arg, &p, 8); if (arg == p || value < 0 || value > 0777) fatal("%s line %d: Bad mask.", filename, linenum); if (*activep) options->fwd_opts.streamlocal_bind_mask = (mode_t)value; break; case sStreamLocalBindUnlink: intptr = &options->fwd_opts.streamlocal_bind_unlink; goto parse_flag; case sFingerprintHash: arg = strdelim(&cp); if (!arg || *arg == '\0') fatal("%.200s line %d: Missing argument.", filename, linenum); if ((value = ssh_digest_alg_by_name(arg)) == -1) fatal("%.200s line %d: Invalid hash algorithm \"%s\".", filename, linenum, arg); if (*activep) options->fingerprint_hash = value; break; + case sUseBlacklist: + intptr = &options->use_blacklist; + goto parse_flag; + case sDeprecated: logit("%s line %d: Deprecated option %s", filename, linenum, arg); while (arg) arg = strdelim(&cp); break; case sUnsupported: logit("%s line %d: Unsupported option %s", filename, linenum, arg); while (arg) arg = strdelim(&cp); break; default: fatal("%s line %d: Missing handler for opcode %s (%d)", filename, linenum, arg, opcode); } if ((arg = strdelim(&cp)) != NULL && *arg != '\0') fatal("%s line %d: garbage at end of line; \"%.200s\".", filename, linenum, arg); return 0; } /* Reads the server configuration file. */ void load_server_config(const char *filename, Buffer *conf) { char line[4096], *cp; FILE *f; int lineno = 0; debug2("%s: filename %s", __func__, filename); if ((f = fopen(filename, "r")) == NULL) { perror(filename); exit(1); } buffer_clear(conf); while (fgets(line, sizeof(line), f)) { lineno++; if (strlen(line) == sizeof(line) - 1) fatal("%s line %d too long", filename, lineno); /* * Trim out comments and strip whitespace * NB - preserve newlines, they are needed to reproduce * line numbers later for error messages */ if ((cp = strchr(line, '#')) != NULL) memcpy(cp, "\n", 2); cp = line + strspn(line, " \t\r"); buffer_append(conf, cp, strlen(cp)); } buffer_append(conf, "\0", 1); fclose(f); debug2("%s: done config len = %d", __func__, buffer_len(conf)); } void parse_server_match_config(ServerOptions *options, struct connection_info *connectinfo) { ServerOptions mo; initialize_server_options(&mo); parse_server_config(&mo, "reprocess config", &cfg, connectinfo); copy_set_server_options(options, &mo, 0); } int parse_server_match_testspec(struct connection_info *ci, char *spec) { char *p; while ((p = strsep(&spec, ",")) && *p != '\0') { if (strncmp(p, "addr=", 5) == 0) { ci->address = xstrdup(p + 5); } else if (strncmp(p, "host=", 5) == 0) { ci->host = xstrdup(p + 5); } else if (strncmp(p, "user=", 5) == 0) { ci->user = xstrdup(p + 5); } else if (strncmp(p, "laddr=", 6) == 0) { ci->laddress = xstrdup(p + 6); } else if (strncmp(p, "lport=", 6) == 0) { ci->lport = a2port(p + 6); if (ci->lport == -1) { fprintf(stderr, "Invalid port '%s' in test mode" " specification %s\n", p+6, p); return -1; } } else { fprintf(stderr, "Invalid test mode specification %s\n", p); return -1; } } return 0; } /* * returns 1 for a complete spec, 0 for partial spec and -1 for an * empty spec. */ int server_match_spec_complete(struct connection_info *ci) { if (ci->user && ci->host && ci->address) return 1; /* complete */ if (!ci->user && !ci->host && !ci->address) return -1; /* empty */ return 0; /* partial */ } /* * Copy any supported values that are set. * * If the preauth flag is set, we do not bother copying the string or * array values that are not used pre-authentication, because any that we * do use must be explictly sent in mm_getpwnamallow(). */ void copy_set_server_options(ServerOptions *dst, ServerOptions *src, int preauth) { #define M_CP_INTOPT(n) do {\ if (src->n != -1) \ dst->n = src->n; \ } while (0) M_CP_INTOPT(password_authentication); M_CP_INTOPT(gss_authentication); M_CP_INTOPT(rsa_authentication); M_CP_INTOPT(pubkey_authentication); M_CP_INTOPT(kerberos_authentication); M_CP_INTOPT(hostbased_authentication); M_CP_INTOPT(hostbased_uses_name_from_packet_only); M_CP_INTOPT(kbd_interactive_authentication); M_CP_INTOPT(permit_root_login); M_CP_INTOPT(permit_empty_passwd); M_CP_INTOPT(allow_tcp_forwarding); M_CP_INTOPT(allow_streamlocal_forwarding); M_CP_INTOPT(allow_agent_forwarding); M_CP_INTOPT(permit_tun); M_CP_INTOPT(fwd_opts.gateway_ports); M_CP_INTOPT(x11_display_offset); M_CP_INTOPT(x11_forwarding); M_CP_INTOPT(x11_use_localhost); M_CP_INTOPT(permit_tty); M_CP_INTOPT(permit_user_rc); M_CP_INTOPT(max_sessions); M_CP_INTOPT(max_authtries); M_CP_INTOPT(ip_qos_interactive); M_CP_INTOPT(ip_qos_bulk); M_CP_INTOPT(rekey_limit); M_CP_INTOPT(rekey_interval); /* M_CP_STROPT and M_CP_STRARRAYOPT should not appear before here */ #define M_CP_STROPT(n) do {\ if (src->n != NULL && dst->n != src->n) { \ free(dst->n); \ dst->n = src->n; \ } \ } while(0) #define M_CP_STRARRAYOPT(n, num_n) do {\ if (src->num_n != 0) { \ for (dst->num_n = 0; dst->num_n < src->num_n; dst->num_n++) \ dst->n[dst->num_n] = xstrdup(src->n[dst->num_n]); \ } \ } while(0) /* See comment in servconf.h */ COPY_MATCH_STRING_OPTS(); /* Arguments that accept '+...' need to be expanded */ assemble_algorithms(dst); /* * The only things that should be below this point are string options * which are only used after authentication. */ if (preauth) return; /* These options may be "none" to clear a global setting */ M_CP_STROPT(adm_forced_command); if (option_clear_or_none(dst->adm_forced_command)) { free(dst->adm_forced_command); dst->adm_forced_command = NULL; } M_CP_STROPT(chroot_directory); if (option_clear_or_none(dst->chroot_directory)) { free(dst->chroot_directory); dst->chroot_directory = NULL; } } #undef M_CP_INTOPT #undef M_CP_STROPT #undef M_CP_STRARRAYOPT void parse_server_config(ServerOptions *options, const char *filename, Buffer *conf, struct connection_info *connectinfo) { int active, linenum, bad_options = 0; char *cp, *obuf, *cbuf; debug2("%s: config %s len %d", __func__, filename, buffer_len(conf)); obuf = cbuf = xstrdup(buffer_ptr(conf)); active = connectinfo ? 0 : 1; linenum = 1; while ((cp = strsep(&cbuf, "\n")) != NULL) { if (process_server_config_line(options, cp, filename, linenum++, &active, connectinfo) != 0) bad_options++; } free(obuf); if (bad_options > 0) fatal("%s: terminating, %d bad configuration options", filename, bad_options); process_queued_listen_addrs(options); } static const char * fmt_multistate_int(int val, const struct multistate *m) { u_int i; for (i = 0; m[i].key != NULL; i++) { if (m[i].value == val) return m[i].key; } return "UNKNOWN"; } static const char * fmt_intarg(ServerOpCodes code, int val) { if (val == -1) return "unset"; switch (code) { case sAddressFamily: return fmt_multistate_int(val, multistate_addressfamily); case sPermitRootLogin: return fmt_multistate_int(val, multistate_permitrootlogin); case sGatewayPorts: return fmt_multistate_int(val, multistate_gatewayports); case sCompression: return fmt_multistate_int(val, multistate_compression); case sUsePrivilegeSeparation: return fmt_multistate_int(val, multistate_privsep); case sAllowTcpForwarding: return fmt_multistate_int(val, multistate_tcpfwd); case sAllowStreamLocalForwarding: return fmt_multistate_int(val, multistate_tcpfwd); case sFingerprintHash: return ssh_digest_alg_name(val); case sProtocol: switch (val) { case SSH_PROTO_1: return "1"; case SSH_PROTO_2: return "2"; case (SSH_PROTO_1|SSH_PROTO_2): return "2,1"; default: return "UNKNOWN"; } default: switch (val) { case 0: return "no"; case 1: return "yes"; default: return "UNKNOWN"; } } } static const char * lookup_opcode_name(ServerOpCodes code) { u_int i; for (i = 0; keywords[i].name != NULL; i++) if (keywords[i].opcode == code) return(keywords[i].name); return "UNKNOWN"; } static void dump_cfg_int(ServerOpCodes code, int val) { printf("%s %d\n", lookup_opcode_name(code), val); } static void dump_cfg_oct(ServerOpCodes code, int val) { printf("%s 0%o\n", lookup_opcode_name(code), val); } static void dump_cfg_fmtint(ServerOpCodes code, int val) { printf("%s %s\n", lookup_opcode_name(code), fmt_intarg(code, val)); } static void dump_cfg_string(ServerOpCodes code, const char *val) { if (val == NULL) return; printf("%s %s\n", lookup_opcode_name(code), val == NULL ? "none" : val); } static void dump_cfg_strarray(ServerOpCodes code, u_int count, char **vals) { u_int i; for (i = 0; i < count; i++) printf("%s %s\n", lookup_opcode_name(code), vals[i]); } static void dump_cfg_strarray_oneline(ServerOpCodes code, u_int count, char **vals) { u_int i; if (count <= 0) return; printf("%s", lookup_opcode_name(code)); for (i = 0; i < count; i++) printf(" %s", vals[i]); printf("\n"); } void dump_config(ServerOptions *o) { u_int i; int ret; struct addrinfo *ai; char addr[NI_MAXHOST], port[NI_MAXSERV], *s = NULL; char *laddr1 = xstrdup(""), *laddr2 = NULL; /* these are usually at the top of the config */ for (i = 0; i < o->num_ports; i++) printf("port %d\n", o->ports[i]); dump_cfg_fmtint(sProtocol, o->protocol); dump_cfg_fmtint(sAddressFamily, o->address_family); /* * ListenAddress must be after Port. add_one_listen_addr pushes * addresses onto a stack, so to maintain ordering we need to * print these in reverse order. */ for (ai = o->listen_addrs; ai; ai = ai->ai_next) { if ((ret = getnameinfo(ai->ai_addr, ai->ai_addrlen, addr, sizeof(addr), port, sizeof(port), NI_NUMERICHOST|NI_NUMERICSERV)) != 0) { error("getnameinfo failed: %.100s", (ret != EAI_SYSTEM) ? gai_strerror(ret) : strerror(errno)); } else { laddr2 = laddr1; if (ai->ai_family == AF_INET6) xasprintf(&laddr1, "listenaddress [%s]:%s\n%s", addr, port, laddr2); else xasprintf(&laddr1, "listenaddress %s:%s\n%s", addr, port, laddr2); free(laddr2); } } printf("%s", laddr1); free(laddr1); /* integer arguments */ #ifdef USE_PAM dump_cfg_fmtint(sUsePAM, o->use_pam); #endif dump_cfg_int(sServerKeyBits, o->server_key_bits); dump_cfg_int(sLoginGraceTime, o->login_grace_time); dump_cfg_int(sKeyRegenerationTime, o->key_regeneration_time); dump_cfg_int(sX11DisplayOffset, o->x11_display_offset); dump_cfg_int(sMaxAuthTries, o->max_authtries); dump_cfg_int(sMaxSessions, o->max_sessions); dump_cfg_int(sClientAliveInterval, o->client_alive_interval); dump_cfg_int(sClientAliveCountMax, o->client_alive_count_max); dump_cfg_oct(sStreamLocalBindMask, o->fwd_opts.streamlocal_bind_mask); /* formatted integer arguments */ dump_cfg_fmtint(sPermitRootLogin, o->permit_root_login); dump_cfg_fmtint(sIgnoreRhosts, o->ignore_rhosts); dump_cfg_fmtint(sIgnoreUserKnownHosts, o->ignore_user_known_hosts); dump_cfg_fmtint(sRhostsRSAAuthentication, o->rhosts_rsa_authentication); dump_cfg_fmtint(sHostbasedAuthentication, o->hostbased_authentication); dump_cfg_fmtint(sHostbasedUsesNameFromPacketOnly, o->hostbased_uses_name_from_packet_only); dump_cfg_fmtint(sRSAAuthentication, o->rsa_authentication); dump_cfg_fmtint(sPubkeyAuthentication, o->pubkey_authentication); #ifdef KRB5 dump_cfg_fmtint(sKerberosAuthentication, o->kerberos_authentication); dump_cfg_fmtint(sKerberosOrLocalPasswd, o->kerberos_or_local_passwd); dump_cfg_fmtint(sKerberosTicketCleanup, o->kerberos_ticket_cleanup); # ifdef USE_AFS dump_cfg_fmtint(sKerberosGetAFSToken, o->kerberos_get_afs_token); # endif #endif #ifdef GSSAPI dump_cfg_fmtint(sGssAuthentication, o->gss_authentication); dump_cfg_fmtint(sGssCleanupCreds, o->gss_cleanup_creds); #endif dump_cfg_fmtint(sPasswordAuthentication, o->password_authentication); dump_cfg_fmtint(sKbdInteractiveAuthentication, o->kbd_interactive_authentication); dump_cfg_fmtint(sChallengeResponseAuthentication, o->challenge_response_authentication); dump_cfg_fmtint(sPrintMotd, o->print_motd); #ifndef DISABLE_LASTLOG dump_cfg_fmtint(sPrintLastLog, o->print_lastlog); #endif dump_cfg_fmtint(sX11Forwarding, o->x11_forwarding); dump_cfg_fmtint(sX11UseLocalhost, o->x11_use_localhost); dump_cfg_fmtint(sPermitTTY, o->permit_tty); dump_cfg_fmtint(sPermitUserRC, o->permit_user_rc); dump_cfg_fmtint(sStrictModes, o->strict_modes); dump_cfg_fmtint(sTCPKeepAlive, o->tcp_keep_alive); dump_cfg_fmtint(sEmptyPasswd, o->permit_empty_passwd); dump_cfg_fmtint(sPermitUserEnvironment, o->permit_user_env); dump_cfg_fmtint(sUseLogin, o->use_login); dump_cfg_fmtint(sCompression, o->compression); dump_cfg_fmtint(sGatewayPorts, o->fwd_opts.gateway_ports); dump_cfg_fmtint(sUseDNS, o->use_dns); dump_cfg_fmtint(sAllowTcpForwarding, o->allow_tcp_forwarding); dump_cfg_fmtint(sAllowAgentForwarding, o->allow_agent_forwarding); dump_cfg_fmtint(sAllowStreamLocalForwarding, o->allow_streamlocal_forwarding); dump_cfg_fmtint(sUsePrivilegeSeparation, use_privsep); dump_cfg_fmtint(sFingerprintHash, o->fingerprint_hash); + dump_cfg_fmtint(sUseBlacklist, o->use_blacklist); /* string arguments */ dump_cfg_string(sPidFile, o->pid_file); dump_cfg_string(sXAuthLocation, o->xauth_location); dump_cfg_string(sCiphers, o->ciphers ? o->ciphers : KEX_SERVER_ENCRYPT); dump_cfg_string(sMacs, o->macs ? o->macs : KEX_SERVER_MAC); dump_cfg_string(sBanner, o->banner); dump_cfg_string(sForceCommand, o->adm_forced_command); dump_cfg_string(sChrootDirectory, o->chroot_directory); dump_cfg_string(sTrustedUserCAKeys, o->trusted_user_ca_keys); dump_cfg_string(sRevokedKeys, o->revoked_keys_file); dump_cfg_string(sAuthorizedPrincipalsFile, o->authorized_principals_file); dump_cfg_string(sVersionAddendum, *o->version_addendum == '\0' ? "none" : o->version_addendum); dump_cfg_string(sAuthorizedKeysCommand, o->authorized_keys_command); dump_cfg_string(sAuthorizedKeysCommandUser, o->authorized_keys_command_user); dump_cfg_string(sAuthorizedPrincipalsCommand, o->authorized_principals_command); dump_cfg_string(sAuthorizedPrincipalsCommandUser, o->authorized_principals_command_user); dump_cfg_string(sHostKeyAgent, o->host_key_agent); dump_cfg_string(sKexAlgorithms, o->kex_algorithms ? o->kex_algorithms : KEX_SERVER_KEX); dump_cfg_string(sHostbasedAcceptedKeyTypes, o->hostbased_key_types ? o->hostbased_key_types : KEX_DEFAULT_PK_ALG); dump_cfg_string(sHostKeyAlgorithms, o->hostkeyalgorithms ? o->hostkeyalgorithms : KEX_DEFAULT_PK_ALG); dump_cfg_string(sPubkeyAcceptedKeyTypes, o->pubkey_key_types ? o->pubkey_key_types : KEX_DEFAULT_PK_ALG); /* string arguments requiring a lookup */ dump_cfg_string(sLogLevel, log_level_name(o->log_level)); dump_cfg_string(sLogFacility, log_facility_name(o->log_facility)); /* string array arguments */ dump_cfg_strarray_oneline(sAuthorizedKeysFile, o->num_authkeys_files, o->authorized_keys_files); dump_cfg_strarray(sHostKeyFile, o->num_host_key_files, o->host_key_files); dump_cfg_strarray(sHostCertificate, o->num_host_cert_files, o->host_cert_files); dump_cfg_strarray(sAllowUsers, o->num_allow_users, o->allow_users); dump_cfg_strarray(sDenyUsers, o->num_deny_users, o->deny_users); dump_cfg_strarray(sAllowGroups, o->num_allow_groups, o->allow_groups); dump_cfg_strarray(sDenyGroups, o->num_deny_groups, o->deny_groups); dump_cfg_strarray(sAcceptEnv, o->num_accept_env, o->accept_env); dump_cfg_strarray_oneline(sAuthenticationMethods, o->num_auth_methods, o->auth_methods); /* other arguments */ for (i = 0; i < o->num_subsystems; i++) printf("subsystem %s %s\n", o->subsystem_name[i], o->subsystem_args[i]); printf("maxstartups %d:%d:%d\n", o->max_startups_begin, o->max_startups_rate, o->max_startups); for (i = 0; tunmode_desc[i].val != -1; i++) if (tunmode_desc[i].val == o->permit_tun) { s = tunmode_desc[i].text; break; } dump_cfg_string(sPermitTunnel, s); printf("ipqos %s ", iptos2str(o->ip_qos_interactive)); printf("%s\n", iptos2str(o->ip_qos_bulk)); printf("rekeylimit %llu %d\n", (unsigned long long)o->rekey_limit, o->rekey_interval); channel_print_adm_permitted_opens(); } Index: projects/clang390-import/crypto/openssh/servconf.h =================================================================== --- projects/clang390-import/crypto/openssh/servconf.h (revision 305080) +++ projects/clang390-import/crypto/openssh/servconf.h (revision 305081) @@ -1,254 +1,255 @@ /* $OpenBSD: servconf.h,v 1.120 2015/07/10 06:21:53 markus Exp $ */ /* * Author: Tatu Ylonen * Copyright (c) 1995 Tatu Ylonen , Espoo, Finland * All rights reserved * Definitions for server configuration data and for the functions reading it. * * As far as I am concerned, the code I have written for this software * can be used freely for any purpose. Any derived versions of this * software must be clearly marked as such, and if the derived work is * incompatible with the protocol description in the RFC file, it must be * called by a name other than "ssh" or "Secure Shell". */ #ifndef SERVCONF_H #define SERVCONF_H #define MAX_PORTS 256 /* Max # ports. */ #define MAX_ALLOW_USERS 256 /* Max # users on allow list. */ #define MAX_DENY_USERS 256 /* Max # users on deny list. */ #define MAX_ALLOW_GROUPS 256 /* Max # groups on allow list. */ #define MAX_DENY_GROUPS 256 /* Max # groups on deny list. */ #define MAX_SUBSYSTEMS 256 /* Max # subsystems. */ #define MAX_HOSTKEYS 256 /* Max # hostkeys. */ #define MAX_HOSTCERTS 256 /* Max # host certificates. */ #define MAX_ACCEPT_ENV 256 /* Max # of env vars. */ #define MAX_MATCH_GROUPS 256 /* Max # of groups for Match. */ #define MAX_AUTHKEYS_FILES 256 /* Max # of authorized_keys files. */ #define MAX_AUTH_METHODS 256 /* Max # of AuthenticationMethods. */ /* permit_root_login */ #define PERMIT_NOT_SET -1 #define PERMIT_NO 0 #define PERMIT_FORCED_ONLY 1 #define PERMIT_NO_PASSWD 2 #define PERMIT_YES 3 /* use_privsep */ #define PRIVSEP_OFF 0 #define PRIVSEP_ON 1 #define PRIVSEP_NOSANDBOX 2 /* AllowTCPForwarding */ #define FORWARD_DENY 0 #define FORWARD_REMOTE (1) #define FORWARD_LOCAL (1<<1) #define FORWARD_ALLOW (FORWARD_REMOTE|FORWARD_LOCAL) #define DEFAULT_AUTH_FAIL_MAX 6 /* Default for MaxAuthTries */ #define DEFAULT_SESSIONS_MAX 10 /* Default for MaxSessions */ /* Magic name for internal sftp-server */ #define INTERNAL_SFTP_NAME "internal-sftp" typedef struct { u_int num_ports; u_int ports_from_cmdline; int ports[MAX_PORTS]; /* Port number to listen on. */ u_int num_queued_listens; char **queued_listen_addrs; int *queued_listen_ports; struct addrinfo *listen_addrs; /* Addresses on which the server listens. */ int address_family; /* Address family used by the server. */ char *host_key_files[MAX_HOSTKEYS]; /* Files containing host keys. */ int num_host_key_files; /* Number of files for host keys. */ char *host_cert_files[MAX_HOSTCERTS]; /* Files containing host certs. */ int num_host_cert_files; /* Number of files for host certs. */ char *host_key_agent; /* ssh-agent socket for host keys. */ char *pid_file; /* Where to put our pid */ int server_key_bits;/* Size of the server key. */ int login_grace_time; /* Disconnect if no auth in this time * (sec). */ int key_regeneration_time; /* Server key lifetime (seconds). */ int permit_root_login; /* PERMIT_*, see above */ int ignore_rhosts; /* Ignore .rhosts and .shosts. */ int ignore_user_known_hosts; /* Ignore ~/.ssh/known_hosts * for RhostsRsaAuth */ int print_motd; /* If true, print /etc/motd. */ int print_lastlog; /* If true, print lastlog */ int x11_forwarding; /* If true, permit inet (spoofing) X11 fwd. */ int x11_display_offset; /* What DISPLAY number to start * searching at */ int x11_use_localhost; /* If true, use localhost for fake X11 server. */ char *xauth_location; /* Location of xauth program */ int permit_tty; /* If false, deny pty allocation */ int permit_user_rc; /* If false, deny ~/.ssh/rc execution */ int strict_modes; /* If true, require string home dir modes. */ int tcp_keep_alive; /* If true, set SO_KEEPALIVE. */ int ip_qos_interactive; /* IP ToS/DSCP/class for interactive */ int ip_qos_bulk; /* IP ToS/DSCP/class for bulk traffic */ char *ciphers; /* Supported SSH2 ciphers. */ char *macs; /* Supported SSH2 macs. */ char *kex_algorithms; /* SSH2 kex methods in order of preference. */ int protocol; /* Supported protocol versions. */ struct ForwardOptions fwd_opts; /* forwarding options */ SyslogFacility log_facility; /* Facility for system logging. */ LogLevel log_level; /* Level for system logging. */ int rhosts_rsa_authentication; /* If true, permit rhosts RSA * authentication. */ int hostbased_authentication; /* If true, permit ssh2 hostbased auth */ int hostbased_uses_name_from_packet_only; /* experimental */ char *hostbased_key_types; /* Key types allowed for hostbased */ char *hostkeyalgorithms; /* SSH2 server key types */ int rsa_authentication; /* If true, permit RSA authentication. */ int pubkey_authentication; /* If true, permit ssh2 pubkey authentication. */ char *pubkey_key_types; /* Key types allowed for public key */ int kerberos_authentication; /* If true, permit Kerberos * authentication. */ int kerberos_or_local_passwd; /* If true, permit kerberos * and any other password * authentication mechanism, * such as SecurID or * /etc/passwd */ int kerberos_ticket_cleanup; /* If true, destroy ticket * file on logout. */ int kerberos_get_afs_token; /* If true, try to get AFS token if * authenticated with Kerberos. */ int gss_authentication; /* If true, permit GSSAPI authentication */ int gss_cleanup_creds; /* If true, destroy cred cache on logout */ int gss_strict_acceptor; /* If true, restrict the GSSAPI acceptor name */ int password_authentication; /* If true, permit password * authentication. */ int kbd_interactive_authentication; /* If true, permit */ int challenge_response_authentication; int permit_empty_passwd; /* If false, do not permit empty * passwords. */ int permit_user_env; /* If true, read ~/.ssh/environment */ int use_login; /* If true, login(1) is used */ int compression; /* If true, compression is allowed */ int allow_tcp_forwarding; /* One of FORWARD_* */ int allow_streamlocal_forwarding; /* One of FORWARD_* */ int allow_agent_forwarding; u_int num_allow_users; char *allow_users[MAX_ALLOW_USERS]; u_int num_deny_users; char *deny_users[MAX_DENY_USERS]; u_int num_allow_groups; char *allow_groups[MAX_ALLOW_GROUPS]; u_int num_deny_groups; char *deny_groups[MAX_DENY_GROUPS]; u_int num_subsystems; char *subsystem_name[MAX_SUBSYSTEMS]; char *subsystem_command[MAX_SUBSYSTEMS]; char *subsystem_args[MAX_SUBSYSTEMS]; u_int num_accept_env; char *accept_env[MAX_ACCEPT_ENV]; int max_startups_begin; int max_startups_rate; int max_startups; int max_authtries; int max_sessions; char *banner; /* SSH-2 banner message */ int use_dns; int client_alive_interval; /* * poke the client this often to * see if it's still there */ int client_alive_count_max; /* * If the client is unresponsive * for this many intervals above, * disconnect the session */ u_int num_authkeys_files; /* Files containing public keys */ char *authorized_keys_files[MAX_AUTHKEYS_FILES]; char *adm_forced_command; int use_pam; /* Enable auth via PAM */ int permit_tun; int num_permitted_opens; char *chroot_directory; char *revoked_keys_file; char *trusted_user_ca_keys; char *authorized_keys_command; char *authorized_keys_command_user; char *authorized_principals_file; char *authorized_principals_command; char *authorized_principals_command_user; int64_t rekey_limit; int rekey_interval; char *version_addendum; /* Appended to SSH banner */ u_int num_auth_methods; char *auth_methods[MAX_AUTH_METHODS]; int fingerprint_hash; + int use_blacklist; } ServerOptions; /* Information about the incoming connection as used by Match */ struct connection_info { const char *user; const char *host; /* possibly resolved hostname */ const char *address; /* remote address */ const char *laddress; /* local address */ int lport; /* local port */ }; /* * These are string config options that must be copied between the * Match sub-config and the main config, and must be sent from the * privsep slave to the privsep master. We use a macro to ensure all * the options are copied and the copies are done in the correct order. * * NB. an option must appear in servconf.c:copy_set_server_options() or * COPY_MATCH_STRING_OPTS here but never both. */ #define COPY_MATCH_STRING_OPTS() do { \ M_CP_STROPT(banner); \ M_CP_STROPT(trusted_user_ca_keys); \ M_CP_STROPT(revoked_keys_file); \ M_CP_STROPT(authorized_keys_command); \ M_CP_STROPT(authorized_keys_command_user); \ M_CP_STROPT(authorized_principals_file); \ M_CP_STROPT(authorized_principals_command); \ M_CP_STROPT(authorized_principals_command_user); \ M_CP_STROPT(hostbased_key_types); \ M_CP_STROPT(pubkey_key_types); \ M_CP_STRARRAYOPT(authorized_keys_files, num_authkeys_files); \ M_CP_STRARRAYOPT(allow_users, num_allow_users); \ M_CP_STRARRAYOPT(deny_users, num_deny_users); \ M_CP_STRARRAYOPT(allow_groups, num_allow_groups); \ M_CP_STRARRAYOPT(deny_groups, num_deny_groups); \ M_CP_STRARRAYOPT(accept_env, num_accept_env); \ M_CP_STRARRAYOPT(auth_methods, num_auth_methods); \ } while (0) struct connection_info *get_connection_info(int, int); void initialize_server_options(ServerOptions *); void fill_default_server_options(ServerOptions *); int process_server_config_line(ServerOptions *, char *, const char *, int, int *, struct connection_info *); void load_server_config(const char *, Buffer *); void parse_server_config(ServerOptions *, const char *, Buffer *, struct connection_info *); void parse_server_match_config(ServerOptions *, struct connection_info *); int parse_server_match_testspec(struct connection_info *, char *); int server_match_spec_complete(struct connection_info *); void copy_set_server_options(ServerOptions *, ServerOptions *, int); void dump_config(ServerOptions *); char *derelativise_path(const char *); #endif /* SERVCONF_H */ Index: projects/clang390-import/crypto/openssh/sshd.c =================================================================== --- projects/clang390-import/crypto/openssh/sshd.c (revision 305080) +++ projects/clang390-import/crypto/openssh/sshd.c (revision 305081) @@ -1,2700 +1,2705 @@ /* $OpenBSD: sshd.c,v 1.465 2016/02/15 09:47:49 dtucker Exp $ */ /* * Author: Tatu Ylonen * Copyright (c) 1995 Tatu Ylonen , Espoo, Finland * All rights reserved * This program is the ssh daemon. It listens for connections from clients, * and performs authentication, executes use commands or shell, and forwards * information to/from the application to the user client over an encrypted * connection. This can also handle forwarding of X11, TCP/IP, and * authentication agent connections. * * As far as I am concerned, the code I have written for this software * can be used freely for any purpose. Any derived versions of this * software must be clearly marked as such, and if the derived work is * incompatible with the protocol description in the RFC file, it must be * called by a name other than "ssh" or "Secure Shell". * * SSH2 implementation: * Privilege Separation: * * Copyright (c) 2000, 2001, 2002 Markus Friedl. All rights reserved. * Copyright (c) 2002 Niels Provos. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "includes.h" __RCSID("$FreeBSD$"); #include #include #include #include #ifdef HAVE_SYS_STAT_H # include #endif #ifdef HAVE_SYS_TIME_H # include #endif #include "openbsd-compat/sys-tree.h" #include "openbsd-compat/sys-queue.h" #include #include #include #include #ifdef HAVE_PATHS_H #include #endif #include #include #include #include #include #include #include #include #include #ifdef WITH_OPENSSL #include #include #include #include "openbsd-compat/openssl-compat.h" #endif #ifdef HAVE_SECUREWARE #include #include #endif #ifdef __FreeBSD__ #include #if defined(GSSAPI) && defined(HAVE_GSSAPI_GSSAPI_H) #include #elif defined(GSSAPI) && defined(HAVE_GSSAPI_H) #include #endif #endif #include "xmalloc.h" #include "ssh.h" #include "ssh1.h" #include "ssh2.h" #include "rsa.h" #include "sshpty.h" #include "packet.h" #include "log.h" #include "buffer.h" #include "misc.h" #include "match.h" #include "servconf.h" #include "uidswap.h" #include "compat.h" #include "cipher.h" #include "digest.h" #include "key.h" #include "kex.h" #include "myproposal.h" #include "authfile.h" #include "pathnames.h" #include "atomicio.h" #include "canohost.h" #include "hostfile.h" #include "auth.h" #include "authfd.h" #include "msg.h" #include "dispatch.h" #include "channels.h" #include "session.h" #include "monitor_mm.h" #include "monitor.h" #ifdef GSSAPI #include "ssh-gss.h" #endif #include "monitor_wrap.h" #include "ssh-sandbox.h" #include "version.h" #include "ssherr.h" +#include "blacklist_client.h" #ifdef LIBWRAP #include #include int allow_severity; int deny_severity; #endif /* LIBWRAP */ #ifndef O_NOCTTY #define O_NOCTTY 0 #endif /* Re-exec fds */ #define REEXEC_DEVCRYPTO_RESERVED_FD (STDERR_FILENO + 1) #define REEXEC_STARTUP_PIPE_FD (STDERR_FILENO + 2) #define REEXEC_CONFIG_PASS_FD (STDERR_FILENO + 3) #define REEXEC_MIN_FREE_FD (STDERR_FILENO + 4) extern char *__progname; /* Server configuration options. */ ServerOptions options; /* Name of the server configuration file. */ char *config_file_name = _PATH_SERVER_CONFIG_FILE; /* * Debug mode flag. This can be set on the command line. If debug * mode is enabled, extra debugging output will be sent to the system * log, the daemon will not go to background, and will exit after processing * the first connection. */ int debug_flag = 0; /* Flag indicating that the daemon should only test the configuration and keys. */ int test_flag = 0; /* Flag indicating that the daemon is being started from inetd. */ int inetd_flag = 0; /* Flag indicating that sshd should not detach and become a daemon. */ int no_daemon_flag = 0; /* debug goes to stderr unless inetd_flag is set */ int log_stderr = 0; /* Saved arguments to main(). */ char **saved_argv; int saved_argc; /* re-exec */ int rexeced_flag = 0; int rexec_flag = 1; int rexec_argc = 0; char **rexec_argv; /* * The sockets that the server is listening; this is used in the SIGHUP * signal handler. */ #define MAX_LISTEN_SOCKS 16 int listen_socks[MAX_LISTEN_SOCKS]; int num_listen_socks = 0; /* * the client's version string, passed by sshd2 in compat mode. if != NULL, * sshd will skip the version-number exchange */ char *client_version_string = NULL; char *server_version_string = NULL; /* Daemon's agent connection */ int auth_sock = -1; int have_agent = 0; /* * Any really sensitive data in the application is contained in this * structure. The idea is that this structure could be locked into memory so * that the pages do not get written into swap. However, there are some * problems. The private key contains BIGNUMs, and we do not (in principle) * have access to the internals of them, and locking just the structure is * not very useful. Currently, memory locking is not implemented. */ struct { Key *server_key; /* ephemeral server key */ Key *ssh1_host_key; /* ssh1 host key */ Key **host_keys; /* all private host keys */ Key **host_pubkeys; /* all public host keys */ Key **host_certificates; /* all public host certificates */ int have_ssh1_key; int have_ssh2_key; u_char ssh1_cookie[SSH_SESSION_KEY_LENGTH]; } sensitive_data; /* * Flag indicating whether the RSA server key needs to be regenerated. * Is set in the SIGALRM handler and cleared when the key is regenerated. */ static volatile sig_atomic_t key_do_regen = 0; /* This is set to true when a signal is received. */ static volatile sig_atomic_t received_sighup = 0; static volatile sig_atomic_t received_sigterm = 0; /* session identifier, used by RSA-auth */ u_char session_id[16]; /* same for ssh2 */ u_char *session_id2 = NULL; u_int session_id2_len = 0; /* record remote hostname or ip */ u_int utmp_len = HOST_NAME_MAX+1; /* options.max_startup sized array of fd ints */ int *startup_pipes = NULL; int startup_pipe; /* in child */ /* variables used for privilege separation */ int use_privsep = -1; struct monitor *pmonitor = NULL; int privsep_is_preauth = 1; /* global authentication context */ Authctxt *the_authctxt = NULL; /* sshd_config buffer */ Buffer cfg; /* message to be displayed after login */ Buffer loginmsg; /* Unprivileged user */ struct passwd *privsep_pw = NULL; /* Prototypes for various functions defined later in this file. */ void destroy_sensitive_data(void); void demote_sensitive_data(void); #ifdef WITH_SSH1 static void do_ssh1_kex(void); #endif static void do_ssh2_kex(void); /* * Close all listening sockets */ static void close_listen_socks(void) { int i; for (i = 0; i < num_listen_socks; i++) close(listen_socks[i]); num_listen_socks = -1; } static void close_startup_pipes(void) { int i; if (startup_pipes) for (i = 0; i < options.max_startups; i++) if (startup_pipes[i] != -1) close(startup_pipes[i]); } /* * Signal handler for SIGHUP. Sshd execs itself when it receives SIGHUP; * the effect is to reread the configuration file (and to regenerate * the server key). */ /*ARGSUSED*/ static void sighup_handler(int sig) { int save_errno = errno; received_sighup = 1; signal(SIGHUP, sighup_handler); errno = save_errno; } /* * Called from the main program after receiving SIGHUP. * Restarts the server. */ static void sighup_restart(void) { logit("Received SIGHUP; restarting."); platform_pre_restart(); close_listen_socks(); close_startup_pipes(); alarm(0); /* alarm timer persists across exec */ signal(SIGHUP, SIG_IGN); /* will be restored after exec */ execv(saved_argv[0], saved_argv); logit("RESTART FAILED: av[0]='%.100s', error: %.100s.", saved_argv[0], strerror(errno)); exit(1); } /* * Generic signal handler for terminating signals in the master daemon. */ /*ARGSUSED*/ static void sigterm_handler(int sig) { received_sigterm = sig; } /* * SIGCHLD handler. This is called whenever a child dies. This will then * reap any zombies left by exited children. */ /*ARGSUSED*/ static void main_sigchld_handler(int sig) { int save_errno = errno; pid_t pid; int status; while ((pid = waitpid(-1, &status, WNOHANG)) > 0 || (pid < 0 && errno == EINTR)) ; signal(SIGCHLD, main_sigchld_handler); errno = save_errno; } /* * Signal handler for the alarm after the login grace period has expired. */ /*ARGSUSED*/ static void grace_alarm_handler(int sig) { if (use_privsep && pmonitor != NULL && pmonitor->m_pid > 0) kill(pmonitor->m_pid, SIGALRM); /* * Try to kill any processes that we have spawned, E.g. authorized * keys command helpers. */ if (getpgid(0) == getpid()) { signal(SIGTERM, SIG_IGN); kill(0, SIGTERM); } + BLACKLIST_NOTIFY(BLACKLIST_AUTH_FAIL); + /* Log error and exit. */ sigdie("Timeout before authentication for %s", get_remote_ipaddr()); } /* * Signal handler for the key regeneration alarm. Note that this * alarm only occurs in the daemon waiting for connections, and it does not * do anything with the private key or random state before forking. * Thus there should be no concurrency control/asynchronous execution * problems. */ static void generate_ephemeral_server_key(void) { verbose("Generating %s%d bit RSA key.", sensitive_data.server_key ? "new " : "", options.server_key_bits); if (sensitive_data.server_key != NULL) key_free(sensitive_data.server_key); sensitive_data.server_key = key_generate(KEY_RSA1, options.server_key_bits); verbose("RSA key generation complete."); arc4random_buf(sensitive_data.ssh1_cookie, SSH_SESSION_KEY_LENGTH); } /*ARGSUSED*/ static void key_regeneration_alarm(int sig) { int save_errno = errno; signal(SIGALRM, SIG_DFL); errno = save_errno; key_do_regen = 1; } static void sshd_exchange_identification(int sock_in, int sock_out) { u_int i; int mismatch; int remote_major, remote_minor; int major, minor; char *s, *newline = "\n"; char buf[256]; /* Must not be larger than remote_version. */ char remote_version[256]; /* Must be at least as big as buf. */ if ((options.protocol & SSH_PROTO_1) && (options.protocol & SSH_PROTO_2)) { major = PROTOCOL_MAJOR_1; minor = 99; } else if (options.protocol & SSH_PROTO_2) { major = PROTOCOL_MAJOR_2; minor = PROTOCOL_MINOR_2; newline = "\r\n"; } else { major = PROTOCOL_MAJOR_1; minor = PROTOCOL_MINOR_1; } xasprintf(&server_version_string, "SSH-%d.%d-%.100s%s%s%s", major, minor, SSH_VERSION, *options.version_addendum == '\0' ? "" : " ", options.version_addendum, newline); /* Send our protocol version identification. */ if (atomicio(vwrite, sock_out, server_version_string, strlen(server_version_string)) != strlen(server_version_string)) { logit("Could not write ident string to %s", get_remote_ipaddr()); cleanup_exit(255); } /* Read other sides version identification. */ memset(buf, 0, sizeof(buf)); for (i = 0; i < sizeof(buf) - 1; i++) { if (atomicio(read, sock_in, &buf[i], 1) != 1) { logit("Did not receive identification string from %s", get_remote_ipaddr()); cleanup_exit(255); } if (buf[i] == '\r') { buf[i] = 0; /* Kludge for F-Secure Macintosh < 1.0.2 */ if (i == 12 && strncmp(buf, "SSH-1.5-W1.0", 12) == 0) break; continue; } if (buf[i] == '\n') { buf[i] = 0; break; } } buf[sizeof(buf) - 1] = 0; client_version_string = xstrdup(buf); /* * Check that the versions match. In future this might accept * several versions and set appropriate flags to handle them. */ if (sscanf(client_version_string, "SSH-%d.%d-%[^\n]\n", &remote_major, &remote_minor, remote_version) != 3) { s = "Protocol mismatch.\n"; (void) atomicio(vwrite, sock_out, s, strlen(s)); logit("Bad protocol version identification '%.100s' " "from %s port %d", client_version_string, get_remote_ipaddr(), get_remote_port()); close(sock_in); close(sock_out); cleanup_exit(255); } debug("Client protocol version %d.%d; client software version %.100s", remote_major, remote_minor, remote_version); active_state->compat = compat_datafellows(remote_version); if ((datafellows & SSH_BUG_PROBE) != 0) { logit("probed from %s with %s. Don't panic.", get_remote_ipaddr(), client_version_string); cleanup_exit(255); } if ((datafellows & SSH_BUG_SCANNER) != 0) { logit("scanned from %s with %s. Don't panic.", get_remote_ipaddr(), client_version_string); cleanup_exit(255); } if ((datafellows & SSH_BUG_RSASIGMD5) != 0) { logit("Client version \"%.100s\" uses unsafe RSA signature " "scheme; disabling use of RSA keys", remote_version); } if ((datafellows & SSH_BUG_DERIVEKEY) != 0) { fatal("Client version \"%.100s\" uses unsafe key agreement; " "refusing connection", remote_version); } mismatch = 0; switch (remote_major) { case 1: if (remote_minor == 99) { if (options.protocol & SSH_PROTO_2) enable_compat20(); else mismatch = 1; break; } if (!(options.protocol & SSH_PROTO_1)) { mismatch = 1; break; } if (remote_minor < 3) { packet_disconnect("Your ssh version is too old and " "is no longer supported. Please install a newer version."); } else if (remote_minor == 3) { /* note that this disables agent-forwarding */ enable_compat13(); } break; case 2: if (options.protocol & SSH_PROTO_2) { enable_compat20(); break; } /* FALLTHROUGH */ default: mismatch = 1; break; } chop(server_version_string); debug("Local version string %.200s", server_version_string); if (mismatch) { s = "Protocol major versions differ.\n"; (void) atomicio(vwrite, sock_out, s, strlen(s)); close(sock_in); close(sock_out); logit("Protocol major versions differ for %s: %.200s vs. %.200s", get_remote_ipaddr(), server_version_string, client_version_string); cleanup_exit(255); } } /* Destroy the host and server keys. They will no longer be needed. */ void destroy_sensitive_data(void) { int i; if (sensitive_data.server_key) { key_free(sensitive_data.server_key); sensitive_data.server_key = NULL; } for (i = 0; i < options.num_host_key_files; i++) { if (sensitive_data.host_keys[i]) { key_free(sensitive_data.host_keys[i]); sensitive_data.host_keys[i] = NULL; } if (sensitive_data.host_certificates[i]) { key_free(sensitive_data.host_certificates[i]); sensitive_data.host_certificates[i] = NULL; } } sensitive_data.ssh1_host_key = NULL; explicit_bzero(sensitive_data.ssh1_cookie, SSH_SESSION_KEY_LENGTH); } /* Demote private to public keys for network child */ void demote_sensitive_data(void) { Key *tmp; int i; if (sensitive_data.server_key) { tmp = key_demote(sensitive_data.server_key); key_free(sensitive_data.server_key); sensitive_data.server_key = tmp; } for (i = 0; i < options.num_host_key_files; i++) { if (sensitive_data.host_keys[i]) { tmp = key_demote(sensitive_data.host_keys[i]); key_free(sensitive_data.host_keys[i]); sensitive_data.host_keys[i] = tmp; if (tmp->type == KEY_RSA1) sensitive_data.ssh1_host_key = tmp; } /* Certs do not need demotion */ } /* We do not clear ssh1_host key and cookie. XXX - Okay Niels? */ } static void privsep_preauth_child(void) { u_int32_t rnd[256]; gid_t gidset[1]; /* Enable challenge-response authentication for privilege separation */ privsep_challenge_enable(); #ifdef GSSAPI /* Cache supported mechanism OIDs for later use */ if (options.gss_authentication) ssh_gssapi_prepare_supported_oids(); #endif arc4random_stir(); arc4random_buf(rnd, sizeof(rnd)); #ifdef WITH_OPENSSL RAND_seed(rnd, sizeof(rnd)); if ((RAND_bytes((u_char *)rnd, 1)) != 1) fatal("%s: RAND_bytes failed", __func__); #endif explicit_bzero(rnd, sizeof(rnd)); /* Demote the private keys to public keys. */ demote_sensitive_data(); /* Demote the child */ if (getuid() == 0 || geteuid() == 0) { /* Change our root directory */ if (chroot(_PATH_PRIVSEP_CHROOT_DIR) == -1) fatal("chroot(\"%s\"): %s", _PATH_PRIVSEP_CHROOT_DIR, strerror(errno)); if (chdir("/") == -1) fatal("chdir(\"/\"): %s", strerror(errno)); /* Drop our privileges */ debug3("privsep user:group %u:%u", (u_int)privsep_pw->pw_uid, (u_int)privsep_pw->pw_gid); gidset[0] = privsep_pw->pw_gid; if (setgroups(1, gidset) < 0) fatal("setgroups: %.100s", strerror(errno)); permanently_set_uid(privsep_pw); } } static int privsep_preauth(Authctxt *authctxt) { int status, r; pid_t pid; struct ssh_sandbox *box = NULL; /* Set up unprivileged child process to deal with network data */ pmonitor = monitor_init(); /* Store a pointer to the kex for later rekeying */ pmonitor->m_pkex = &active_state->kex; if (use_privsep == PRIVSEP_ON) box = ssh_sandbox_init(pmonitor); pid = fork(); if (pid == -1) { fatal("fork of unprivileged child failed"); } else if (pid != 0) { debug2("Network child is on pid %ld", (long)pid); pmonitor->m_pid = pid; if (have_agent) { r = ssh_get_authentication_socket(&auth_sock); if (r != 0) { error("Could not get agent socket: %s", ssh_err(r)); have_agent = 0; } } if (box != NULL) ssh_sandbox_parent_preauth(box, pid); monitor_child_preauth(authctxt, pmonitor); /* Sync memory */ monitor_sync(pmonitor); /* Wait for the child's exit status */ while (waitpid(pid, &status, 0) < 0) { if (errno == EINTR) continue; pmonitor->m_pid = -1; fatal("%s: waitpid: %s", __func__, strerror(errno)); } privsep_is_preauth = 0; pmonitor->m_pid = -1; if (WIFEXITED(status)) { if (WEXITSTATUS(status) != 0) fatal("%s: preauth child exited with status %d", __func__, WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) fatal("%s: preauth child terminated by signal %d", __func__, WTERMSIG(status)); if (box != NULL) ssh_sandbox_parent_finish(box); return 1; } else { /* child */ close(pmonitor->m_sendfd); close(pmonitor->m_log_recvfd); /* Arrange for logging to be sent to the monitor */ set_log_handler(mm_log_handler, pmonitor); privsep_preauth_child(); setproctitle("%s", "[net]"); if (box != NULL) ssh_sandbox_child(box); return 0; } } static void privsep_postauth(Authctxt *authctxt) { u_int32_t rnd[256]; #ifdef DISABLE_FD_PASSING if (1) { #else if (authctxt->pw->pw_uid == 0 || options.use_login) { #endif /* File descriptor passing is broken or root login */ use_privsep = 0; goto skip; } /* New socket pair */ monitor_reinit(pmonitor); pmonitor->m_pid = fork(); if (pmonitor->m_pid == -1) fatal("fork of unprivileged child failed"); else if (pmonitor->m_pid != 0) { verbose("User child is on pid %ld", (long)pmonitor->m_pid); buffer_clear(&loginmsg); monitor_child_postauth(pmonitor); /* NEVERREACHED */ exit(0); } /* child */ close(pmonitor->m_sendfd); pmonitor->m_sendfd = -1; /* Demote the private keys to public keys. */ demote_sensitive_data(); arc4random_stir(); arc4random_buf(rnd, sizeof(rnd)); #ifdef WITH_OPENSSL RAND_seed(rnd, sizeof(rnd)); if ((RAND_bytes((u_char *)rnd, 1)) != 1) fatal("%s: RAND_bytes failed", __func__); #endif explicit_bzero(rnd, sizeof(rnd)); /* Drop privileges */ do_setusercontext(authctxt->pw); skip: /* It is safe now to apply the key state */ monitor_apply_keystate(pmonitor); /* * Tell the packet layer that authentication was successful, since * this information is not part of the key state. */ packet_set_authenticated(); } static char * list_hostkey_types(void) { Buffer b; const char *p; char *ret; int i; Key *key; buffer_init(&b); for (i = 0; i < options.num_host_key_files; i++) { key = sensitive_data.host_keys[i]; if (key == NULL) key = sensitive_data.host_pubkeys[i]; if (key == NULL || key->type == KEY_RSA1) continue; /* Check that the key is accepted in HostkeyAlgorithms */ if (match_pattern_list(sshkey_ssh_name(key), options.hostkeyalgorithms, 0) != 1) { debug3("%s: %s key not permitted by HostkeyAlgorithms", __func__, sshkey_ssh_name(key)); continue; } switch (key->type) { case KEY_RSA: case KEY_DSA: case KEY_ECDSA: case KEY_ED25519: if (buffer_len(&b) > 0) buffer_append(&b, ",", 1); p = key_ssh_name(key); buffer_append(&b, p, strlen(p)); /* for RSA we also support SHA2 signatures */ if (key->type == KEY_RSA) { p = ",rsa-sha2-512,rsa-sha2-256"; buffer_append(&b, p, strlen(p)); } break; } /* If the private key has a cert peer, then list that too */ key = sensitive_data.host_certificates[i]; if (key == NULL) continue; switch (key->type) { case KEY_RSA_CERT: case KEY_DSA_CERT: case KEY_ECDSA_CERT: case KEY_ED25519_CERT: if (buffer_len(&b) > 0) buffer_append(&b, ",", 1); p = key_ssh_name(key); buffer_append(&b, p, strlen(p)); break; } } buffer_append(&b, "\0", 1); ret = xstrdup(buffer_ptr(&b)); buffer_free(&b); debug("list_hostkey_types: %s", ret); return ret; } static Key * get_hostkey_by_type(int type, int nid, int need_private, struct ssh *ssh) { int i; Key *key; for (i = 0; i < options.num_host_key_files; i++) { switch (type) { case KEY_RSA_CERT: case KEY_DSA_CERT: case KEY_ECDSA_CERT: case KEY_ED25519_CERT: key = sensitive_data.host_certificates[i]; break; default: key = sensitive_data.host_keys[i]; if (key == NULL && !need_private) key = sensitive_data.host_pubkeys[i]; break; } if (key != NULL && key->type == type && (key->type != KEY_ECDSA || key->ecdsa_nid == nid)) return need_private ? sensitive_data.host_keys[i] : key; } return NULL; } Key * get_hostkey_public_by_type(int type, int nid, struct ssh *ssh) { return get_hostkey_by_type(type, nid, 0, ssh); } Key * get_hostkey_private_by_type(int type, int nid, struct ssh *ssh) { return get_hostkey_by_type(type, nid, 1, ssh); } Key * get_hostkey_by_index(int ind) { if (ind < 0 || ind >= options.num_host_key_files) return (NULL); return (sensitive_data.host_keys[ind]); } Key * get_hostkey_public_by_index(int ind, struct ssh *ssh) { if (ind < 0 || ind >= options.num_host_key_files) return (NULL); return (sensitive_data.host_pubkeys[ind]); } int get_hostkey_index(Key *key, int compare, struct ssh *ssh) { int i; for (i = 0; i < options.num_host_key_files; i++) { if (key_is_cert(key)) { if (key == sensitive_data.host_certificates[i] || (compare && sensitive_data.host_certificates[i] && sshkey_equal(key, sensitive_data.host_certificates[i]))) return (i); } else { if (key == sensitive_data.host_keys[i] || (compare && sensitive_data.host_keys[i] && sshkey_equal(key, sensitive_data.host_keys[i]))) return (i); if (key == sensitive_data.host_pubkeys[i] || (compare && sensitive_data.host_pubkeys[i] && sshkey_equal(key, sensitive_data.host_pubkeys[i]))) return (i); } } return (-1); } /* Inform the client of all hostkeys */ static void notify_hostkeys(struct ssh *ssh) { struct sshbuf *buf; struct sshkey *key; int i, nkeys, r; char *fp; /* Some clients cannot cope with the hostkeys message, skip those. */ if (datafellows & SSH_BUG_HOSTKEYS) return; if ((buf = sshbuf_new()) == NULL) fatal("%s: sshbuf_new", __func__); for (i = nkeys = 0; i < options.num_host_key_files; i++) { key = get_hostkey_public_by_index(i, ssh); if (key == NULL || key->type == KEY_UNSPEC || key->type == KEY_RSA1 || sshkey_is_cert(key)) continue; fp = sshkey_fingerprint(key, options.fingerprint_hash, SSH_FP_DEFAULT); debug3("%s: key %d: %s %s", __func__, i, sshkey_ssh_name(key), fp); free(fp); if (nkeys == 0) { packet_start(SSH2_MSG_GLOBAL_REQUEST); packet_put_cstring("hostkeys-00@openssh.com"); packet_put_char(0); /* want-reply */ } sshbuf_reset(buf); if ((r = sshkey_putb(key, buf)) != 0) fatal("%s: couldn't put hostkey %d: %s", __func__, i, ssh_err(r)); packet_put_string(sshbuf_ptr(buf), sshbuf_len(buf)); nkeys++; } debug3("%s: sent %d hostkeys", __func__, nkeys); if (nkeys == 0) fatal("%s: no hostkeys", __func__); packet_send(); sshbuf_free(buf); } /* * returns 1 if connection should be dropped, 0 otherwise. * dropping starts at connection #max_startups_begin with a probability * of (max_startups_rate/100). the probability increases linearly until * all connections are dropped for startups > max_startups */ static int drop_connection(int startups) { int p, r; if (startups < options.max_startups_begin) return 0; if (startups >= options.max_startups) return 1; if (options.max_startups_rate == 100) return 1; p = 100 - options.max_startups_rate; p *= startups - options.max_startups_begin; p /= options.max_startups - options.max_startups_begin; p += options.max_startups_rate; r = arc4random_uniform(100); debug("drop_connection: p %d, r %d", p, r); return (r < p) ? 1 : 0; } static void usage(void) { if (options.version_addendum && *options.version_addendum != '\0') fprintf(stderr, "%s %s, %s\n", SSH_RELEASE, options.version_addendum, OPENSSL_VERSION); else fprintf(stderr, "%s, %s\n", SSH_RELEASE, OPENSSL_VERSION); fprintf(stderr, "usage: sshd [-46DdeiqTt] [-b bits] [-C connection_spec] [-c host_cert_file]\n" " [-E log_file] [-f config_file] [-g login_grace_time]\n" " [-h host_key_file] [-k key_gen_time] [-o option] [-p port]\n" " [-u len]\n" ); exit(1); } static void send_rexec_state(int fd, Buffer *conf) { Buffer m; debug3("%s: entering fd = %d config len %d", __func__, fd, buffer_len(conf)); /* * Protocol from reexec master to child: * string configuration * u_int ephemeral_key_follows * bignum e (only if ephemeral_key_follows == 1) * bignum n " * bignum d " * bignum iqmp " * bignum p " * bignum q " * string rngseed (only if OpenSSL is not self-seeded) */ buffer_init(&m); buffer_put_cstring(&m, buffer_ptr(conf)); #ifdef WITH_SSH1 if (sensitive_data.server_key != NULL && sensitive_data.server_key->type == KEY_RSA1) { buffer_put_int(&m, 1); buffer_put_bignum(&m, sensitive_data.server_key->rsa->e); buffer_put_bignum(&m, sensitive_data.server_key->rsa->n); buffer_put_bignum(&m, sensitive_data.server_key->rsa->d); buffer_put_bignum(&m, sensitive_data.server_key->rsa->iqmp); buffer_put_bignum(&m, sensitive_data.server_key->rsa->p); buffer_put_bignum(&m, sensitive_data.server_key->rsa->q); } else #endif buffer_put_int(&m, 0); #if defined(WITH_OPENSSL) && !defined(OPENSSL_PRNG_ONLY) rexec_send_rng_seed(&m); #endif if (ssh_msg_send(fd, 0, &m) == -1) fatal("%s: ssh_msg_send failed", __func__); buffer_free(&m); debug3("%s: done", __func__); } static void recv_rexec_state(int fd, Buffer *conf) { Buffer m; char *cp; u_int len; debug3("%s: entering fd = %d", __func__, fd); buffer_init(&m); if (ssh_msg_recv(fd, &m) == -1) fatal("%s: ssh_msg_recv failed", __func__); if (buffer_get_char(&m) != 0) fatal("%s: rexec version mismatch", __func__); cp = buffer_get_string(&m, &len); if (conf != NULL) buffer_append(conf, cp, len + 1); free(cp); if (buffer_get_int(&m)) { #ifdef WITH_SSH1 if (sensitive_data.server_key != NULL) key_free(sensitive_data.server_key); sensitive_data.server_key = key_new_private(KEY_RSA1); buffer_get_bignum(&m, sensitive_data.server_key->rsa->e); buffer_get_bignum(&m, sensitive_data.server_key->rsa->n); buffer_get_bignum(&m, sensitive_data.server_key->rsa->d); buffer_get_bignum(&m, sensitive_data.server_key->rsa->iqmp); buffer_get_bignum(&m, sensitive_data.server_key->rsa->p); buffer_get_bignum(&m, sensitive_data.server_key->rsa->q); if (rsa_generate_additional_parameters( sensitive_data.server_key->rsa) != 0) fatal("%s: rsa_generate_additional_parameters " "error", __func__); #endif } #if defined(WITH_OPENSSL) && !defined(OPENSSL_PRNG_ONLY) rexec_recv_rng_seed(&m); #endif buffer_free(&m); debug3("%s: done", __func__); } /* Accept a connection from inetd */ static void server_accept_inetd(int *sock_in, int *sock_out) { int fd; startup_pipe = -1; if (rexeced_flag) { close(REEXEC_CONFIG_PASS_FD); *sock_in = *sock_out = dup(STDIN_FILENO); if (!debug_flag) { startup_pipe = dup(REEXEC_STARTUP_PIPE_FD); close(REEXEC_STARTUP_PIPE_FD); } } else { *sock_in = dup(STDIN_FILENO); *sock_out = dup(STDOUT_FILENO); } /* * We intentionally do not close the descriptors 0, 1, and 2 * as our code for setting the descriptors won't work if * ttyfd happens to be one of those. */ if ((fd = open(_PATH_DEVNULL, O_RDWR, 0)) != -1) { dup2(fd, STDIN_FILENO); dup2(fd, STDOUT_FILENO); if (!log_stderr) dup2(fd, STDERR_FILENO); if (fd > (log_stderr ? STDERR_FILENO : STDOUT_FILENO)) close(fd); } debug("inetd sockets after dupping: %d, %d", *sock_in, *sock_out); } /* * Listen for TCP connections */ static void server_listen(void) { int ret, listen_sock, on = 1; struct addrinfo *ai; char ntop[NI_MAXHOST], strport[NI_MAXSERV]; int socksize; socklen_t len; for (ai = options.listen_addrs; ai; ai = ai->ai_next) { if (ai->ai_family != AF_INET && ai->ai_family != AF_INET6) continue; if (num_listen_socks >= MAX_LISTEN_SOCKS) fatal("Too many listen sockets. " "Enlarge MAX_LISTEN_SOCKS"); if ((ret = getnameinfo(ai->ai_addr, ai->ai_addrlen, ntop, sizeof(ntop), strport, sizeof(strport), NI_NUMERICHOST|NI_NUMERICSERV)) != 0) { error("getnameinfo failed: %.100s", ssh_gai_strerror(ret)); continue; } /* Create socket for listening. */ listen_sock = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); if (listen_sock < 0) { /* kernel may not support ipv6 */ verbose("socket: %.100s", strerror(errno)); continue; } if (set_nonblock(listen_sock) == -1) { close(listen_sock); continue; } /* * Set socket options. * Allow local port reuse in TIME_WAIT. */ if (setsockopt(listen_sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) error("setsockopt SO_REUSEADDR: %s", strerror(errno)); /* Only communicate in IPv6 over AF_INET6 sockets. */ if (ai->ai_family == AF_INET6) sock_set_v6only(listen_sock); debug("Bind to port %s on %s.", strport, ntop); len = sizeof(socksize); getsockopt(listen_sock, SOL_SOCKET, SO_RCVBUF, &socksize, &len); debug("Server TCP RWIN socket size: %d", socksize); /* Bind the socket to the desired port. */ if (bind(listen_sock, ai->ai_addr, ai->ai_addrlen) < 0) { error("Bind to port %s on %s failed: %.200s.", strport, ntop, strerror(errno)); close(listen_sock); continue; } listen_socks[num_listen_socks] = listen_sock; num_listen_socks++; /* Start listening on the port. */ if (listen(listen_sock, SSH_LISTEN_BACKLOG) < 0) fatal("listen on [%s]:%s: %.100s", ntop, strport, strerror(errno)); logit("Server listening on %s port %s.", ntop, strport); } freeaddrinfo(options.listen_addrs); if (!num_listen_socks) fatal("Cannot bind any address."); } /* * The main TCP accept loop. Note that, for the non-debug case, returns * from this function are in a forked subprocess. */ static void server_accept_loop(int *sock_in, int *sock_out, int *newsock, int *config_s) { fd_set *fdset; int i, j, ret, maxfd; int key_used = 0, startups = 0; int startup_p[2] = { -1 , -1 }; struct sockaddr_storage from; socklen_t fromlen; pid_t pid; u_char rnd[256]; /* setup fd set for accept */ fdset = NULL; maxfd = 0; for (i = 0; i < num_listen_socks; i++) if (listen_socks[i] > maxfd) maxfd = listen_socks[i]; /* pipes connected to unauthenticated childs */ startup_pipes = xcalloc(options.max_startups, sizeof(int)); for (i = 0; i < options.max_startups; i++) startup_pipes[i] = -1; /* * Stay listening for connections until the system crashes or * the daemon is killed with a signal. */ for (;;) { if (received_sighup) sighup_restart(); free(fdset); fdset = xcalloc(howmany(maxfd + 1, NFDBITS), sizeof(fd_mask)); for (i = 0; i < num_listen_socks; i++) FD_SET(listen_socks[i], fdset); for (i = 0; i < options.max_startups; i++) if (startup_pipes[i] != -1) FD_SET(startup_pipes[i], fdset); /* Wait in select until there is a connection. */ ret = select(maxfd+1, fdset, NULL, NULL, NULL); if (ret < 0 && errno != EINTR) error("select: %.100s", strerror(errno)); if (received_sigterm) { logit("Received signal %d; terminating.", (int) received_sigterm); close_listen_socks(); if (options.pid_file != NULL) unlink(options.pid_file); exit(received_sigterm == SIGTERM ? 0 : 255); } if (key_used && key_do_regen) { generate_ephemeral_server_key(); key_used = 0; key_do_regen = 0; } if (ret < 0) continue; for (i = 0; i < options.max_startups; i++) if (startup_pipes[i] != -1 && FD_ISSET(startup_pipes[i], fdset)) { /* * the read end of the pipe is ready * if the child has closed the pipe * after successful authentication * or if the child has died */ close(startup_pipes[i]); startup_pipes[i] = -1; startups--; } for (i = 0; i < num_listen_socks; i++) { if (!FD_ISSET(listen_socks[i], fdset)) continue; fromlen = sizeof(from); *newsock = accept(listen_socks[i], (struct sockaddr *)&from, &fromlen); if (*newsock < 0) { if (errno != EINTR && errno != EWOULDBLOCK && errno != ECONNABORTED && errno != EAGAIN) error("accept: %.100s", strerror(errno)); if (errno == EMFILE || errno == ENFILE) usleep(100 * 1000); continue; } if (unset_nonblock(*newsock) == -1) { close(*newsock); continue; } if (drop_connection(startups) == 1) { debug("drop connection #%d", startups); close(*newsock); continue; } if (pipe(startup_p) == -1) { close(*newsock); continue; } if (rexec_flag && socketpair(AF_UNIX, SOCK_STREAM, 0, config_s) == -1) { error("reexec socketpair: %s", strerror(errno)); close(*newsock); close(startup_p[0]); close(startup_p[1]); continue; } for (j = 0; j < options.max_startups; j++) if (startup_pipes[j] == -1) { startup_pipes[j] = startup_p[0]; if (maxfd < startup_p[0]) maxfd = startup_p[0]; startups++; break; } /* * Got connection. Fork a child to handle it, unless * we are in debugging mode. */ if (debug_flag) { /* * In debugging mode. Close the listening * socket, and start processing the * connection without forking. */ debug("Server will not fork when running in debugging mode."); close_listen_socks(); *sock_in = *newsock; *sock_out = *newsock; close(startup_p[0]); close(startup_p[1]); startup_pipe = -1; pid = getpid(); if (rexec_flag) { send_rexec_state(config_s[0], &cfg); close(config_s[0]); } break; } /* * Normal production daemon. Fork, and have * the child process the connection. The * parent continues listening. */ platform_pre_fork(); if ((pid = fork()) == 0) { /* * Child. Close the listening and * max_startup sockets. Start using * the accepted socket. Reinitialize * logging (since our pid has changed). * We break out of the loop to handle * the connection. */ platform_post_fork_child(); startup_pipe = startup_p[1]; close_startup_pipes(); close_listen_socks(); *sock_in = *newsock; *sock_out = *newsock; log_init(__progname, options.log_level, options.log_facility, log_stderr); if (rexec_flag) close(config_s[0]); break; } /* Parent. Stay in the loop. */ platform_post_fork_parent(pid); if (pid < 0) error("fork: %.100s", strerror(errno)); else debug("Forked child %ld.", (long)pid); close(startup_p[1]); if (rexec_flag) { send_rexec_state(config_s[0], &cfg); close(config_s[0]); close(config_s[1]); } /* * Mark that the key has been used (it * was "given" to the child). */ if ((options.protocol & SSH_PROTO_1) && key_used == 0) { /* Schedule server key regeneration alarm. */ signal(SIGALRM, key_regeneration_alarm); alarm(options.key_regeneration_time); key_used = 1; } close(*newsock); /* * Ensure that our random state differs * from that of the child */ arc4random_stir(); arc4random_buf(rnd, sizeof(rnd)); #ifdef WITH_OPENSSL RAND_seed(rnd, sizeof(rnd)); if ((RAND_bytes((u_char *)rnd, 1)) != 1) fatal("%s: RAND_bytes failed", __func__); #endif explicit_bzero(rnd, sizeof(rnd)); } /* child process check (or debug mode) */ if (num_listen_socks < 0) break; } } /* * Main program for the daemon. */ int main(int ac, char **av) { extern char *optarg; extern int optind; int r, opt, i, j, on = 1; int sock_in = -1, sock_out = -1, newsock = -1; const char *remote_ip; int remote_port; char *fp, *line, *laddr, *logfile = NULL; int config_s[2] = { -1 , -1 }; u_int n; u_int64_t ibytes, obytes; mode_t new_umask; Key *key; Key *pubkey; int keytype; Authctxt *authctxt; struct connection_info *connection_info = get_connection_info(0, 0); ssh_malloc_init(); /* must be called before any mallocs */ #ifdef HAVE_SECUREWARE (void)set_auth_parameters(ac, av); #endif __progname = ssh_get_progname(av[0]); /* Save argv. Duplicate so setproctitle emulation doesn't clobber it */ saved_argc = ac; rexec_argc = ac; saved_argv = xcalloc(ac + 1, sizeof(*saved_argv)); for (i = 0; i < ac; i++) saved_argv[i] = xstrdup(av[i]); saved_argv[i] = NULL; #ifndef HAVE_SETPROCTITLE /* Prepare for later setproctitle emulation */ compat_init_setproctitle(ac, av); av = saved_argv; #endif if (geteuid() == 0 && setgroups(0, NULL) == -1) debug("setgroups(): %.200s", strerror(errno)); /* Ensure that fds 0, 1 and 2 are open or directed to /dev/null */ sanitise_stdfd(); /* Initialize configuration options to their default values. */ initialize_server_options(&options); /* Parse command-line arguments. */ while ((opt = getopt(ac, av, "C:E:b:c:f:g:h:k:o:p:u:46DQRTdeiqrt")) != -1) { switch (opt) { case '4': options.address_family = AF_INET; break; case '6': options.address_family = AF_INET6; break; case 'f': config_file_name = optarg; break; case 'c': if (options.num_host_cert_files >= MAX_HOSTCERTS) { fprintf(stderr, "too many host certificates.\n"); exit(1); } options.host_cert_files[options.num_host_cert_files++] = derelativise_path(optarg); break; case 'd': if (debug_flag == 0) { debug_flag = 1; options.log_level = SYSLOG_LEVEL_DEBUG1; } else if (options.log_level < SYSLOG_LEVEL_DEBUG3) options.log_level++; break; case 'D': no_daemon_flag = 1; break; case 'E': logfile = optarg; /* FALLTHROUGH */ case 'e': log_stderr = 1; break; case 'i': inetd_flag = 1; break; case 'r': rexec_flag = 0; break; case 'R': rexeced_flag = 1; inetd_flag = 1; break; case 'Q': /* ignored */ break; case 'q': options.log_level = SYSLOG_LEVEL_QUIET; break; case 'b': options.server_key_bits = (int)strtonum(optarg, 256, 32768, NULL); break; case 'p': options.ports_from_cmdline = 1; if (options.num_ports >= MAX_PORTS) { fprintf(stderr, "too many ports.\n"); exit(1); } options.ports[options.num_ports++] = a2port(optarg); if (options.ports[options.num_ports-1] <= 0) { fprintf(stderr, "Bad port number.\n"); exit(1); } break; case 'g': if ((options.login_grace_time = convtime(optarg)) == -1) { fprintf(stderr, "Invalid login grace time.\n"); exit(1); } break; case 'k': if ((options.key_regeneration_time = convtime(optarg)) == -1) { fprintf(stderr, "Invalid key regeneration interval.\n"); exit(1); } break; case 'h': if (options.num_host_key_files >= MAX_HOSTKEYS) { fprintf(stderr, "too many host keys.\n"); exit(1); } options.host_key_files[options.num_host_key_files++] = derelativise_path(optarg); break; case 't': test_flag = 1; break; case 'T': test_flag = 2; break; case 'C': if (parse_server_match_testspec(connection_info, optarg) == -1) exit(1); break; case 'u': utmp_len = (u_int)strtonum(optarg, 0, HOST_NAME_MAX+1+1, NULL); if (utmp_len > HOST_NAME_MAX+1) { fprintf(stderr, "Invalid utmp length.\n"); exit(1); } break; case 'o': line = xstrdup(optarg); if (process_server_config_line(&options, line, "command-line", 0, NULL, NULL) != 0) exit(1); free(line); break; case '?': default: usage(); break; } } if (rexeced_flag || inetd_flag) rexec_flag = 0; if (!test_flag && (rexec_flag && (av[0] == NULL || *av[0] != '/'))) fatal("sshd re-exec requires execution with an absolute path"); if (rexeced_flag) closefrom(REEXEC_MIN_FREE_FD); else closefrom(REEXEC_DEVCRYPTO_RESERVED_FD); #ifdef WITH_OPENSSL OpenSSL_add_all_algorithms(); #endif /* If requested, redirect the logs to the specified logfile. */ if (logfile != NULL) log_redirect_stderr_to(logfile); /* * Force logging to stderr until we have loaded the private host * key (unless started from inetd) */ log_init(__progname, options.log_level == SYSLOG_LEVEL_NOT_SET ? SYSLOG_LEVEL_INFO : options.log_level, options.log_facility == SYSLOG_FACILITY_NOT_SET ? SYSLOG_FACILITY_AUTH : options.log_facility, log_stderr || !inetd_flag); /* * Unset KRB5CCNAME, otherwise the user's session may inherit it from * root's environment */ if (getenv("KRB5CCNAME") != NULL) (void) unsetenv("KRB5CCNAME"); #ifdef _UNICOS /* Cray can define user privs drop all privs now! * Not needed on PRIV_SU systems! */ drop_cray_privs(); #endif sensitive_data.server_key = NULL; sensitive_data.ssh1_host_key = NULL; sensitive_data.have_ssh1_key = 0; sensitive_data.have_ssh2_key = 0; /* * If we're doing an extended config test, make sure we have all of * the parameters we need. If we're not doing an extended test, * do not silently ignore connection test params. */ if (test_flag >= 2 && server_match_spec_complete(connection_info) == 0) fatal("user, host and addr are all required when testing " "Match configs"); if (test_flag < 2 && server_match_spec_complete(connection_info) >= 0) fatal("Config test connection parameter (-C) provided without " "test mode (-T)"); /* Fetch our configuration */ buffer_init(&cfg); if (rexeced_flag) recv_rexec_state(REEXEC_CONFIG_PASS_FD, &cfg); else if (strcasecmp(config_file_name, "none") != 0) load_server_config(config_file_name, &cfg); parse_server_config(&options, rexeced_flag ? "rexec" : config_file_name, &cfg, NULL); seed_rng(); /* Fill in default values for those options not explicitly set. */ fill_default_server_options(&options); /* challenge-response is implemented via keyboard interactive */ if (options.challenge_response_authentication) options.kbd_interactive_authentication = 1; /* Check that options are sensible */ if (options.authorized_keys_command_user == NULL && (options.authorized_keys_command != NULL && strcasecmp(options.authorized_keys_command, "none") != 0)) fatal("AuthorizedKeysCommand set without " "AuthorizedKeysCommandUser"); if (options.authorized_principals_command_user == NULL && (options.authorized_principals_command != NULL && strcasecmp(options.authorized_principals_command, "none") != 0)) fatal("AuthorizedPrincipalsCommand set without " "AuthorizedPrincipalsCommandUser"); /* * Check whether there is any path through configured auth methods. * Unfortunately it is not possible to verify this generally before * daemonisation in the presence of Match block, but this catches * and warns for trivial misconfigurations that could break login. */ if (options.num_auth_methods != 0) { if ((options.protocol & SSH_PROTO_1)) fatal("AuthenticationMethods is not supported with " "SSH protocol 1"); for (n = 0; n < options.num_auth_methods; n++) { if (auth2_methods_valid(options.auth_methods[n], 1) == 0) break; } if (n >= options.num_auth_methods) fatal("AuthenticationMethods cannot be satisfied by " "enabled authentication methods"); } /* set default channel AF */ channel_set_af(options.address_family); /* Check that there are no remaining arguments. */ if (optind < ac) { fprintf(stderr, "Extra argument %s.\n", av[optind]); exit(1); } debug("sshd version %s, %s", SSH_VERSION, #ifdef WITH_OPENSSL SSLeay_version(SSLEAY_VERSION) #else "without OpenSSL" #endif ); /* Store privilege separation user for later use if required. */ if ((privsep_pw = getpwnam(SSH_PRIVSEP_USER)) == NULL) { if (use_privsep || options.kerberos_authentication) fatal("Privilege separation user %s does not exist", SSH_PRIVSEP_USER); } else { explicit_bzero(privsep_pw->pw_passwd, strlen(privsep_pw->pw_passwd)); privsep_pw = pwcopy(privsep_pw); free(privsep_pw->pw_passwd); privsep_pw->pw_passwd = xstrdup("*"); } endpwent(); /* load host keys */ sensitive_data.host_keys = xcalloc(options.num_host_key_files, sizeof(Key *)); sensitive_data.host_pubkeys = xcalloc(options.num_host_key_files, sizeof(Key *)); if (options.host_key_agent) { if (strcmp(options.host_key_agent, SSH_AUTHSOCKET_ENV_NAME)) setenv(SSH_AUTHSOCKET_ENV_NAME, options.host_key_agent, 1); if ((r = ssh_get_authentication_socket(NULL)) == 0) have_agent = 1; else error("Could not connect to agent \"%s\": %s", options.host_key_agent, ssh_err(r)); } for (i = 0; i < options.num_host_key_files; i++) { if (options.host_key_files[i] == NULL) continue; key = key_load_private(options.host_key_files[i], "", NULL); pubkey = key_load_public(options.host_key_files[i], NULL); if (pubkey == NULL && key != NULL) pubkey = key_demote(key); sensitive_data.host_keys[i] = key; sensitive_data.host_pubkeys[i] = pubkey; if (key == NULL && pubkey != NULL && pubkey->type != KEY_RSA1 && have_agent) { debug("will rely on agent for hostkey %s", options.host_key_files[i]); keytype = pubkey->type; } else if (key != NULL) { keytype = key->type; } else { error("Could not load host key: %s", options.host_key_files[i]); sensitive_data.host_keys[i] = NULL; sensitive_data.host_pubkeys[i] = NULL; continue; } switch (keytype) { case KEY_RSA1: sensitive_data.ssh1_host_key = key; sensitive_data.have_ssh1_key = 1; break; case KEY_RSA: case KEY_DSA: case KEY_ECDSA: case KEY_ED25519: if (have_agent || key != NULL) sensitive_data.have_ssh2_key = 1; break; } if ((fp = sshkey_fingerprint(pubkey, options.fingerprint_hash, SSH_FP_DEFAULT)) == NULL) fatal("sshkey_fingerprint failed"); debug("%s host key #%d: %s %s", key ? "private" : "agent", i, keytype == KEY_RSA1 ? sshkey_type(pubkey) : sshkey_ssh_name(pubkey), fp); free(fp); } if ((options.protocol & SSH_PROTO_1) && !sensitive_data.have_ssh1_key) { logit("Disabling protocol version 1. Could not load host key"); options.protocol &= ~SSH_PROTO_1; } if ((options.protocol & SSH_PROTO_2) && !sensitive_data.have_ssh2_key) { logit("Disabling protocol version 2. Could not load host key"); options.protocol &= ~SSH_PROTO_2; } if (!(options.protocol & (SSH_PROTO_1|SSH_PROTO_2))) { logit("sshd: no hostkeys available -- exiting."); exit(1); } /* * Load certificates. They are stored in an array at identical * indices to the public keys that they relate to. */ sensitive_data.host_certificates = xcalloc(options.num_host_key_files, sizeof(Key *)); for (i = 0; i < options.num_host_key_files; i++) sensitive_data.host_certificates[i] = NULL; for (i = 0; i < options.num_host_cert_files; i++) { if (options.host_cert_files[i] == NULL) continue; key = key_load_public(options.host_cert_files[i], NULL); if (key == NULL) { error("Could not load host certificate: %s", options.host_cert_files[i]); continue; } if (!key_is_cert(key)) { error("Certificate file is not a certificate: %s", options.host_cert_files[i]); key_free(key); continue; } /* Find matching private key */ for (j = 0; j < options.num_host_key_files; j++) { if (key_equal_public(key, sensitive_data.host_keys[j])) { sensitive_data.host_certificates[j] = key; break; } } if (j >= options.num_host_key_files) { error("No matching private key for certificate: %s", options.host_cert_files[i]); key_free(key); continue; } sensitive_data.host_certificates[j] = key; debug("host certificate: #%d type %d %s", j, key->type, key_type(key)); } #ifdef WITH_SSH1 /* Check certain values for sanity. */ if (options.protocol & SSH_PROTO_1) { if (options.server_key_bits < SSH_RSA_MINIMUM_MODULUS_SIZE || options.server_key_bits > OPENSSL_RSA_MAX_MODULUS_BITS) { fprintf(stderr, "Bad server key size.\n"); exit(1); } /* * Check that server and host key lengths differ sufficiently. This * is necessary to make double encryption work with rsaref. Oh, I * hate software patents. I dont know if this can go? Niels */ if (options.server_key_bits > BN_num_bits(sensitive_data.ssh1_host_key->rsa->n) - SSH_KEY_BITS_RESERVED && options.server_key_bits < BN_num_bits(sensitive_data.ssh1_host_key->rsa->n) + SSH_KEY_BITS_RESERVED) { options.server_key_bits = BN_num_bits(sensitive_data.ssh1_host_key->rsa->n) + SSH_KEY_BITS_RESERVED; debug("Forcing server key to %d bits to make it differ from host key.", options.server_key_bits); } } #endif if (use_privsep) { struct stat st; if ((stat(_PATH_PRIVSEP_CHROOT_DIR, &st) == -1) || (S_ISDIR(st.st_mode) == 0)) fatal("Missing privilege separation directory: %s", _PATH_PRIVSEP_CHROOT_DIR); #ifdef HAVE_CYGWIN if (check_ntsec(_PATH_PRIVSEP_CHROOT_DIR) && (st.st_uid != getuid () || (st.st_mode & (S_IWGRP|S_IWOTH)) != 0)) #else if (st.st_uid != 0 || (st.st_mode & (S_IWGRP|S_IWOTH)) != 0) #endif fatal("%s must be owned by root and not group or " "world-writable.", _PATH_PRIVSEP_CHROOT_DIR); } if (test_flag > 1) { if (server_match_spec_complete(connection_info) == 1) parse_server_match_config(&options, connection_info); dump_config(&options); } /* Configuration looks good, so exit if in test mode. */ if (test_flag) exit(0); /* * Clear out any supplemental groups we may have inherited. This * prevents inadvertent creation of files with bad modes (in the * portable version at least, it's certainly possible for PAM * to create a file, and we can't control the code in every * module which might be used). */ if (setgroups(0, NULL) < 0) debug("setgroups() failed: %.200s", strerror(errno)); if (rexec_flag) { rexec_argv = xcalloc(rexec_argc + 2, sizeof(char *)); for (i = 0; i < rexec_argc; i++) { debug("rexec_argv[%d]='%s'", i, saved_argv[i]); rexec_argv[i] = saved_argv[i]; } rexec_argv[rexec_argc] = "-R"; rexec_argv[rexec_argc + 1] = NULL; } /* Ensure that umask disallows at least group and world write */ new_umask = umask(0077) | 0022; (void) umask(new_umask); /* Initialize the log (it is reinitialized below in case we forked). */ if (debug_flag && (!inetd_flag || rexeced_flag)) log_stderr = 1; log_init(__progname, options.log_level, options.log_facility, log_stderr); /* * If not in debugging mode, and not started from inetd, disconnect * from the controlling terminal, and fork. The original process * exits. */ if (!(debug_flag || inetd_flag || no_daemon_flag)) { #ifdef TIOCNOTTY int fd; #endif /* TIOCNOTTY */ if (daemon(0, 0) < 0) fatal("daemon() failed: %.200s", strerror(errno)); /* Disconnect from the controlling tty. */ #ifdef TIOCNOTTY fd = open(_PATH_TTY, O_RDWR | O_NOCTTY); if (fd >= 0) { (void) ioctl(fd, TIOCNOTTY, NULL); close(fd); } #endif /* TIOCNOTTY */ } /* Reinitialize the log (because of the fork above). */ log_init(__progname, options.log_level, options.log_facility, log_stderr); /* Avoid killing the process in high-pressure swapping environments. */ if (!inetd_flag && madvise(NULL, 0, MADV_PROTECT) != 0) debug("madvise(): %.200s", strerror(errno)); /* Chdir to the root directory so that the current disk can be unmounted if desired. */ if (chdir("/") == -1) error("chdir(\"/\"): %s", strerror(errno)); /* ignore SIGPIPE */ signal(SIGPIPE, SIG_IGN); /* Get a connection, either from inetd or a listening TCP socket */ if (inetd_flag) { server_accept_inetd(&sock_in, &sock_out); } else { platform_pre_listen(); server_listen(); if (options.protocol & SSH_PROTO_1) generate_ephemeral_server_key(); signal(SIGHUP, sighup_handler); signal(SIGCHLD, main_sigchld_handler); signal(SIGTERM, sigterm_handler); signal(SIGQUIT, sigterm_handler); /* * Write out the pid file after the sigterm handler * is setup and the listen sockets are bound */ if (options.pid_file != NULL && !debug_flag) { FILE *f = fopen(options.pid_file, "w"); if (f == NULL) { error("Couldn't create pid file \"%s\": %s", options.pid_file, strerror(errno)); } else { fprintf(f, "%ld\n", (long) getpid()); fclose(f); } } /* Accept a connection and return in a forked child */ server_accept_loop(&sock_in, &sock_out, &newsock, config_s); } /* This is the child processing a new connection. */ setproctitle("%s", "[accepted]"); /* * Create a new session and process group since the 4.4BSD * setlogin() affects the entire process group. We don't * want the child to be able to affect the parent. */ #if !defined(SSHD_ACQUIRES_CTTY) /* * If setsid is called, on some platforms sshd will later acquire a * controlling terminal which will result in "could not set * controlling tty" errors. */ if (!debug_flag && !inetd_flag && setsid() < 0) error("setsid: %.100s", strerror(errno)); #endif if (rexec_flag) { int fd; debug("rexec start in %d out %d newsock %d pipe %d sock %d", sock_in, sock_out, newsock, startup_pipe, config_s[0]); dup2(newsock, STDIN_FILENO); dup2(STDIN_FILENO, STDOUT_FILENO); if (startup_pipe == -1) close(REEXEC_STARTUP_PIPE_FD); else if (startup_pipe != REEXEC_STARTUP_PIPE_FD) { dup2(startup_pipe, REEXEC_STARTUP_PIPE_FD); close(startup_pipe); startup_pipe = REEXEC_STARTUP_PIPE_FD; } dup2(config_s[1], REEXEC_CONFIG_PASS_FD); close(config_s[1]); execv(rexec_argv[0], rexec_argv); /* Reexec has failed, fall back and continue */ error("rexec of %s failed: %s", rexec_argv[0], strerror(errno)); recv_rexec_state(REEXEC_CONFIG_PASS_FD, NULL); log_init(__progname, options.log_level, options.log_facility, log_stderr); /* Clean up fds */ close(REEXEC_CONFIG_PASS_FD); newsock = sock_out = sock_in = dup(STDIN_FILENO); if ((fd = open(_PATH_DEVNULL, O_RDWR, 0)) != -1) { dup2(fd, STDIN_FILENO); dup2(fd, STDOUT_FILENO); if (fd > STDERR_FILENO) close(fd); } debug("rexec cleanup in %d out %d newsock %d pipe %d sock %d", sock_in, sock_out, newsock, startup_pipe, config_s[0]); } /* Executed child processes don't need these. */ fcntl(sock_out, F_SETFD, FD_CLOEXEC); fcntl(sock_in, F_SETFD, FD_CLOEXEC); /* * Disable the key regeneration alarm. We will not regenerate the * key since we are no longer in a position to give it to anyone. We * will not restart on SIGHUP since it no longer makes sense. */ alarm(0); signal(SIGALRM, SIG_DFL); signal(SIGHUP, SIG_DFL); signal(SIGTERM, SIG_DFL); signal(SIGQUIT, SIG_DFL); signal(SIGCHLD, SIG_DFL); signal(SIGINT, SIG_DFL); #ifdef __FreeBSD__ /* * Initialize the resolver. This may not happen automatically * before privsep chroot(). */ if ((_res.options & RES_INIT) == 0) { debug("res_init()"); res_init(); } #ifdef GSSAPI /* * Force GSS-API to parse its configuration and load any * mechanism plugins. */ { gss_OID_set mechs; OM_uint32 minor_status; gss_indicate_mechs(&minor_status, &mechs); gss_release_oid_set(&minor_status, &mechs); } #endif #endif /* * Register our connection. This turns encryption off because we do * not have a key. */ packet_set_connection(sock_in, sock_out); packet_set_server(); /* Set SO_KEEPALIVE if requested. */ if (options.tcp_keep_alive && packet_connection_is_on_socket() && setsockopt(sock_in, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof(on)) < 0) error("setsockopt SO_KEEPALIVE: %.100s", strerror(errno)); if ((remote_port = get_remote_port()) < 0) { debug("get_remote_port failed"); cleanup_exit(255); } /* * We use get_canonical_hostname with usedns = 0 instead of * get_remote_ipaddr here so IP options will be checked. */ (void) get_canonical_hostname(0); /* * The rest of the code depends on the fact that * get_remote_ipaddr() caches the remote ip, even if * the socket goes away. */ remote_ip = get_remote_ipaddr(); #ifdef SSH_AUDIT_EVENTS audit_connection_from(remote_ip, remote_port); #endif #ifdef LIBWRAP allow_severity = options.log_facility|LOG_INFO; deny_severity = options.log_facility|LOG_WARNING; /* Check whether logins are denied from this host. */ if (packet_connection_is_on_socket()) { struct request_info req; request_init(&req, RQ_DAEMON, __progname, RQ_FILE, sock_in, 0); fromhost(&req); if (!hosts_access(&req)) { debug("Connection refused by tcp wrapper"); refuse(&req); /* NOTREACHED */ fatal("libwrap refuse returns"); } } #endif /* LIBWRAP */ /* Log the connection. */ laddr = get_local_ipaddr(sock_in); verbose("Connection from %s port %d on %s port %d", remote_ip, remote_port, laddr, get_local_port()); free(laddr); /* * We don't want to listen forever unless the other side * successfully authenticates itself. So we set up an alarm which is * cleared after successful authentication. A limit of zero * indicates no limit. Note that we don't set the alarm in debugging * mode; it is just annoying to have the server exit just when you * are about to discover the bug. */ signal(SIGALRM, grace_alarm_handler); if (!debug_flag) alarm(options.login_grace_time); sshd_exchange_identification(sock_in, sock_out); /* In inetd mode, generate ephemeral key only for proto 1 connections */ if (!compat20 && inetd_flag && sensitive_data.server_key == NULL) generate_ephemeral_server_key(); packet_set_nonblocking(); /* allocate authentication context */ authctxt = xcalloc(1, sizeof(*authctxt)); authctxt->loginmsg = &loginmsg; /* XXX global for cleanup, access from other modules */ the_authctxt = authctxt; /* prepare buffer to collect messages to display to user after login */ buffer_init(&loginmsg); auth_debug_reset(); + + BLACKLIST_INIT(); if (use_privsep) { if (privsep_preauth(authctxt) == 1) goto authenticated; } else if (compat20 && have_agent) { if ((r = ssh_get_authentication_socket(&auth_sock)) != 0) { error("Unable to get agent socket: %s", ssh_err(r)); have_agent = 0; } } /* perform the key exchange */ /* authenticate user and start session */ if (compat20) { do_ssh2_kex(); do_authentication2(authctxt); } else { #ifdef WITH_SSH1 do_ssh1_kex(); do_authentication(authctxt); #else fatal("ssh1 not supported"); #endif } /* * If we use privilege separation, the unprivileged child transfers * the current keystate and exits */ if (use_privsep) { mm_send_keystate(pmonitor); exit(0); } authenticated: /* * Cancel the alarm we set to limit the time taken for * authentication. */ alarm(0); signal(SIGALRM, SIG_DFL); authctxt->authenticated = 1; if (startup_pipe != -1) { close(startup_pipe); startup_pipe = -1; } #ifdef SSH_AUDIT_EVENTS audit_event(SSH_AUTH_SUCCESS); #endif #ifdef GSSAPI if (options.gss_authentication) { temporarily_use_uid(authctxt->pw); ssh_gssapi_storecreds(); restore_uid(); } #endif #ifdef USE_PAM if (options.use_pam) { do_pam_setcred(1); do_pam_session(); } #endif /* * In privilege separation, we fork another child and prepare * file descriptor passing. */ if (use_privsep) { privsep_postauth(authctxt); /* the monitor process [priv] will not return */ if (!compat20) destroy_sensitive_data(); } packet_set_timeout(options.client_alive_interval, options.client_alive_count_max); /* Try to send all our hostkeys to the client */ if (compat20) notify_hostkeys(active_state); /* Start session. */ do_authenticated(authctxt); /* The connection has been terminated. */ packet_get_bytes(&ibytes, &obytes); verbose("Transferred: sent %llu, received %llu bytes", (unsigned long long)obytes, (unsigned long long)ibytes); verbose("Closing connection to %.500s port %d", remote_ip, remote_port); #ifdef USE_PAM if (options.use_pam) finish_pam(); #endif /* USE_PAM */ #ifdef SSH_AUDIT_EVENTS PRIVSEP(audit_event(SSH_CONNECTION_CLOSE)); #endif packet_close(); if (use_privsep) mm_terminate(); exit(0); } #ifdef WITH_SSH1 /* * Decrypt session_key_int using our private server key and private host key * (key with larger modulus first). */ int ssh1_session_key(BIGNUM *session_key_int) { int rsafail = 0; if (BN_cmp(sensitive_data.server_key->rsa->n, sensitive_data.ssh1_host_key->rsa->n) > 0) { /* Server key has bigger modulus. */ if (BN_num_bits(sensitive_data.server_key->rsa->n) < BN_num_bits(sensitive_data.ssh1_host_key->rsa->n) + SSH_KEY_BITS_RESERVED) { fatal("do_connection: %s: " "server_key %d < host_key %d + SSH_KEY_BITS_RESERVED %d", get_remote_ipaddr(), BN_num_bits(sensitive_data.server_key->rsa->n), BN_num_bits(sensitive_data.ssh1_host_key->rsa->n), SSH_KEY_BITS_RESERVED); } if (rsa_private_decrypt(session_key_int, session_key_int, sensitive_data.server_key->rsa) != 0) rsafail++; if (rsa_private_decrypt(session_key_int, session_key_int, sensitive_data.ssh1_host_key->rsa) != 0) rsafail++; } else { /* Host key has bigger modulus (or they are equal). */ if (BN_num_bits(sensitive_data.ssh1_host_key->rsa->n) < BN_num_bits(sensitive_data.server_key->rsa->n) + SSH_KEY_BITS_RESERVED) { fatal("do_connection: %s: " "host_key %d < server_key %d + SSH_KEY_BITS_RESERVED %d", get_remote_ipaddr(), BN_num_bits(sensitive_data.ssh1_host_key->rsa->n), BN_num_bits(sensitive_data.server_key->rsa->n), SSH_KEY_BITS_RESERVED); } if (rsa_private_decrypt(session_key_int, session_key_int, sensitive_data.ssh1_host_key->rsa) != 0) rsafail++; if (rsa_private_decrypt(session_key_int, session_key_int, sensitive_data.server_key->rsa) != 0) rsafail++; } return (rsafail); } /* * SSH1 key exchange */ static void do_ssh1_kex(void) { int i, len; int rsafail = 0; BIGNUM *session_key_int, *fake_key_int, *real_key_int; u_char session_key[SSH_SESSION_KEY_LENGTH]; u_char fake_key_bytes[4096 / 8]; size_t fake_key_len; u_char cookie[8]; u_int cipher_type, auth_mask, protocol_flags; /* * Generate check bytes that the client must send back in the user * packet in order for it to be accepted; this is used to defy ip * spoofing attacks. Note that this only works against somebody * doing IP spoofing from a remote machine; any machine on the local * network can still see outgoing packets and catch the random * cookie. This only affects rhosts authentication, and this is one * of the reasons why it is inherently insecure. */ arc4random_buf(cookie, sizeof(cookie)); /* * Send our public key. We include in the packet 64 bits of random * data that must be matched in the reply in order to prevent IP * spoofing. */ packet_start(SSH_SMSG_PUBLIC_KEY); for (i = 0; i < 8; i++) packet_put_char(cookie[i]); /* Store our public server RSA key. */ packet_put_int(BN_num_bits(sensitive_data.server_key->rsa->n)); packet_put_bignum(sensitive_data.server_key->rsa->e); packet_put_bignum(sensitive_data.server_key->rsa->n); /* Store our public host RSA key. */ packet_put_int(BN_num_bits(sensitive_data.ssh1_host_key->rsa->n)); packet_put_bignum(sensitive_data.ssh1_host_key->rsa->e); packet_put_bignum(sensitive_data.ssh1_host_key->rsa->n); /* Put protocol flags. */ packet_put_int(SSH_PROTOFLAG_HOST_IN_FWD_OPEN); /* Declare which ciphers we support. */ packet_put_int(cipher_mask_ssh1(0)); /* Declare supported authentication types. */ auth_mask = 0; if (options.rhosts_rsa_authentication) auth_mask |= 1 << SSH_AUTH_RHOSTS_RSA; if (options.rsa_authentication) auth_mask |= 1 << SSH_AUTH_RSA; if (options.challenge_response_authentication == 1) auth_mask |= 1 << SSH_AUTH_TIS; if (options.password_authentication) auth_mask |= 1 << SSH_AUTH_PASSWORD; packet_put_int(auth_mask); /* Send the packet and wait for it to be sent. */ packet_send(); packet_write_wait(); debug("Sent %d bit server key and %d bit host key.", BN_num_bits(sensitive_data.server_key->rsa->n), BN_num_bits(sensitive_data.ssh1_host_key->rsa->n)); /* Read clients reply (cipher type and session key). */ packet_read_expect(SSH_CMSG_SESSION_KEY); /* Get cipher type and check whether we accept this. */ cipher_type = packet_get_char(); if (!(cipher_mask_ssh1(0) & (1 << cipher_type))) packet_disconnect("Warning: client selects unsupported cipher."); /* Get check bytes from the packet. These must match those we sent earlier with the public key packet. */ for (i = 0; i < 8; i++) if (cookie[i] != packet_get_char()) packet_disconnect("IP Spoofing check bytes do not match."); debug("Encryption type: %.200s", cipher_name(cipher_type)); /* Get the encrypted integer. */ if ((real_key_int = BN_new()) == NULL) fatal("do_ssh1_kex: BN_new failed"); packet_get_bignum(real_key_int); protocol_flags = packet_get_int(); packet_set_protocol_flags(protocol_flags); packet_check_eom(); /* Setup a fake key in case RSA decryption fails */ if ((fake_key_int = BN_new()) == NULL) fatal("do_ssh1_kex: BN_new failed"); fake_key_len = BN_num_bytes(real_key_int); if (fake_key_len > sizeof(fake_key_bytes)) fake_key_len = sizeof(fake_key_bytes); arc4random_buf(fake_key_bytes, fake_key_len); if (BN_bin2bn(fake_key_bytes, fake_key_len, fake_key_int) == NULL) fatal("do_ssh1_kex: BN_bin2bn failed"); /* Decrypt real_key_int using host/server keys */ rsafail = PRIVSEP(ssh1_session_key(real_key_int)); /* If decryption failed, use the fake key. Else, the real key. */ if (rsafail) session_key_int = fake_key_int; else session_key_int = real_key_int; /* * Extract session key from the decrypted integer. The key is in the * least significant 256 bits of the integer; the first byte of the * key is in the highest bits. */ (void) BN_mask_bits(session_key_int, sizeof(session_key) * 8); len = BN_num_bytes(session_key_int); if (len < 0 || (u_int)len > sizeof(session_key)) { error("do_ssh1_kex: bad session key len from %s: " "session_key_int %d > sizeof(session_key) %lu", get_remote_ipaddr(), len, (u_long)sizeof(session_key)); rsafail++; } else { explicit_bzero(session_key, sizeof(session_key)); BN_bn2bin(session_key_int, session_key + sizeof(session_key) - len); derive_ssh1_session_id( sensitive_data.ssh1_host_key->rsa->n, sensitive_data.server_key->rsa->n, cookie, session_id); /* * Xor the first 16 bytes of the session key with the * session id. */ for (i = 0; i < 16; i++) session_key[i] ^= session_id[i]; } /* Destroy the private and public keys. No longer. */ destroy_sensitive_data(); if (use_privsep) mm_ssh1_session_id(session_id); /* Destroy the decrypted integer. It is no longer needed. */ BN_clear_free(real_key_int); BN_clear_free(fake_key_int); /* Set the session key. From this on all communications will be encrypted. */ packet_set_encryption_key(session_key, SSH_SESSION_KEY_LENGTH, cipher_type); /* Destroy our copy of the session key. It is no longer needed. */ explicit_bzero(session_key, sizeof(session_key)); debug("Received session key; encryption turned on."); /* Send an acknowledgment packet. Note that this packet is sent encrypted. */ packet_start(SSH_SMSG_SUCCESS); packet_send(); packet_write_wait(); } #endif int sshd_hostkey_sign(Key *privkey, Key *pubkey, u_char **signature, size_t *slen, const u_char *data, size_t dlen, const char *alg, u_int flag) { int r; u_int xxx_slen, xxx_dlen = dlen; if (privkey) { if (PRIVSEP(key_sign(privkey, signature, &xxx_slen, data, xxx_dlen, alg) < 0)) fatal("%s: key_sign failed", __func__); if (slen) *slen = xxx_slen; } else if (use_privsep) { if (mm_key_sign(pubkey, signature, &xxx_slen, data, xxx_dlen, alg) < 0) fatal("%s: pubkey_sign failed", __func__); if (slen) *slen = xxx_slen; } else { if ((r = ssh_agent_sign(auth_sock, pubkey, signature, slen, data, dlen, alg, datafellows)) != 0) fatal("%s: ssh_agent_sign failed: %s", __func__, ssh_err(r)); } return 0; } /* SSH2 key exchange */ static void do_ssh2_kex(void) { char *myproposal[PROPOSAL_MAX] = { KEX_SERVER }; struct kex *kex; int r; myproposal[PROPOSAL_KEX_ALGS] = compat_kex_proposal( options.kex_algorithms); myproposal[PROPOSAL_ENC_ALGS_CTOS] = compat_cipher_proposal( options.ciphers); myproposal[PROPOSAL_ENC_ALGS_STOC] = compat_cipher_proposal( options.ciphers); myproposal[PROPOSAL_MAC_ALGS_CTOS] = myproposal[PROPOSAL_MAC_ALGS_STOC] = options.macs; if (options.compression == COMP_NONE) { myproposal[PROPOSAL_COMP_ALGS_CTOS] = myproposal[PROPOSAL_COMP_ALGS_STOC] = "none"; } else if (options.compression == COMP_DELAYED) { myproposal[PROPOSAL_COMP_ALGS_CTOS] = myproposal[PROPOSAL_COMP_ALGS_STOC] = "none,zlib@openssh.com"; } if (options.rekey_limit || options.rekey_interval) packet_set_rekey_limits(options.rekey_limit, (time_t)options.rekey_interval); myproposal[PROPOSAL_SERVER_HOST_KEY_ALGS] = compat_pkalg_proposal( list_hostkey_types()); /* start key exchange */ if ((r = kex_setup(active_state, myproposal)) != 0) fatal("kex_setup: %s", ssh_err(r)); kex = active_state->kex; #ifdef WITH_OPENSSL kex->kex[KEX_DH_GRP1_SHA1] = kexdh_server; kex->kex[KEX_DH_GRP14_SHA1] = kexdh_server; kex->kex[KEX_DH_GEX_SHA1] = kexgex_server; kex->kex[KEX_DH_GEX_SHA256] = kexgex_server; # ifdef OPENSSL_HAS_ECC kex->kex[KEX_ECDH_SHA2] = kexecdh_server; # endif #endif kex->kex[KEX_C25519_SHA256] = kexc25519_server; kex->server = 1; kex->client_version_string=client_version_string; kex->server_version_string=server_version_string; kex->load_host_public_key=&get_hostkey_public_by_type; kex->load_host_private_key=&get_hostkey_private_by_type; kex->host_key_index=&get_hostkey_index; kex->sign = sshd_hostkey_sign; dispatch_run(DISPATCH_BLOCK, &kex->done, active_state); session_id2 = kex->session_id; session_id2_len = kex->session_id_len; #ifdef DEBUG_KEXDH /* send 1st encrypted/maced/compressed message */ packet_start(SSH2_MSG_IGNORE); packet_put_cstring("markus"); packet_send(); packet_write_wait(); #endif debug("KEX done"); } /* server specific fatal cleanup */ void cleanup_exit(int i) { if (the_authctxt) { do_cleanup(the_authctxt); if (use_privsep && privsep_is_preauth && pmonitor != NULL && pmonitor->m_pid > 1) { debug("Killing privsep child %d", pmonitor->m_pid); if (kill(pmonitor->m_pid, SIGKILL) != 0 && errno != ESRCH) error("%s: kill(%d): %s", __func__, pmonitor->m_pid, strerror(errno)); } } #ifdef SSH_AUDIT_EVENTS /* done after do_cleanup so it can cancel the PAM auth 'thread' */ if (!use_privsep || mm_is_monitor()) audit_event(SSH_CONNECTION_ABANDON); #endif _exit(i); } Index: projects/clang390-import/crypto/openssh/sshd_config =================================================================== --- projects/clang390-import/crypto/openssh/sshd_config (revision 305080) +++ projects/clang390-import/crypto/openssh/sshd_config (revision 305081) @@ -1,136 +1,137 @@ # $OpenBSD: sshd_config,v 1.98 2016/02/17 05:29:04 djm Exp $ # $FreeBSD$ # This is the sshd server system-wide configuration file. See # sshd_config(5) for more information. # This sshd was compiled with PATH=/usr/bin:/bin:/usr/sbin:/sbin # The strategy used for options in the default sshd_config shipped with # OpenSSH is to specify options with their default value where # possible, but leave them commented. Uncommented options override the # default value. # Note that some of FreeBSD's defaults differ from OpenBSD's, and # FreeBSD has a few additional options. #Port 22 #AddressFamily any #ListenAddress 0.0.0.0 #ListenAddress :: # The default requires explicit activation of protocol 1 #Protocol 2 # HostKey for protocol version 1 #HostKey /etc/ssh/ssh_host_key # HostKeys for protocol version 2 #HostKey /etc/ssh/ssh_host_rsa_key #HostKey /etc/ssh/ssh_host_dsa_key #HostKey /etc/ssh/ssh_host_ecdsa_key #HostKey /etc/ssh/ssh_host_ed25519_key # Lifetime and size of ephemeral version 1 server key #KeyRegenerationInterval 1h #ServerKeyBits 1024 # Ciphers and keying #RekeyLimit default none # Logging # obsoletes QuietMode and FascistLogging #SyslogFacility AUTH #LogLevel INFO # Authentication: #LoginGraceTime 2m #PermitRootLogin no #StrictModes yes #MaxAuthTries 6 #MaxSessions 10 #RSAAuthentication yes #PubkeyAuthentication yes # The default is to check both .ssh/authorized_keys and .ssh/authorized_keys2 #AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys2 #AuthorizedPrincipalsFile none #AuthorizedKeysCommand none #AuthorizedKeysCommandUser nobody # For this to work you will also need host keys in /etc/ssh/ssh_known_hosts #RhostsRSAAuthentication no # similar for protocol version 2 #HostbasedAuthentication no # Change to yes if you don't trust ~/.ssh/known_hosts for # RhostsRSAAuthentication and HostbasedAuthentication #IgnoreUserKnownHosts no # Don't read the user's ~/.rhosts and ~/.shosts files #IgnoreRhosts yes # Change to yes to enable built-in password authentication. #PasswordAuthentication no #PermitEmptyPasswords no # Change to no to disable PAM authentication #ChallengeResponseAuthentication yes # Kerberos options #KerberosAuthentication no #KerberosOrLocalPasswd yes #KerberosTicketCleanup yes #KerberosGetAFSToken no # GSSAPI options #GSSAPIAuthentication no #GSSAPICleanupCredentials yes # Set this to 'no' to disable PAM authentication, account processing, # and session processing. If this is enabled, PAM authentication will # be allowed through the ChallengeResponseAuthentication and # PasswordAuthentication. Depending on your PAM configuration, # PAM authentication via ChallengeResponseAuthentication may bypass # the setting of "PermitRootLogin without-password". # If you just want the PAM account and session checks to run without # PAM authentication, then enable this but set PasswordAuthentication # and ChallengeResponseAuthentication to 'no'. #UsePAM yes #AllowAgentForwarding yes #AllowTcpForwarding yes #GatewayPorts no #X11Forwarding yes #X11DisplayOffset 10 #X11UseLocalhost yes #PermitTTY yes #PrintMotd yes #PrintLastLog yes #TCPKeepAlive yes #UseLogin no #UsePrivilegeSeparation sandbox #PermitUserEnvironment no #Compression delayed #ClientAliveInterval 0 #ClientAliveCountMax 3 #UseDNS yes #PidFile /var/run/sshd.pid #MaxStartups 10:30:100 #PermitTunnel no #ChrootDirectory none +#UseBlacklist no #VersionAddendum FreeBSD-20160310 # no default banner path #Banner none # override default of no subsystems Subsystem sftp /usr/libexec/sftp-server # Example of overriding settings on a per-user basis #Match User anoncvs # X11Forwarding no # AllowTcpForwarding no # PermitTTY no # ForceCommand cvs server Index: projects/clang390-import/crypto/openssh/sshd_config.5 =================================================================== --- projects/clang390-import/crypto/openssh/sshd_config.5 (revision 305080) +++ projects/clang390-import/crypto/openssh/sshd_config.5 (revision 305081) @@ -1,1767 +1,1776 @@ .\" .\" Author: Tatu Ylonen .\" Copyright (c) 1995 Tatu Ylonen , Espoo, Finland .\" All rights reserved .\" .\" As far as I am concerned, the code I have written for this software .\" can be used freely for any purpose. Any derived versions of this .\" software must be clearly marked as such, and if the derived work is .\" incompatible with the protocol description in the RFC file, it must be .\" called by a name other than "ssh" or "Secure Shell". .\" .\" Copyright (c) 1999,2000 Markus Friedl. All rights reserved. .\" Copyright (c) 1999 Aaron Campbell. All rights reserved. .\" Copyright (c) 1999 Theo de Raadt. All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR .\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES .\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. .\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, .\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT .\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .\" $OpenBSD: sshd_config.5,v 1.220 2016/02/17 08:57:34 djm Exp $ .\" $FreeBSD$ .Dd $Mdocdate: February 17 2016 $ .Dt SSHD_CONFIG 5 .Os .Sh NAME .Nm sshd_config .Nd OpenSSH SSH daemon configuration file .Sh SYNOPSIS .Nm /etc/ssh/sshd_config .Sh DESCRIPTION .Xr sshd 8 reads configuration data from .Pa /etc/ssh/sshd_config (or the file specified with .Fl f on the command line). The file contains keyword-argument pairs, one per line. Lines starting with .Ql # and empty lines are interpreted as comments. Arguments may optionally be enclosed in double quotes .Pq \&" in order to represent arguments containing spaces. .Pp The possible keywords and their meanings are as follows (note that keywords are case-insensitive and arguments are case-sensitive): .Bl -tag -width Ds .It Cm AcceptEnv Specifies what environment variables sent by the client will be copied into the session's .Xr environ 7 . See .Cm SendEnv in .Xr ssh_config 5 for how to configure the client. The .Ev TERM environment variable is always sent whenever the client requests a pseudo-terminal as it is required by the protocol. Variables are specified by name, which may contain the wildcard characters .Ql * and .Ql \&? . Multiple environment variables may be separated by whitespace or spread across multiple .Cm AcceptEnv directives. Be warned that some environment variables could be used to bypass restricted user environments. For this reason, care should be taken in the use of this directive. The default is not to accept any environment variables. .It Cm AddressFamily Specifies which address family should be used by .Xr sshd 8 . Valid arguments are .Dq any , .Dq inet (use IPv4 only), or .Dq inet6 (use IPv6 only). The default is .Dq any . .It Cm AllowAgentForwarding Specifies whether .Xr ssh-agent 1 forwarding is permitted. The default is .Dq yes . Note that disabling agent forwarding does not improve security unless users are also denied shell access, as they can always install their own forwarders. .It Cm AllowGroups This keyword can be followed by a list of group name patterns, separated by spaces. If specified, login is allowed only for users whose primary group or supplementary group list matches one of the patterns. Only group names are valid; a numerical group ID is not recognized. By default, login is allowed for all groups. The allow/deny directives are processed in the following order: .Cm DenyUsers , .Cm AllowUsers , .Cm DenyGroups , and finally .Cm AllowGroups . .Pp See PATTERNS in .Xr ssh_config 5 for more information on patterns. .It Cm AllowTcpForwarding Specifies whether TCP forwarding is permitted. The available options are .Dq yes or .Dq all to allow TCP forwarding, .Dq no to prevent all TCP forwarding, .Dq local to allow local (from the perspective of .Xr ssh 1 ) forwarding only or .Dq remote to allow remote forwarding only. The default is .Dq yes . Note that disabling TCP forwarding does not improve security unless users are also denied shell access, as they can always install their own forwarders. .It Cm AllowStreamLocalForwarding Specifies whether StreamLocal (Unix-domain socket) forwarding is permitted. The available options are .Dq yes or .Dq all to allow StreamLocal forwarding, .Dq no to prevent all StreamLocal forwarding, .Dq local to allow local (from the perspective of .Xr ssh 1 ) forwarding only or .Dq remote to allow remote forwarding only. The default is .Dq yes . Note that disabling StreamLocal forwarding does not improve security unless users are also denied shell access, as they can always install their own forwarders. .It Cm AllowUsers This keyword can be followed by a list of user name patterns, separated by spaces. If specified, login is allowed only for user names that match one of the patterns. Only user names are valid; a numerical user ID is not recognized. By default, login is allowed for all users. If the pattern takes the form USER@HOST then USER and HOST are separately checked, restricting logins to particular users from particular hosts. The allow/deny directives are processed in the following order: .Cm DenyUsers , .Cm AllowUsers , .Cm DenyGroups , and finally .Cm AllowGroups . .Pp See PATTERNS in .Xr ssh_config 5 for more information on patterns. .It Cm AuthenticationMethods Specifies the authentication methods that must be successfully completed for a user to be granted access. This option must be followed by one or more comma-separated lists of authentication method names. Successful authentication requires completion of every method in at least one of these lists. .Pp For example, an argument of .Dq publickey,password publickey,keyboard-interactive would require the user to complete public key authentication, followed by either password or keyboard interactive authentication. Only methods that are next in one or more lists are offered at each stage, so for this example, it would not be possible to attempt password or keyboard-interactive authentication before public key. .Pp For keyboard interactive authentication it is also possible to restrict authentication to a specific device by appending a colon followed by the device identifier .Dq bsdauth , .Dq pam , or .Dq skey , depending on the server configuration. For example, .Dq keyboard-interactive:bsdauth would restrict keyboard interactive authentication to the .Dq bsdauth device. .Pp If the .Dq publickey method is listed more than once, .Xr sshd 8 verifies that keys that have been used successfully are not reused for subsequent authentications. For example, an .Cm AuthenticationMethods of .Dq publickey,publickey will require successful authentication using two different public keys. .Pp This option will yield a fatal error if enabled if protocol 1 is also enabled. Note that each authentication method listed should also be explicitly enabled in the configuration. The default is not to require multiple authentication; successful completion of a single authentication method is sufficient. .It Cm AuthorizedKeysCommand Specifies a program to be used to look up the user's public keys. The program must be owned by root, not writable by group or others and specified by an absolute path. .Pp Arguments to .Cm AuthorizedKeysCommand may be provided using the following tokens, which will be expanded at runtime: %% is replaced by a literal '%', %u is replaced by the username being authenticated, %h is replaced by the home directory of the user being authenticated, %t is replaced with the key type offered for authentication, %f is replaced with the fingerprint of the key, and %k is replaced with the key being offered for authentication. If no arguments are specified then the username of the target user will be supplied. .Pp The program should produce on standard output zero or more lines of authorized_keys output (see AUTHORIZED_KEYS in .Xr sshd 8 ) . If a key supplied by AuthorizedKeysCommand does not successfully authenticate and authorize the user then public key authentication continues using the usual .Cm AuthorizedKeysFile files. By default, no AuthorizedKeysCommand is run. .It Cm AuthorizedKeysCommandUser Specifies the user under whose account the AuthorizedKeysCommand is run. It is recommended to use a dedicated user that has no other role on the host than running authorized keys commands. If .Cm AuthorizedKeysCommand is specified but .Cm AuthorizedKeysCommandUser is not, then .Xr sshd 8 will refuse to start. .It Cm AuthorizedKeysFile Specifies the file that contains the public keys that can be used for user authentication. The format is described in the AUTHORIZED_KEYS FILE FORMAT section of .Xr sshd 8 . .Cm AuthorizedKeysFile may contain tokens of the form %T which are substituted during connection setup. The following tokens are defined: %% is replaced by a literal '%', %h is replaced by the home directory of the user being authenticated, and %u is replaced by the username of that user. After expansion, .Cm AuthorizedKeysFile is taken to be an absolute path or one relative to the user's home directory. Multiple files may be listed, separated by whitespace. Alternately this option may be set to .Dq none to skip checking for user keys in files. The default is .Dq .ssh/authorized_keys .ssh/authorized_keys2 . .It Cm AuthorizedPrincipalsCommand Specifies a program to be used to generate the list of allowed certificate principals as per .Cm AuthorizedPrincipalsFile . The program must be owned by root, not writable by group or others and specified by an absolute path. .Pp Arguments to .Cm AuthorizedPrincipalsCommand may be provided using the following tokens, which will be expanded at runtime: %% is replaced by a literal '%', %u is replaced by the username being authenticated and %h is replaced by the home directory of the user being authenticated. .Pp The program should produce on standard output zero or more lines of .Cm AuthorizedPrincipalsFile output. If either .Cm AuthorizedPrincipalsCommand or .Cm AuthorizedPrincipalsFile is specified, then certificates offered by the client for authentication must contain a principal that is listed. By default, no AuthorizedPrincipalsCommand is run. .It Cm AuthorizedPrincipalsCommandUser Specifies the user under whose account the AuthorizedPrincipalsCommand is run. It is recommended to use a dedicated user that has no other role on the host than running authorized principals commands. If .Cm AuthorizedPrincipalsCommand is specified but .Cm AuthorizedPrincipalsCommandUser is not, then .Xr sshd 8 will refuse to start. .It Cm AuthorizedPrincipalsFile Specifies a file that lists principal names that are accepted for certificate authentication. When using certificates signed by a key listed in .Cm TrustedUserCAKeys , this file lists names, one of which must appear in the certificate for it to be accepted for authentication. Names are listed one per line preceded by key options (as described in AUTHORIZED_KEYS FILE FORMAT in .Xr sshd 8 ) . Empty lines and comments starting with .Ql # are ignored. .Pp .Cm AuthorizedPrincipalsFile may contain tokens of the form %T which are substituted during connection setup. The following tokens are defined: %% is replaced by a literal '%', %h is replaced by the home directory of the user being authenticated, and %u is replaced by the username of that user. After expansion, .Cm AuthorizedPrincipalsFile is taken to be an absolute path or one relative to the user's home directory. .Pp The default is .Dq none , i.e. not to use a principals file \(en in this case, the username of the user must appear in a certificate's principals list for it to be accepted. Note that .Cm AuthorizedPrincipalsFile is only used when authentication proceeds using a CA listed in .Cm TrustedUserCAKeys and is not consulted for certification authorities trusted via .Pa ~/.ssh/authorized_keys , though the .Cm principals= key option offers a similar facility (see .Xr sshd 8 for details). .It Cm Banner The contents of the specified file are sent to the remote user before authentication is allowed. If the argument is .Dq none then no banner is displayed. By default, no banner is displayed. .It Cm ChallengeResponseAuthentication Specifies whether challenge-response authentication is allowed (e.g. via PAM or through authentication styles supported in .Xr login.conf 5 ) The default is .Dq yes . .It Cm ChrootDirectory Specifies the pathname of a directory to .Xr chroot 2 to after authentication. At session startup .Xr sshd 8 checks that all components of the pathname are root-owned directories which are not writable by any other user or group. After the chroot, .Xr sshd 8 changes the working directory to the user's home directory. .Pp The pathname may contain the following tokens that are expanded at runtime once the connecting user has been authenticated: %% is replaced by a literal '%', %h is replaced by the home directory of the user being authenticated, and %u is replaced by the username of that user. .Pp The .Cm ChrootDirectory must contain the necessary files and directories to support the user's session. For an interactive session this requires at least a shell, typically .Xr sh 1 , and basic .Pa /dev nodes such as .Xr null 4 , .Xr zero 4 , .Xr stdin 4 , .Xr stdout 4 , .Xr stderr 4 , and .Xr tty 4 devices. For file transfer sessions using .Dq sftp , no additional configuration of the environment is necessary if the in-process sftp server is used, though sessions which use logging may require .Pa /dev/log inside the chroot directory on some operating systems (see .Xr sftp-server 8 for details). .Pp For safety, it is very important that the directory hierarchy be prevented from modification by other processes on the system (especially those outside the jail). Misconfiguration can lead to unsafe environments which .Xr sshd 8 cannot detect. .Pp The default is .Dq none , indicating not to .Xr chroot 2 . .It Cm Ciphers Specifies the ciphers allowed. Multiple ciphers must be comma-separated. If the specified value begins with a .Sq + character, then the specified ciphers will be appended to the default set instead of replacing them. .Pp The supported ciphers are: .Pp .Bl -item -compact -offset indent .It 3des-cbc .It aes128-cbc .It aes192-cbc .It aes256-cbc .It aes128-ctr .It aes192-ctr .It aes256-ctr .It aes128-gcm@openssh.com .It aes256-gcm@openssh.com .It arcfour .It arcfour128 .It arcfour256 .It blowfish-cbc .It cast128-cbc .It chacha20-poly1305@openssh.com .El .Pp The default is: .Bd -literal -offset indent chacha20-poly1305@openssh.com, aes128-ctr,aes192-ctr,aes256-ctr, aes128-gcm@openssh.com,aes256-gcm@openssh.com, aes128-cbc,aes192-cbc,aes256-cbc .Ed .Pp The list of available ciphers may also be obtained using the .Fl Q option of .Xr ssh 1 with an argument of .Dq cipher . .It Cm ClientAliveCountMax Sets the number of client alive messages (see below) which may be sent without .Xr sshd 8 receiving any messages back from the client. If this threshold is reached while client alive messages are being sent, sshd will disconnect the client, terminating the session. It is important to note that the use of client alive messages is very different from .Cm TCPKeepAlive (below). The client alive messages are sent through the encrypted channel and therefore will not be spoofable. The TCP keepalive option enabled by .Cm TCPKeepAlive is spoofable. The client alive mechanism is valuable when the client or server depend on knowing when a connection has become inactive. .Pp The default value is 3. If .Cm ClientAliveInterval (see below) is set to 15, and .Cm ClientAliveCountMax is left at the default, unresponsive SSH clients will be disconnected after approximately 45 seconds. .It Cm ClientAliveInterval Sets a timeout interval in seconds after which if no data has been received from the client, .Xr sshd 8 will send a message through the encrypted channel to request a response from the client. The default is 0, indicating that these messages will not be sent to the client. .It Cm Compression Specifies whether compression is allowed, or delayed until the user has authenticated successfully. The argument must be .Dq yes , .Dq delayed , or .Dq no . The default is .Dq delayed . .It Cm DenyGroups This keyword can be followed by a list of group name patterns, separated by spaces. Login is disallowed for users whose primary group or supplementary group list matches one of the patterns. Only group names are valid; a numerical group ID is not recognized. By default, login is allowed for all groups. The allow/deny directives are processed in the following order: .Cm DenyUsers , .Cm AllowUsers , .Cm DenyGroups , and finally .Cm AllowGroups . .Pp See PATTERNS in .Xr ssh_config 5 for more information on patterns. .It Cm DenyUsers This keyword can be followed by a list of user name patterns, separated by spaces. Login is disallowed for user names that match one of the patterns. Only user names are valid; a numerical user ID is not recognized. By default, login is allowed for all users. If the pattern takes the form USER@HOST then USER and HOST are separately checked, restricting logins to particular users from particular hosts. The allow/deny directives are processed in the following order: .Cm DenyUsers , .Cm AllowUsers , .Cm DenyGroups , and finally .Cm AllowGroups . .Pp See PATTERNS in .Xr ssh_config 5 for more information on patterns. .It Cm FingerprintHash Specifies the hash algorithm used when logging key fingerprints. Valid options are: .Dq md5 and .Dq sha256 . The default is .Dq sha256 . .It Cm ForceCommand Forces the execution of the command specified by .Cm ForceCommand , ignoring any command supplied by the client and .Pa ~/.ssh/rc if present. The command is invoked by using the user's login shell with the -c option. This applies to shell, command, or subsystem execution. It is most useful inside a .Cm Match block. The command originally supplied by the client is available in the .Ev SSH_ORIGINAL_COMMAND environment variable. Specifying a command of .Dq internal-sftp will force the use of an in-process sftp server that requires no support files when used with .Cm ChrootDirectory . The default is .Dq none . .It Cm GatewayPorts Specifies whether remote hosts are allowed to connect to ports forwarded for the client. By default, .Xr sshd 8 binds remote port forwardings to the loopback address. This prevents other remote hosts from connecting to forwarded ports. .Cm GatewayPorts can be used to specify that sshd should allow remote port forwardings to bind to non-loopback addresses, thus allowing other hosts to connect. The argument may be .Dq no to force remote port forwardings to be available to the local host only, .Dq yes to force remote port forwardings to bind to the wildcard address, or .Dq clientspecified to allow the client to select the address to which the forwarding is bound. The default is .Dq no . .It Cm GSSAPIAuthentication Specifies whether user authentication based on GSSAPI is allowed. The default is .Dq no . .It Cm GSSAPICleanupCredentials Specifies whether to automatically destroy the user's credentials cache on logout. The default is .Dq yes . .It Cm GSSAPIStrictAcceptorCheck Determines whether to be strict about the identity of the GSSAPI acceptor a client authenticates against. If set to .Dq yes then the client must authenticate against the .Pa host service on the current hostname. If set to .Dq no then the client may authenticate against any service key stored in the machine's default store. This facility is provided to assist with operation on multi homed machines. The default is .Dq yes . .It Cm HostbasedAcceptedKeyTypes Specifies the key types that will be accepted for hostbased authentication as a comma-separated pattern list. Alternately if the specified value begins with a .Sq + character, then the specified key types will be appended to the default set instead of replacing them. The default for this option is: .Bd -literal -offset 3n ecdsa-sha2-nistp256-cert-v01@openssh.com, ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, ssh-ed25519,ssh-rsa .Ed .Pp The .Fl Q option of .Xr ssh 1 may be used to list supported key types. .It Cm HostbasedAuthentication Specifies whether rhosts or /etc/hosts.equiv authentication together with successful public key client host authentication is allowed (host-based authentication). The default is .Dq no . .It Cm HostbasedUsesNameFromPacketOnly Specifies whether or not the server will attempt to perform a reverse name lookup when matching the name in the .Pa ~/.shosts , .Pa ~/.rhosts , and .Pa /etc/hosts.equiv files during .Cm HostbasedAuthentication . A setting of .Dq yes means that .Xr sshd 8 uses the name supplied by the client rather than attempting to resolve the name from the TCP connection itself. The default is .Dq no . .It Cm HostCertificate Specifies a file containing a public host certificate. The certificate's public key must match a private host key already specified by .Cm HostKey . The default behaviour of .Xr sshd 8 is not to load any certificates. .It Cm HostKey Specifies a file containing a private host key used by SSH. The default is .Pa /etc/ssh/ssh_host_key for protocol version 1, and .Pa /etc/ssh/ssh_host_dsa_key , .Pa /etc/ssh/ssh_host_ecdsa_key , .Pa /etc/ssh/ssh_host_ed25519_key and .Pa /etc/ssh/ssh_host_rsa_key for protocol version 2. .Pp Note that .Xr sshd 8 will refuse to use a file if it is group/world-accessible and that the .Cm HostKeyAlgorithms option restricts which of the keys are actually used by .Xr sshd 8 . .Pp It is possible to have multiple host key files. .Dq rsa1 keys are used for version 1 and .Dq dsa , .Dq ecdsa , .Dq ed25519 or .Dq rsa are used for version 2 of the SSH protocol. It is also possible to specify public host key files instead. In this case operations on the private key will be delegated to an .Xr ssh-agent 1 . .It Cm HostKeyAgent Identifies the UNIX-domain socket used to communicate with an agent that has access to the private host keys. If .Dq SSH_AUTH_SOCK is specified, the location of the socket will be read from the .Ev SSH_AUTH_SOCK environment variable. .It Cm HostKeyAlgorithms Specifies the host key algorithms that the server offers. The default for this option is: .Bd -literal -offset 3n ecdsa-sha2-nistp256-cert-v01@openssh.com, ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, ssh-ed25519,ssh-rsa .Ed .Pp The list of available key types may also be obtained using the .Fl Q option of .Xr ssh 1 with an argument of .Dq key . .It Cm IgnoreRhosts Specifies that .Pa .rhosts and .Pa .shosts files will not be used in .Cm RhostsRSAAuthentication or .Cm HostbasedAuthentication . .Pp .Pa /etc/hosts.equiv and .Pa /etc/ssh/shosts.equiv are still used. The default is .Dq yes . .It Cm IgnoreUserKnownHosts Specifies whether .Xr sshd 8 should ignore the user's .Pa ~/.ssh/known_hosts during .Cm RhostsRSAAuthentication or .Cm HostbasedAuthentication . The default is .Dq no . .It Cm IPQoS Specifies the IPv4 type-of-service or DSCP class for the connection. Accepted values are .Dq af11 , .Dq af12 , .Dq af13 , .Dq af21 , .Dq af22 , .Dq af23 , .Dq af31 , .Dq af32 , .Dq af33 , .Dq af41 , .Dq af42 , .Dq af43 , .Dq cs0 , .Dq cs1 , .Dq cs2 , .Dq cs3 , .Dq cs4 , .Dq cs5 , .Dq cs6 , .Dq cs7 , .Dq ef , .Dq lowdelay , .Dq throughput , .Dq reliability , or a numeric value. This option may take one or two arguments, separated by whitespace. If one argument is specified, it is used as the packet class unconditionally. If two values are specified, the first is automatically selected for interactive sessions and the second for non-interactive sessions. The default is .Dq lowdelay for interactive sessions and .Dq throughput for non-interactive sessions. .It Cm KbdInteractiveAuthentication Specifies whether to allow keyboard-interactive authentication. The argument to this keyword must be .Dq yes or .Dq no . The default is to use whatever value .Cm ChallengeResponseAuthentication is set to (by default .Dq yes ) . .It Cm KerberosAuthentication Specifies whether the password provided by the user for .Cm PasswordAuthentication will be validated through the Kerberos KDC. To use this option, the server needs a Kerberos servtab which allows the verification of the KDC's identity. The default is .Dq no . .It Cm KerberosGetAFSToken If AFS is active and the user has a Kerberos 5 TGT, attempt to acquire an AFS token before accessing the user's home directory. The default is .Dq no . .It Cm KerberosOrLocalPasswd If password authentication through Kerberos fails then the password will be validated via any additional local mechanism such as .Pa /etc/passwd . The default is .Dq yes . .It Cm KerberosTicketCleanup Specifies whether to automatically destroy the user's ticket cache file on logout. The default is .Dq yes . .It Cm KexAlgorithms Specifies the available KEX (Key Exchange) algorithms. Multiple algorithms must be comma-separated. Alternately if the specified value begins with a .Sq + character, then the specified methods will be appended to the default set instead of replacing them. The supported algorithms are: .Pp .Bl -item -compact -offset indent .It curve25519-sha256@libssh.org .It diffie-hellman-group1-sha1 .It diffie-hellman-group14-sha1 .It diffie-hellman-group-exchange-sha1 .It diffie-hellman-group-exchange-sha256 .It ecdh-sha2-nistp256 .It ecdh-sha2-nistp384 .It ecdh-sha2-nistp521 .El .Pp The default is: .Bd -literal -offset indent curve25519-sha256@libssh.org, ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group14-sha1 .Ed .Pp The list of available key exchange algorithms may also be obtained using the .Fl Q option of .Xr ssh 1 with an argument of .Dq kex . .It Cm KeyRegenerationInterval In protocol version 1, the ephemeral server key is automatically regenerated after this many seconds (if it has been used). The purpose of regeneration is to prevent decrypting captured sessions by later breaking into the machine and stealing the keys. The key is never stored anywhere. If the value is 0, the key is never regenerated. The default is 3600 (seconds). .It Cm ListenAddress Specifies the local addresses .Xr sshd 8 should listen on. The following forms may be used: .Pp .Bl -item -offset indent -compact .It .Cm ListenAddress .Sm off .Ar host | Ar IPv4_addr | Ar IPv6_addr .Sm on .It .Cm ListenAddress .Sm off .Ar host | Ar IPv4_addr : Ar port .Sm on .It .Cm ListenAddress .Sm off .Oo .Ar host | Ar IPv6_addr Oc : Ar port .Sm on .El .Pp If .Ar port is not specified, sshd will listen on the address and all .Cm Port options specified. The default is to listen on all local addresses. Multiple .Cm ListenAddress options are permitted. .It Cm LoginGraceTime The server disconnects after this time if the user has not successfully logged in. If the value is 0, there is no time limit. The default is 120 seconds. .It Cm LogLevel Gives the verbosity level that is used when logging messages from .Xr sshd 8 . The possible values are: QUIET, FATAL, ERROR, INFO, VERBOSE, DEBUG, DEBUG1, DEBUG2, and DEBUG3. The default is INFO. DEBUG and DEBUG1 are equivalent. DEBUG2 and DEBUG3 each specify higher levels of debugging output. Logging with a DEBUG level violates the privacy of users and is not recommended. .It Cm MACs Specifies the available MAC (message authentication code) algorithms. The MAC algorithm is used for data integrity protection. Multiple algorithms must be comma-separated. If the specified value begins with a .Sq + character, then the specified algorithms will be appended to the default set instead of replacing them. .Pp The algorithms that contain .Dq -etm calculate the MAC after encryption (encrypt-then-mac). These are considered safer and their use recommended. The supported MACs are: .Pp .Bl -item -compact -offset indent .It hmac-md5 .It hmac-md5-96 .It hmac-ripemd160 .It hmac-sha1 .It hmac-sha1-96 .It hmac-sha2-256 .It hmac-sha2-512 .It umac-64@openssh.com .It umac-128@openssh.com .It hmac-md5-etm@openssh.com .It hmac-md5-96-etm@openssh.com .It hmac-ripemd160-etm@openssh.com .It hmac-sha1-etm@openssh.com .It hmac-sha1-96-etm@openssh.com .It hmac-sha2-256-etm@openssh.com .It hmac-sha2-512-etm@openssh.com .It umac-64-etm@openssh.com .It umac-128-etm@openssh.com .El .Pp The default is: .Bd -literal -offset indent umac-64-etm@openssh.com,umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com,umac-128@openssh.com, hmac-sha2-256,hmac-sha2-512,hmac-sha1 .Ed .Pp The list of available MAC algorithms may also be obtained using the .Fl Q option of .Xr ssh 1 with an argument of .Dq mac . .It Cm Match Introduces a conditional block. If all of the criteria on the .Cm Match line are satisfied, the keywords on the following lines override those set in the global section of the config file, until either another .Cm Match line or the end of the file. If a keyword appears in multiple .Cm Match blocks that are satisfied, only the first instance of the keyword is applied. .Pp The arguments to .Cm Match are one or more criteria-pattern pairs or the single token .Cm All which matches all criteria. The available criteria are .Cm User , .Cm Group , .Cm Host , .Cm LocalAddress , .Cm LocalPort , and .Cm Address . The match patterns may consist of single entries or comma-separated lists and may use the wildcard and negation operators described in the PATTERNS section of .Xr ssh_config 5 . .Pp The patterns in an .Cm Address criteria may additionally contain addresses to match in CIDR address/masklen format, e.g.\& .Dq 192.0.2.0/24 or .Dq 3ffe:ffff::/32 . Note that the mask length provided must be consistent with the address - it is an error to specify a mask length that is too long for the address or one with bits set in this host portion of the address. For example, .Dq 192.0.2.0/33 and .Dq 192.0.2.0/8 respectively. .Pp Only a subset of keywords may be used on the lines following a .Cm Match keyword. Available keywords are .Cm AcceptEnv , .Cm AllowAgentForwarding , .Cm AllowGroups , .Cm AllowStreamLocalForwarding , .Cm AllowTcpForwarding , .Cm AllowUsers , .Cm AuthenticationMethods , .Cm AuthorizedKeysCommand , .Cm AuthorizedKeysCommandUser , .Cm AuthorizedKeysFile , .Cm AuthorizedPrincipalsCommand , .Cm AuthorizedPrincipalsCommandUser , .Cm AuthorizedPrincipalsFile , .Cm Banner , .Cm ChrootDirectory , .Cm DenyGroups , .Cm DenyUsers , .Cm ForceCommand , .Cm GatewayPorts , .Cm GSSAPIAuthentication , .Cm HostbasedAcceptedKeyTypes , .Cm HostbasedAuthentication , .Cm HostbasedUsesNameFromPacketOnly , .Cm IPQoS , .Cm KbdInteractiveAuthentication , .Cm KerberosAuthentication , .Cm MaxAuthTries , .Cm MaxSessions , .Cm PasswordAuthentication , .Cm PermitEmptyPasswords , .Cm PermitOpen , .Cm PermitRootLogin , .Cm PermitTTY , .Cm PermitTunnel , .Cm PermitUserRC , .Cm PubkeyAcceptedKeyTypes , .Cm PubkeyAuthentication , .Cm RekeyLimit , .Cm RevokedKeys , .Cm RhostsRSAAuthentication , .Cm RSAAuthentication , .Cm StreamLocalBindMask , .Cm StreamLocalBindUnlink , .Cm TrustedUserCAKeys , .Cm X11DisplayOffset , .Cm X11Forwarding and .Cm X11UseLocalHost . .It Cm MaxAuthTries Specifies the maximum number of authentication attempts permitted per connection. Once the number of failures reaches half this value, additional failures are logged. The default is 6. .It Cm MaxSessions Specifies the maximum number of open shell, login or subsystem (e.g. sftp) sessions permitted per network connection. Multiple sessions may be established by clients that support connection multiplexing. Setting .Cm MaxSessions to 1 will effectively disable session multiplexing, whereas setting it to 0 will prevent all shell, login and subsystem sessions while still permitting forwarding. The default is 10. .It Cm MaxStartups Specifies the maximum number of concurrent unauthenticated connections to the SSH daemon. Additional connections will be dropped until authentication succeeds or the .Cm LoginGraceTime expires for a connection. The default is 10:30:100. .Pp Alternatively, random early drop can be enabled by specifying the three colon separated values .Dq start:rate:full (e.g. "10:30:60"). .Xr sshd 8 will refuse connection attempts with a probability of .Dq rate/100 (30%) if there are currently .Dq start (10) unauthenticated connections. The probability increases linearly and all connection attempts are refused if the number of unauthenticated connections reaches .Dq full (60). .It Cm PasswordAuthentication Specifies whether password authentication is allowed. See also .Cm UsePAM . The default is .Dq no . .It Cm PermitEmptyPasswords When password authentication is allowed, it specifies whether the server allows login to accounts with empty password strings. The default is .Dq no . .It Cm PermitOpen Specifies the destinations to which TCP port forwarding is permitted. The forwarding specification must be one of the following forms: .Pp .Bl -item -offset indent -compact .It .Cm PermitOpen .Sm off .Ar host : port .Sm on .It .Cm PermitOpen .Sm off .Ar IPv4_addr : port .Sm on .It .Cm PermitOpen .Sm off .Ar \&[ IPv6_addr \&] : port .Sm on .El .Pp Multiple forwards may be specified by separating them with whitespace. An argument of .Dq any can be used to remove all restrictions and permit any forwarding requests. An argument of .Dq none can be used to prohibit all forwarding requests. By default all port forwarding requests are permitted. .It Cm PermitRootLogin Specifies whether root can log in using .Xr ssh 1 . The argument must be .Dq yes , .Dq prohibit-password , .Dq without-password , .Dq forced-commands-only , or .Dq no . The default is .Dq no . Note that if .Cm ChallengeResponseAuthentication is .Dq yes , the root user may be allowed in with its password even if .Cm PermitRootLogin is set to .Dq without-password . .Pp If this option is set to .Dq prohibit-password or .Dq without-password , password and keyboard-interactive authentication are disabled for root. .Pp If this option is set to .Dq forced-commands-only , root login with public key authentication will be allowed, but only if the .Ar command option has been specified (which may be useful for taking remote backups even if root login is normally not allowed). All other authentication methods are disabled for root. .Pp If this option is set to .Dq no , root is not allowed to log in. .It Cm PermitTunnel Specifies whether .Xr tun 4 device forwarding is allowed. The argument must be .Dq yes , .Dq point-to-point (layer 3), .Dq ethernet (layer 2), or .Dq no . Specifying .Dq yes permits both .Dq point-to-point and .Dq ethernet . The default is .Dq no . .Pp Independent of this setting, the permissions of the selected .Xr tun 4 device must allow access to the user. .It Cm PermitTTY Specifies whether .Xr pty 4 allocation is permitted. The default is .Dq yes . .It Cm PermitUserEnvironment Specifies whether .Pa ~/.ssh/environment and .Cm environment= options in .Pa ~/.ssh/authorized_keys are processed by .Xr sshd 8 . The default is .Dq no . Enabling environment processing may enable users to bypass access restrictions in some configurations using mechanisms such as .Ev LD_PRELOAD . .It Cm PermitUserRC Specifies whether any .Pa ~/.ssh/rc file is executed. The default is .Dq yes . .It Cm PidFile Specifies the file that contains the process ID of the SSH daemon, or .Dq none to not write one. The default is .Pa /var/run/sshd.pid . .It Cm Port Specifies the port number that .Xr sshd 8 listens on. The default is 22. Multiple options of this type are permitted. See also .Cm ListenAddress . .It Cm PrintLastLog Specifies whether .Xr sshd 8 should print the date and time of the last user login when a user logs in interactively. The default is .Dq yes . .It Cm PrintMotd Specifies whether .Xr sshd 8 should print .Pa /etc/motd when a user logs in interactively. (On some systems it is also printed by the shell, .Pa /etc/profile , or equivalent.) The default is .Dq yes . .It Cm Protocol Specifies the protocol versions .Xr sshd 8 supports. The possible values are .Sq 1 and .Sq 2 . Multiple versions must be comma-separated. The default is .Sq 2 . Protocol 1 suffers from a number of cryptographic weaknesses and should not be used. It is only offered to support legacy devices. .Pp Note that the order of the protocol list does not indicate preference, because the client selects among multiple protocol versions offered by the server. Specifying .Dq 2,1 is identical to .Dq 1,2 . .It Cm PubkeyAcceptedKeyTypes Specifies the key types that will be accepted for public key authentication as a comma-separated pattern list. Alternately if the specified value begins with a .Sq + character, then the specified key types will be appended to the default set instead of replacing them. The default for this option is: .Bd -literal -offset 3n ecdsa-sha2-nistp256-cert-v01@openssh.com, ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, ssh-ed25519,ssh-rsa .Ed .Pp The .Fl Q option of .Xr ssh 1 may be used to list supported key types. .It Cm PubkeyAuthentication Specifies whether public key authentication is allowed. The default is .Dq yes . .It Cm RekeyLimit Specifies the maximum amount of data that may be transmitted before the session key is renegotiated, optionally followed a maximum amount of time that may pass before the session key is renegotiated. The first argument is specified in bytes and may have a suffix of .Sq K , .Sq M , or .Sq G to indicate Kilobytes, Megabytes, or Gigabytes, respectively. The default is between .Sq 1G and .Sq 4G , depending on the cipher. The optional second value is specified in seconds and may use any of the units documented in the .Sx TIME FORMATS section. The default value for .Cm RekeyLimit is .Dq default none , which means that rekeying is performed after the cipher's default amount of data has been sent or received and no time based rekeying is done. .It Cm RevokedKeys Specifies revoked public keys file, or .Dq none to not use one. Keys listed in this file will be refused for public key authentication. Note that if this file is not readable, then public key authentication will be refused for all users. Keys may be specified as a text file, listing one public key per line, or as an OpenSSH Key Revocation List (KRL) as generated by .Xr ssh-keygen 1 . For more information on KRLs, see the KEY REVOCATION LISTS section in .Xr ssh-keygen 1 . .It Cm RhostsRSAAuthentication Specifies whether rhosts or .Pa /etc/hosts.equiv authentication together with successful RSA host authentication is allowed. The default is .Dq no . This option applies to protocol version 1 only. .It Cm RSAAuthentication Specifies whether pure RSA authentication is allowed. The default is .Dq yes . This option applies to protocol version 1 only. .It Cm ServerKeyBits Defines the number of bits in the ephemeral protocol version 1 server key. The default and minimum value is 1024. .It Cm StreamLocalBindMask Sets the octal file creation mode mask .Pq umask used when creating a Unix-domain socket file for local or remote port forwarding. This option is only used for port forwarding to a Unix-domain socket file. .Pp The default value is 0177, which creates a Unix-domain socket file that is readable and writable only by the owner. Note that not all operating systems honor the file mode on Unix-domain socket files. .It Cm StreamLocalBindUnlink Specifies whether to remove an existing Unix-domain socket file for local or remote port forwarding before creating a new one. If the socket file already exists and .Cm StreamLocalBindUnlink is not enabled, .Nm sshd will be unable to forward the port to the Unix-domain socket file. This option is only used for port forwarding to a Unix-domain socket file. .Pp The argument must be .Dq yes or .Dq no . The default is .Dq no . .It Cm StrictModes Specifies whether .Xr sshd 8 should check file modes and ownership of the user's files and home directory before accepting login. This is normally desirable because novices sometimes accidentally leave their directory or files world-writable. The default is .Dq yes . Note that this does not apply to .Cm ChrootDirectory , whose permissions and ownership are checked unconditionally. .It Cm Subsystem Configures an external subsystem (e.g. file transfer daemon). Arguments should be a subsystem name and a command (with optional arguments) to execute upon subsystem request. .Pp The command .Xr sftp-server 8 implements the .Dq sftp file transfer subsystem. .Pp Alternately the name .Dq internal-sftp implements an in-process .Dq sftp server. This may simplify configurations using .Cm ChrootDirectory to force a different filesystem root on clients. .Pp By default no subsystems are defined. .It Cm SyslogFacility Gives the facility code that is used when logging messages from .Xr sshd 8 . The possible values are: DAEMON, USER, AUTH, LOCAL0, LOCAL1, LOCAL2, LOCAL3, LOCAL4, LOCAL5, LOCAL6, LOCAL7. The default is AUTH. .It Cm TCPKeepAlive Specifies whether the system should send TCP keepalive messages to the other side. If they are sent, death of the connection or crash of one of the machines will be properly noticed. However, this means that connections will die if the route is down temporarily, and some people find it annoying. On the other hand, if TCP keepalives are not sent, sessions may hang indefinitely on the server, leaving .Dq ghost users and consuming server resources. .Pp The default is .Dq yes (to send TCP keepalive messages), and the server will notice if the network goes down or the client host crashes. This avoids infinitely hanging sessions. .Pp To disable TCP keepalive messages, the value should be set to .Dq no . .It Cm TrustedUserCAKeys Specifies a file containing public keys of certificate authorities that are trusted to sign user certificates for authentication, or .Dq none to not use one. Keys are listed one per line; empty lines and comments starting with .Ql # are allowed. If a certificate is presented for authentication and has its signing CA key listed in this file, then it may be used for authentication for any user listed in the certificate's principals list. Note that certificates that lack a list of principals will not be permitted for authentication using .Cm TrustedUserCAKeys . For more details on certificates, see the CERTIFICATES section in .Xr ssh-keygen 1 . +.It Cm UseBlacklist +Specifies whether +.Xr sshd 8 +attempts to send authentication success and failure messages +to the +.Xr blacklistd 8 +daemon. +The default is +.Dq no . .It Cm UseDNS Specifies whether .Xr sshd 8 should look up the remote host name, and to check that the resolved host name for the remote IP address maps back to the very same IP address. .Pp If this option is set to .Dq no , then only addresses and not host names may be used in .Pa ~/.ssh/known_hosts .Cm from and .Nm .Cm Match .Cm Host directives. The default is .Dq yes . .It Cm UseLogin Specifies whether .Xr login 1 is used for interactive login sessions. The default is .Dq no . Note that .Xr login 1 is never used for remote command execution. Note also, that if this is enabled, .Cm X11Forwarding will be disabled because .Xr login 1 does not know how to handle .Xr xauth 1 cookies. If .Cm UsePrivilegeSeparation is specified, it will be disabled after authentication. .It Cm UsePAM Enables the Pluggable Authentication Module interface. If set to .Dq yes this will enable PAM authentication using .Cm ChallengeResponseAuthentication and .Cm PasswordAuthentication in addition to PAM account and session module processing for all authentication types. .Pp Because PAM challenge-response authentication usually serves an equivalent role to password authentication, you should disable either .Cm PasswordAuthentication or .Cm ChallengeResponseAuthentication. .Pp If .Cm UsePAM is enabled, you will not be able to run .Xr sshd 8 as a non-root user. The default is .Dq yes . .It Cm UsePrivilegeSeparation Specifies whether .Xr sshd 8 separates privileges by creating an unprivileged child process to deal with incoming network traffic. After successful authentication, another process will be created that has the privilege of the authenticated user. The goal of privilege separation is to prevent privilege escalation by containing any corruption within the unprivileged processes. The argument must be .Dq yes , .Dq no , or .Dq sandbox . If .Cm UsePrivilegeSeparation is set to .Dq sandbox then the pre-authentication unprivileged process is subject to additional restrictions. The default is .Dq sandbox . .It Cm VersionAddendum Optionally specifies additional text to append to the SSH protocol banner sent by the server upon connection. The default is .Dq FreeBSD-20160310 . The value .Dq none may be used to disable this. .It Cm X11DisplayOffset Specifies the first display number available for .Xr sshd 8 Ns 's X11 forwarding. This prevents sshd from interfering with real X11 servers. The default is 10. .It Cm X11Forwarding Specifies whether X11 forwarding is permitted. The argument must be .Dq yes or .Dq no . The default is .Dq yes . .Pp When X11 forwarding is enabled, there may be additional exposure to the server and to client displays if the .Xr sshd 8 proxy display is configured to listen on the wildcard address (see .Cm X11UseLocalhost below), though this is not the default. Additionally, the authentication spoofing and authentication data verification and substitution occur on the client side. The security risk of using X11 forwarding is that the client's X11 display server may be exposed to attack when the SSH client requests forwarding (see the warnings for .Cm ForwardX11 in .Xr ssh_config 5 ) . A system administrator may have a stance in which they want to protect clients that may expose themselves to attack by unwittingly requesting X11 forwarding, which can warrant a .Dq no setting. .Pp Note that disabling X11 forwarding does not prevent users from forwarding X11 traffic, as users can always install their own forwarders. X11 forwarding is automatically disabled if .Cm UseLogin is enabled. .It Cm X11UseLocalhost Specifies whether .Xr sshd 8 should bind the X11 forwarding server to the loopback address or to the wildcard address. By default, sshd binds the forwarding server to the loopback address and sets the hostname part of the .Ev DISPLAY environment variable to .Dq localhost . This prevents remote hosts from connecting to the proxy display. However, some older X11 clients may not function with this configuration. .Cm X11UseLocalhost may be set to .Dq no to specify that the forwarding server should be bound to the wildcard address. The argument must be .Dq yes or .Dq no . The default is .Dq yes . .It Cm XAuthLocation Specifies the full pathname of the .Xr xauth 1 program, or .Dq none to not use one. The default is .Pa /usr/local/bin/xauth . .El .Sh TIME FORMATS .Xr sshd 8 command-line arguments and configuration file options that specify time may be expressed using a sequence of the form: .Sm off .Ar time Op Ar qualifier , .Sm on where .Ar time is a positive integer value and .Ar qualifier is one of the following: .Pp .Bl -tag -width Ds -compact -offset indent .It Aq Cm none seconds .It Cm s | Cm S seconds .It Cm m | Cm M minutes .It Cm h | Cm H hours .It Cm d | Cm D days .It Cm w | Cm W weeks .El .Pp Each member of the sequence is added together to calculate the total time value. .Pp Time format examples: .Pp .Bl -tag -width Ds -compact -offset indent .It 600 600 seconds (10 minutes) .It 10m 10 minutes .It 1h30m 1 hour 30 minutes (90 minutes) .El .Sh FILES .Bl -tag -width Ds .It Pa /etc/ssh/sshd_config Contains configuration data for .Xr sshd 8 . This file should be writable by root only, but it is recommended (though not necessary) that it be world-readable. .El .Sh SEE ALSO .Xr sshd 8 .Sh AUTHORS OpenSSH is a derivative of the original and free ssh 1.2.12 release by Tatu Ylonen. Aaron Campbell, Bob Beck, Markus Friedl, Niels Provos, Theo de Raadt and Dug Song removed many bugs, re-added newer features and created OpenSSH. Markus Friedl contributed the support for SSH protocol versions 1.5 and 2.0. Niels Provos and Markus Friedl contributed support for privilege separation. Index: projects/clang390-import/crypto/openssh =================================================================== --- projects/clang390-import/crypto/openssh (revision 305080) +++ projects/clang390-import/crypto/openssh (revision 305081) Property changes on: projects/clang390-import/crypto/openssh ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/crypto/openssh:r304236-305080 Index: projects/clang390-import/lib/atf/libatf-c/Makefile =================================================================== --- projects/clang390-import/lib/atf/libatf-c/Makefile (revision 305080) +++ projects/clang390-import/lib/atf/libatf-c/Makefile (revision 305081) @@ -1,93 +1,93 @@ #- # Copyright (c) 2011 Google, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ .include .include -_CFLAGS:= ${CFLAGS} -_CPPFLAGS:= ${CPPFLAGS} -_CXXFLAGS:= ${CXXFLAGS} +ATF_BUILD_CFLAGS:= ${CFLAGS:M-[DILOWf]*} +ATF_BUILD_CPPFLAGS:= ${CPPFLAGS:M-[DILOWf]*} +ATF_BUILD_CXXFLAGS:= ${CXXFLAGS:M-[DILOWf]*} LIB= atf-c PRIVATELIB= true SHLIB_MAJOR= 1 -ATF= ${.CURDIR:H:H:H}/contrib/atf +ATF= ${SRCTOP}/contrib/atf .PATH: ${ATF} .PATH: ${ATF}/atf-c .PATH: ${ATF}/atf-c/detail CFLAGS+= -DATF_BUILD_CC='"${CC}"' -CFLAGS+= -DATF_BUILD_CFLAGS='"${_CFLAGS}"' +CFLAGS+= -DATF_BUILD_CFLAGS='"${ATF_BUILD_CFLAGS}"' CFLAGS+= -DATF_BUILD_CPP='"${CPP}"' -CFLAGS+= -DATF_BUILD_CPPFLAGS='"${_CPPFLAGS}"' +CFLAGS+= -DATF_BUILD_CPPFLAGS='"${ATF_BUILD_CPPFLAGS}"' CFLAGS+= -DATF_BUILD_CXX='"${CXX}"' -CFLAGS+= -DATF_BUILD_CXXFLAGS='"${_CXXFLAGS}"' +CFLAGS+= -DATF_BUILD_CXXFLAGS='"${ATF_BUILD_CXXFLAGS}"' CFLAGS+= -I${ATF} CFLAGS+= -I${.CURDIR} CFLAGS+= -I. SRCS= build.c \ check.c \ dynstr.c \ env.c \ error.c \ fs.c \ list.c \ map.c \ process.c \ sanity.c \ text.c \ user.c \ utils.c \ tc.c \ tp.c \ tp_main.c INCS= build.h \ check.h \ defs.h \ error.h \ error_fwd.h \ macros.h \ tc.h \ tp.h \ utils.h INCSDIR= ${INCLUDEDIR}/atf-c INCS+= atf-c.h INCSDIR_atf-c.h= ${INCLUDEDIR} MAN= atf-c.3 MLINKS+= atf-c.3 atf-c-api.3 # Backwards compatibility. .if ${MK_TESTS} != "no" SUBDIR= tests .endif .include "../common.mk" .include Index: projects/clang390-import/lib/atf/libatf-c++/Makefile =================================================================== --- projects/clang390-import/lib/atf/libatf-c++/Makefile (revision 305080) +++ projects/clang390-import/lib/atf/libatf-c++/Makefile (revision 305081) @@ -1,80 +1,80 @@ #- # Copyright (c) 2011 Google, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ .include .include LIB_CXX= atf-c++ PRIVATELIB= true SHLIB_MAJOR= 2 # libatf-c++ depends on the C version of the ATF library to build. LIBADD+= atf_c -LDFLAGS+= -L${.OBJDIR}/../libatf-c +LDFLAGS+= -L${.OBJDIR:H}/libatf-c -ATF= ${.CURDIR:H:H:H}/contrib/atf +ATF= ${SRCTOP}/contrib/atf .PATH: ${ATF} .PATH: ${ATF}/atf-c++ .PATH: ${ATF}/atf-c++/detail CFLAGS+= -I${ATF} -CFLAGS+= -I${.CURDIR}/../libatf-c +CFLAGS+= -I${.CURDIR:H}/libatf-c CFLAGS+= -I. CFLAGS+= -DHAVE_CONFIG_H SRCS= application.cpp \ build.cpp \ check.cpp \ env.cpp \ exceptions.cpp \ fs.cpp \ process.cpp \ tests.cpp \ text.cpp \ utils.cpp INCS= build.hpp \ check.hpp \ macros.hpp \ tests.hpp \ utils.hpp INCSDIR= ${INCLUDEDIR}/atf-c++ INCS+= atf-c++.hpp INCSDIR_atf-c++.hpp= ${INCLUDEDIR} MAN= atf-c++.3 MLINKS+= atf-c++.3 atf-c-api++.3 # Backwards compatibility. .if ${MK_TESTS} != "no" SUBDIR= tests .endif .include "../common.mk" .include Index: projects/clang390-import/lib/libdevctl/devctl.3 =================================================================== --- projects/clang390-import/lib/libdevctl/devctl.3 (revision 305080) +++ projects/clang390-import/lib/libdevctl/devctl.3 (revision 305081) @@ -1,339 +1,378 @@ .\" .\" Copyright (c) 2014 John Baldwin .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd April 27, 2016 +.Dd August 29, 2016 .Dt DEVCTL 3 .Os .Sh NAME .Nm devctl , .Nm devctl_attach , +.Nm devctl_clear_driver , .Nm devctl_delete , .Nm devctl_detach , .Nm devctl_disable , .Nm devctl_enable , .Nm devctl_rescan , .Nm devctl_resume , .Nm devctl_set_driver , .Nm devctl_suspend .Nd device control library .Sh LIBRARY .Lb libdevctl .Sh SYNOPSIS .In devctl.h .Ft int .Fn devctl_attach "const char *device" .Ft int +.Fn devctl_clear_driver "const char *device" "bool force" +.Ft int .Fn devctl_delete "const char *device" "bool force" .Ft int .Fn devctl_detach "const char *device" "bool force" .Ft int .Fn devctl_disable "const char *device" "bool force_detach" .Ft int .Fn devctl_enable "const char *device" .Ft int .Fn devctl_rescan "const char *device" .Ft int .Fn devctl_resume "const char *device" .Ft int .Fn devctl_set_driver "const char *device" "const char *driver" "bool force" .Ft int .Fn devctl_suspend "const char *device" .Sh DESCRIPTION The .Nm library adjusts the state of devices in the kernel's internal device hierarchy. Each control operation accepts a .Fa device argument that identifies the device to adjust. The .Fa device may be specified as either the name of an existing device or as a bus-specific address. The following bus-specific address formats are currently supported: .Bl -tag -offset indent .It Sy pci Ns Fa domain Ns : Ns Fa bus Ns : Ns Fa slot Ns : Ns Fa function A PCI device with the specified .Fa domain , .Fa bus , .Fa slot , and .Fa function . .It Sy pci Ns Fa bus Ns : Ns Fa slot Ns : Ns Fa function A PCI device in domain zero with the specified .Fa bus , .Fa slot , and .Fa function . .It Fa handle A device with an ACPI handle of .Fa handle . The handle must be specified as an absolute path and must begin with a .Dq \e . .El .Pp The .Fn devctl_attach function probes a device and attaches a suitable device driver if one is found. .Pp The .Fn devctl_detach function detaches a device from its current device driver. The device is left detached until either a new driver for its parent bus is loaded or the device is explicitly probed via .Fn devctl_attach . If .Fa force is true, the current device driver will be detached even if the device is busy. .Pp The .Fn devctl_delete function deletes a device from the device tree. No If .Fa force is true, the device is deleted even if the device is physically present. .Pp The .Fn devctl_disable function disables a device. If the device is currently attached to a device driver, the device driver will be detached from the device, but the device will retain its current name. If .Fa force_detach is true, the current device driver will be detached even if the device is busy. The device will remain disabled and detached until it is explicitly enabled via .Fn devctl_enable . .Pp The .Fn devctl_enable function re-enables a disabled device. The device will probe and attach if a suitable device driver is found. .Pp The .Fn devctl_suspend function suspends a device. This may include placing the device in a reduced power state, but any device driver currently attached to the device will remain attached. .Pp The .Fn devctl_resume function resumes a suspended device to a fully working state. .Pp The .Fn devctl_set_driver function attaches a device driver named .Fa driver to a device. If the device is already attached and .Fa force is false, the request will fail. If the device is already attached and .Fa force is true, the device will be detached from its current device driver before it is attached to the new device driver. .Pp The +.Fn devctl_clear_driver +function resets a device so that it can be attached to any valid device +driver rather than only drivers with a previously specified name. +This function is used to undo a previous call to +.Fn devctl_set_driver . +If the device is already attached and +.Fa force +is false, +the request will fail. +If the device is already attached and +.Fa force +is true, +the device will be detached from its current device driver. +After the device's name is reset, +it is reprobed and attached to a suitable device driver if one is found. +.Pp +The .Fn devctl_rescan function rescans a bus device checking for devices that have been added or removed. .Sh RETURN VALUES -.Rv -std devctl_attach devctl_delete devctl_detach devctl_disable \ -devctl_enable devctl_suspend devctl_rescan devctl_resume devctl_set_driver +.Rv -std devctl_attach devctl_clear_driver devctl_delete devctl_detach \ +devctl_disable devctl_enable devctl_suspend devctl_rescan devctl_resume \ +devctl_set_driver .Sh ERRORS In addition to specific errors noted below, all of the .Nm functions may fail for any of the errors described in .Xr open 2 as well as: .Bl -tag -width Er .It Bq Er EINVAL The device name is too long. .It Bq Er ENOENT No existing device matches the specified name or location. .It Bq Er EPERM The current process is not permitted to adjust the state of .Fa device . .El .Pp The .Fn devctl_attach function may fail if: .Bl -tag -width Er .It Bq Er EBUSY The device is already attached. .It Bq Er ENOMEM An internal memory allocation request failed. .It Bq Er ENXIO The device is disabled. .It Bq Er ENXIO No suitable driver for the device could be found, or the driver failed to attach. .El .Pp The .Fn devctl_detach function may fail if: .Bl -tag -width Er .It Bq Er EBUSY The current device driver for .Fa device is busy and cannot detach at this time. Note that some drivers may return this even if .Fa force is true. .It Bq Er ENXIO The device is not attached to a driver. .It Bq Er ENXIO The current device driver for .Fa device does not support detaching. .El .Pp The .Fn devctl_enable function may fail if: .Bl -tag -width Er .It Bq Er EBUSY The device is already enabled. .It Bq Er ENOMEM An internal memory allocation request failed. .It Bq Er ENXIO No suitable driver for the device could be found, or the driver failed to attach. .El .Pp The .Fn devctl_disable function may fail if: .Bl -tag -width Er .It Bq Er EBUSY The current device driver for .Fa device is busy and cannot detach at this time. Note that some drivers may return this even if .Fa force_detach is true. .It Bq Er ENXIO The device is already disabled. .It Bq Er ENXIO The current device driver for .Fa device does not support detaching. .El .Pp The .Fn devctl_suspend function may fail if: .Bl -tag -width Er .It Bq Er EBUSY The device is already suspended. .It Bq Er EINVAL The device to be suspended is the root bus device. .El .Pp The .Fn devctl_resume function may fail if: .Bl -tag -width Er .It Bq Er EINVAL The device is not suspended. .It Bq Er EINVAL The device to be resumed is the root bus device. .El .Pp The .Fn devctl_set_driver function may fail if: .Bl -tag -width Er .It Bq Er EBUSY The device is currently attached to a device driver and .Fa force is false. .It Bq Er EBUSY The current device driver for .Fa device is busy and cannot detach at this time. .It Bq Er EFAULT The .Fa driver argument points outside the process' allocated address space. .It Bq Er ENOENT No device driver with the requested name exists. .It Bq Er ENOMEM An internal memory allocation request failed. .It Bq Er ENXIO The device is disabled. .It Bq Er ENXIO The new device driver failed to attach. +.El +.Pp +The +.Fn devctl_clear_driver +function may fail if: +.Bl -tag -width Er +.It Bq Er EBUSY +The device is currently attached to a device driver and +.Fa force +is false. +.It Bq Er EBUSY +The current device driver for +.Fa device +is busy and cannot detach at this time. +.It Bq Er EINVAL +The device is not configured for a specific device driver name. +.It Bq Er ENXIO +The device driver chosen after reprobing failed to attach. .El .Pp The .Fn devctl_rescan function may fail if: .Bl -tag -width Er .It Bq Er ENXIO The device is not attached to a driver. .It Bq Er ENXIO The bus driver does not support rescanning. .El .Pp The .Fn devctl_delete function may fail if: .Bl -tag -width Er .It Bq Er EBUSY The device is physically present and .Fa force is false. .It Bq Er EINVAL .Fa dev is the root device of the device tree. .El .Sh SEE ALSO .Xr devinfo 3 , .Xr devstat 3 , .Xr devctl 8 .Sh HISTORY The .Nm library first appeared in .Fx 10.3 . .Sh BUGS If a device is suspended individually via .Fn devctl_suspend and the entire machine is subsequently suspended, the device will be resumed when the machine resumes. Index: projects/clang390-import/lib/libdevctl/devctl.c =================================================================== --- projects/clang390-import/lib/libdevctl/devctl.c (revision 305080) +++ projects/clang390-import/lib/libdevctl/devctl.c (revision 305081) @@ -1,139 +1,147 @@ /*- * Copyright (c) 2014 John Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "devctl.h" static int devctl_request(u_long cmd, struct devreq *req) { static int devctl2_fd = -1; if (devctl2_fd == -1) { devctl2_fd = open("/dev/devctl2", O_RDONLY); if (devctl2_fd == -1) return (-1); } return (ioctl(devctl2_fd, cmd, req)); } static int devctl_simple_request(u_long cmd, const char *name, int flags) { struct devreq req; memset(&req, 0, sizeof(req)); if (strlcpy(req.dr_name, name, sizeof(req.dr_name)) >= sizeof(req.dr_name)) { errno = EINVAL; return (-1); } req.dr_flags = flags; return (devctl_request(cmd, &req)); } int devctl_attach(const char *device) { return (devctl_simple_request(DEV_ATTACH, device, 0)); } int devctl_detach(const char *device, bool force) { return (devctl_simple_request(DEV_DETACH, device, force ? DEVF_FORCE_DETACH : 0)); } int devctl_enable(const char *device) { return (devctl_simple_request(DEV_ENABLE, device, 0)); } int devctl_disable(const char *device, bool force_detach) { return (devctl_simple_request(DEV_DISABLE, device, force_detach ? DEVF_FORCE_DETACH : 0)); } int devctl_suspend(const char *device) { return (devctl_simple_request(DEV_SUSPEND, device, 0)); } int devctl_resume(const char *device) { return (devctl_simple_request(DEV_RESUME, device, 0)); } int devctl_set_driver(const char *device, const char *driver, bool force) { struct devreq req; memset(&req, 0, sizeof(req)); if (strlcpy(req.dr_name, device, sizeof(req.dr_name)) >= sizeof(req.dr_name)) { errno = EINVAL; return (-1); } req.dr_data = __DECONST(char *, driver); if (force) req.dr_flags |= DEVF_SET_DRIVER_DETACH; return (devctl_request(DEV_SET_DRIVER, &req)); } int +devctl_clear_driver(const char *device, bool force) +{ + + return (devctl_simple_request(DEV_CLEAR_DRIVER, device, force ? + DEVF_CLEAR_DRIVER_DETACH : 0)); +} + +int devctl_rescan(const char *device) { return (devctl_simple_request(DEV_RESCAN, device, 0)); } int devctl_delete(const char *device, bool force) { return (devctl_simple_request(DEV_DELETE, device, force ? DEVF_FORCE_DELETE : 0)); } Index: projects/clang390-import/lib/libdevctl/devctl.h =================================================================== --- projects/clang390-import/lib/libdevctl/devctl.h (revision 305080) +++ projects/clang390-import/lib/libdevctl/devctl.h (revision 305081) @@ -1,44 +1,45 @@ /*- * Copyright (c) 2014 John Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __DEVCTL_H__ #define __DEVCTL_H__ #include int devctl_attach(const char *device); int devctl_detach(const char *device, bool force); int devctl_enable(const char *device); int devctl_disable(const char *device, bool force_detach); int devctl_suspend(const char *device); int devctl_resume(const char *device); int devctl_set_driver(const char *device, const char *driver, bool force); +int devctl_clear_driver(const char *device, bool force); int devctl_rescan(const char *device); int devctl_delete(const char *device, bool force); #endif /* !__DEVCTL_H__ */ Index: projects/clang390-import/sbin/newfs_msdos/mkfs_msdos.c =================================================================== --- projects/clang390-import/sbin/newfs_msdos/mkfs_msdos.c (revision 305080) +++ projects/clang390-import/sbin/newfs_msdos/mkfs_msdos.c (revision 305081) @@ -1,959 +1,952 @@ /* * Copyright (c) 1998 Robert Nordier * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef lint static const char rcsid[] = "$FreeBSD$"; #endif /* not lint */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mkfs_msdos.h" #define MAXU16 0xffff /* maximum unsigned 16-bit quantity */ #define BPN 4 /* bits per nibble */ #define NPB 2 /* nibbles per byte */ #define DOSMAGIC 0xaa55 /* DOS magic number */ #define MINBPS 512 /* minimum bytes per sector */ #define MAXSPC 128 /* maximum sectors per cluster */ #define MAXNFT 16 /* maximum number of FATs */ #define DEFBLK 4096 /* default block size */ #define DEFBLK16 2048 /* default block size FAT16 */ #define DEFRDE 512 /* default root directory entries */ #define RESFTE 2 /* reserved FAT entries */ #define MINCLS12 1U /* minimum FAT12 clusters */ #define MINCLS16 0xff5U /* minimum FAT16 clusters */ #define MINCLS32 0xfff5U /* minimum FAT32 clusters */ #define MAXCLS12 0xff4U /* maximum FAT12 clusters */ #define MAXCLS16 0xfff4U /* maximum FAT16 clusters */ #define MAXCLS32 0xffffff4U /* maximum FAT32 clusters */ #define mincls(fat) ((fat) == 12 ? MINCLS12 : \ (fat) == 16 ? MINCLS16 : \ MINCLS32) #define maxcls(fat) ((fat) == 12 ? MAXCLS12 : \ (fat) == 16 ? MAXCLS16 : \ MAXCLS32) #define mk1(p, x) \ (p) = (u_int8_t)(x) #define mk2(p, x) \ (p)[0] = (u_int8_t)(x), \ (p)[1] = (u_int8_t)((x) >> 010) #define mk4(p, x) \ (p)[0] = (u_int8_t)(x), \ (p)[1] = (u_int8_t)((x) >> 010), \ (p)[2] = (u_int8_t)((x) >> 020), \ (p)[3] = (u_int8_t)((x) >> 030) struct bs { u_int8_t bsJump[3]; /* bootstrap entry point */ u_int8_t bsOemName[8]; /* OEM name and version */ } __packed; struct bsbpb { u_int8_t bpbBytesPerSec[2]; /* bytes per sector */ u_int8_t bpbSecPerClust; /* sectors per cluster */ u_int8_t bpbResSectors[2]; /* reserved sectors */ u_int8_t bpbFATs; /* number of FATs */ u_int8_t bpbRootDirEnts[2]; /* root directory entries */ u_int8_t bpbSectors[2]; /* total sectors */ u_int8_t bpbMedia; /* media descriptor */ u_int8_t bpbFATsecs[2]; /* sectors per FAT */ u_int8_t bpbSecPerTrack[2]; /* sectors per track */ u_int8_t bpbHeads[2]; /* drive heads */ u_int8_t bpbHiddenSecs[4]; /* hidden sectors */ u_int8_t bpbHugeSectors[4]; /* big total sectors */ } __packed; struct bsxbpb { u_int8_t bpbBigFATsecs[4]; /* big sectors per FAT */ u_int8_t bpbExtFlags[2]; /* FAT control flags */ u_int8_t bpbFSVers[2]; /* file system version */ u_int8_t bpbRootClust[4]; /* root directory start cluster */ u_int8_t bpbFSInfo[2]; /* file system info sector */ u_int8_t bpbBackup[2]; /* backup boot sector */ u_int8_t bpbReserved[12]; /* reserved */ } __packed; struct bsx { u_int8_t exDriveNumber; /* drive number */ u_int8_t exReserved1; /* reserved */ u_int8_t exBootSignature; /* extended boot signature */ u_int8_t exVolumeID[4]; /* volume ID number */ u_int8_t exVolumeLabel[11]; /* volume label */ u_int8_t exFileSysType[8]; /* file system type */ } __packed; struct de { u_int8_t deName[11]; /* name and extension */ u_int8_t deAttributes; /* attributes */ u_int8_t rsvd[10]; /* reserved */ u_int8_t deMTime[2]; /* last-modified time */ u_int8_t deMDate[2]; /* last-modified date */ u_int8_t deStartCluster[2]; /* starting cluster */ u_int8_t deFileSize[4]; /* size */ } __packed; struct bpb { u_int bpbBytesPerSec; /* bytes per sector */ u_int bpbSecPerClust; /* sectors per cluster */ u_int bpbResSectors; /* reserved sectors */ u_int bpbFATs; /* number of FATs */ u_int bpbRootDirEnts; /* root directory entries */ u_int bpbSectors; /* total sectors */ u_int bpbMedia; /* media descriptor */ u_int bpbFATsecs; /* sectors per FAT */ u_int bpbSecPerTrack; /* sectors per track */ u_int bpbHeads; /* drive heads */ u_int bpbHiddenSecs; /* hidden sectors */ u_int bpbHugeSectors; /* big total sectors */ u_int bpbBigFATsecs; /* big sectors per FAT */ u_int bpbRootClust; /* root directory start cluster */ u_int bpbFSInfo; /* file system info sector */ u_int bpbBackup; /* backup boot sector */ }; #define BPBGAP 0, 0, 0, 0, 0, 0 static struct { const char *name; struct bpb bpb; } const stdfmt[] = { {"160", {512, 1, 1, 2, 64, 320, 0xfe, 1, 8, 1, BPBGAP}}, {"180", {512, 1, 1, 2, 64, 360, 0xfc, 2, 9, 1, BPBGAP}}, {"320", {512, 2, 1, 2, 112, 640, 0xff, 1, 8, 2, BPBGAP}}, {"360", {512, 2, 1, 2, 112, 720, 0xfd, 2, 9, 2, BPBGAP}}, {"640", {512, 2, 1, 2, 112, 1280, 0xfb, 2, 8, 2, BPBGAP}}, {"720", {512, 2, 1, 2, 112, 1440, 0xf9, 3, 9, 2, BPBGAP}}, {"1200", {512, 1, 1, 2, 224, 2400, 0xf9, 7, 15, 2, BPBGAP}}, {"1232", {1024,1, 1, 2, 192, 1232, 0xfe, 2, 8, 2, BPBGAP}}, {"1440", {512, 1, 1, 2, 224, 2880, 0xf0, 9, 18, 2, BPBGAP}}, {"2880", {512, 2, 1, 2, 240, 5760, 0xf0, 9, 36, 2, BPBGAP}} }; static const u_int8_t bootcode[] = { 0xfa, /* cli */ 0x31, 0xc0, /* xor ax,ax */ 0x8e, 0xd0, /* mov ss,ax */ 0xbc, 0x00, 0x7c, /* mov sp,7c00h */ 0xfb, /* sti */ 0x8e, 0xd8, /* mov ds,ax */ 0xe8, 0x00, 0x00, /* call $ + 3 */ 0x5e, /* pop si */ 0x83, 0xc6, 0x19, /* add si,+19h */ 0xbb, 0x07, 0x00, /* mov bx,0007h */ 0xfc, /* cld */ 0xac, /* lodsb */ 0x84, 0xc0, /* test al,al */ 0x74, 0x06, /* jz $ + 8 */ 0xb4, 0x0e, /* mov ah,0eh */ 0xcd, 0x10, /* int 10h */ 0xeb, 0xf5, /* jmp $ - 9 */ 0x30, 0xe4, /* xor ah,ah */ 0xcd, 0x16, /* int 16h */ 0xcd, 0x19, /* int 19h */ 0x0d, 0x0a, 'N', 'o', 'n', '-', 's', 'y', 's', 't', 'e', 'm', ' ', 'd', 'i', 's', 'k', 0x0d, 0x0a, 'P', 'r', 'e', 's', 's', ' ', 'a', 'n', 'y', ' ', 'k', 'e', 'y', ' ', 't', 'o', ' ', 'r', 'e', 'b', 'o', 'o', 't', 0x0d, 0x0a, 0 }; static volatile sig_atomic_t got_siginfo; static void infohandler(int); static int check_mounted(const char *, mode_t); static int getstdfmt(const char *, struct bpb *); static int getdiskinfo(int, const char *, const char *, int, struct bpb *); static void print_bpb(struct bpb *); static int ckgeom(const char *, u_int, const char *); static void mklabel(u_int8_t *, const char *); static int oklabel(const char *); static void setstr(u_int8_t *, const char *, size_t); int mkfs_msdos(const char *fname, const char *dtype, const struct msdos_options *op) { char buf[MAXPATHLEN]; struct sigaction si_sa; struct stat sb; struct timeval tv; struct bpb bpb; struct tm *tm; struct bs *bs; struct bsbpb *bsbpb; struct bsxbpb *bsxbpb; struct bsx *bsx; struct de *de; u_int8_t *img; const char *bname; ssize_t n; time_t now; u_int fat, bss, rds, cls, dir, lsn, x, x1, x2; int fd, fd1, rv; struct msdos_options o = *op; img = NULL; rv = -1; if (o.block_size && o.sectors_per_cluster) { warnx("Cannot specify both block size and sectors per cluster"); goto done; } if (o.OEM_string && strlen(o.OEM_string) > 8) { warnx("%s: bad OEM string", o.OEM_string); goto done; } if (o.create_size) { if (o.no_create) { warnx("create (-C) is incompatible with -N"); goto done; } fd = open(fname, O_RDWR | O_CREAT | O_TRUNC, 0644); if (fd == -1) { warnx("failed to create %s", fname); goto done; } if (ftruncate(fd, o.create_size)) { warnx("failed to initialize %jd bytes", (intmax_t)o.create_size); goto done; } } else if ((fd = open(fname, o.no_create ? O_RDONLY : O_RDWR)) == -1) { warn("%s", fname); goto done; } if (fstat(fd, &sb)) { warn("%s", fname); goto done; } if (o.create_size) { if (!S_ISREG(sb.st_mode)) warnx("warning, %s is not a regular file", fname); } else { if (!S_ISCHR(sb.st_mode)) warnx("warning, %s is not a character device", fname); } if (!o.no_create) if (check_mounted(fname, sb.st_mode) == -1) goto done; if (o.offset && o.offset != lseek(fd, o.offset, SEEK_SET)) { warnx("cannot seek to %jd", (intmax_t)o.offset); goto done; } memset(&bpb, 0, sizeof(bpb)); if (o.floppy) { if (getstdfmt(o.floppy, &bpb) == -1) goto done; bpb.bpbHugeSectors = bpb.bpbSectors; bpb.bpbSectors = 0; bpb.bpbBigFATsecs = bpb.bpbFATsecs; bpb.bpbFATsecs = 0; } if (o.drive_heads) bpb.bpbHeads = o.drive_heads; if (o.sectors_per_track) bpb.bpbSecPerTrack = o.sectors_per_track; if (o.bytes_per_sector) bpb.bpbBytesPerSec = o.bytes_per_sector; if (o.size) bpb.bpbHugeSectors = o.size; if (o.hidden_sectors_set) bpb.bpbHiddenSecs = o.hidden_sectors; if (!(o.floppy || (o.drive_heads && o.sectors_per_track && o.bytes_per_sector && o.size && o.hidden_sectors_set))) { - off_t delta; getdiskinfo(fd, fname, dtype, o.hidden_sectors_set, &bpb); bpb.bpbHugeSectors -= (o.offset / bpb.bpbBytesPerSec); - delta = bpb.bpbHugeSectors % bpb.bpbSecPerTrack; - if (delta != 0) { - warnx("trim %d sectors to adjust to a multiple of %d", - (int)delta, bpb.bpbSecPerTrack); - bpb.bpbHugeSectors -= delta; - } if (bpb.bpbSecPerClust == 0) { /* set defaults */ if (bpb.bpbHugeSectors <= 6000) /* about 3MB -> 512 bytes */ bpb.bpbSecPerClust = 1; else if (bpb.bpbHugeSectors <= (1<<17)) /* 64M -> 4k */ bpb.bpbSecPerClust = 8; else if (bpb.bpbHugeSectors <= (1<<19)) /* 256M -> 8k */ bpb.bpbSecPerClust = 16; else if (bpb.bpbHugeSectors <= (1<<21)) /* 1G -> 16k */ bpb.bpbSecPerClust = 32; else bpb.bpbSecPerClust = 64; /* otherwise 32k */ } } if (!powerof2(bpb.bpbBytesPerSec)) { warnx("bytes/sector (%u) is not a power of 2", bpb.bpbBytesPerSec); goto done; } if (bpb.bpbBytesPerSec < MINBPS) { warnx("bytes/sector (%u) is too small; minimum is %u", bpb.bpbBytesPerSec, MINBPS); goto done; } if (o.volume_label && !oklabel(o.volume_label)) { warnx("%s: bad volume label", o.volume_label); goto done; } if (!(fat = o.fat_type)) { if (o.floppy) fat = 12; else if (!o.directory_entries && (o.info_sector || o.backup_sector)) fat = 32; } if ((fat == 32 && o.directory_entries) || (fat != 32 && (o.info_sector || o.backup_sector))) { warnx("-%c is not a legal FAT%s option", fat == 32 ? 'e' : o.info_sector ? 'i' : 'k', fat == 32 ? "32" : "12/16"); goto done; } if (o.floppy && fat == 32) bpb.bpbRootDirEnts = 0; if (fat != 0 && fat != 12 && fat != 16 && fat != 32) { warnx("%d: bad FAT type", fat); goto done; } if (o.block_size) { if (!powerof2(o.block_size)) { warnx("block size (%u) is not a power of 2", o.block_size); goto done; } if (o.block_size < bpb.bpbBytesPerSec) { warnx("block size (%u) is too small; minimum is %u", o.block_size, bpb.bpbBytesPerSec); goto done; } if (o.block_size > bpb.bpbBytesPerSec * MAXSPC) { warnx("block size (%u) is too large; maximum is %u", o.block_size, bpb.bpbBytesPerSec * MAXSPC); goto done; } bpb.bpbSecPerClust = o.block_size / bpb.bpbBytesPerSec; } if (o.sectors_per_cluster) { if (!powerof2(o.sectors_per_cluster)) { warnx("sectors/cluster (%u) is not a power of 2", o.sectors_per_cluster); goto done; } bpb.bpbSecPerClust = o.sectors_per_cluster; } if (o.reserved_sectors) bpb.bpbResSectors = o.reserved_sectors; if (o.num_FAT) { if (o.num_FAT > MAXNFT) { warnx("number of FATs (%u) is too large; maximum is %u", o.num_FAT, MAXNFT); goto done; } bpb.bpbFATs = o.num_FAT; } if (o.directory_entries) bpb.bpbRootDirEnts = o.directory_entries; if (o.media_descriptor_set) { if (o.media_descriptor < 0xf0) { warnx("illegal media descriptor (%#x)", o.media_descriptor); goto done; } bpb.bpbMedia = o.media_descriptor; } if (o.sectors_per_fat) bpb.bpbBigFATsecs = o.sectors_per_fat; if (o.info_sector) bpb.bpbFSInfo = o.info_sector; if (o.backup_sector) bpb.bpbBackup = o.backup_sector; bss = 1; bname = NULL; fd1 = -1; if (o.bootstrap) { bname = o.bootstrap; if (!strchr(bname, '/')) { snprintf(buf, sizeof(buf), "/boot/%s", bname); if (!(bname = strdup(buf))) { warn(NULL); goto done; } } if ((fd1 = open(bname, O_RDONLY)) == -1 || fstat(fd1, &sb)) { warn("%s", bname); goto done; } if (!S_ISREG(sb.st_mode) || sb.st_size % bpb.bpbBytesPerSec || sb.st_size < bpb.bpbBytesPerSec || sb.st_size > bpb.bpbBytesPerSec * MAXU16) { warnx("%s: inappropriate file type or format", bname); goto done; } bss = sb.st_size / bpb.bpbBytesPerSec; } if (!bpb.bpbFATs) bpb.bpbFATs = 2; if (!fat) { if (bpb.bpbHugeSectors < (bpb.bpbResSectors ? bpb.bpbResSectors : bss) + howmany((RESFTE + (bpb.bpbSecPerClust ? MINCLS16 : MAXCLS12 + 1)) * (bpb.bpbSecPerClust ? 16 : 12) / BPN, bpb.bpbBytesPerSec * NPB) * bpb.bpbFATs + howmany(bpb.bpbRootDirEnts ? bpb.bpbRootDirEnts : DEFRDE, bpb.bpbBytesPerSec / sizeof(struct de)) + (bpb.bpbSecPerClust ? MINCLS16 : MAXCLS12 + 1) * (bpb.bpbSecPerClust ? bpb.bpbSecPerClust : howmany(DEFBLK, bpb.bpbBytesPerSec))) fat = 12; else if (bpb.bpbRootDirEnts || bpb.bpbHugeSectors < (bpb.bpbResSectors ? bpb.bpbResSectors : bss) + howmany((RESFTE + MAXCLS16) * 2, bpb.bpbBytesPerSec) * bpb.bpbFATs + howmany(DEFRDE, bpb.bpbBytesPerSec / sizeof(struct de)) + (MAXCLS16 + 1) * (bpb.bpbSecPerClust ? bpb.bpbSecPerClust : howmany(8192, bpb.bpbBytesPerSec))) fat = 16; else fat = 32; } x = bss; if (fat == 32) { if (!bpb.bpbFSInfo) { if (x == MAXU16 || x == bpb.bpbBackup) { warnx("no room for info sector"); goto done; } bpb.bpbFSInfo = x; } if (bpb.bpbFSInfo != MAXU16 && x <= bpb.bpbFSInfo) x = bpb.bpbFSInfo + 1; if (!bpb.bpbBackup) { if (x == MAXU16) { warnx("no room for backup sector"); goto done; } bpb.bpbBackup = x; } else if (bpb.bpbBackup != MAXU16 && bpb.bpbBackup == bpb.bpbFSInfo) { warnx("backup sector would overwrite info sector"); goto done; } if (bpb.bpbBackup != MAXU16 && x <= bpb.bpbBackup) x = bpb.bpbBackup + 1; } if (!bpb.bpbResSectors) bpb.bpbResSectors = fat == 32 ? MAX(x, MAX(16384 / bpb.bpbBytesPerSec, 4)) : x; else if (bpb.bpbResSectors < x) { warnx("too few reserved sectors (need %d have %d)", x, bpb.bpbResSectors); goto done; } if (fat != 32 && !bpb.bpbRootDirEnts) bpb.bpbRootDirEnts = DEFRDE; rds = howmany(bpb.bpbRootDirEnts, bpb.bpbBytesPerSec / sizeof(struct de)); if (!bpb.bpbSecPerClust) for (bpb.bpbSecPerClust = howmany(fat == 16 ? DEFBLK16 : DEFBLK, bpb.bpbBytesPerSec); bpb.bpbSecPerClust < MAXSPC && bpb.bpbResSectors + howmany((RESFTE + maxcls(fat)) * (fat / BPN), bpb.bpbBytesPerSec * NPB) * bpb.bpbFATs + rds + (u_int64_t) (maxcls(fat) + 1) * bpb.bpbSecPerClust <= bpb.bpbHugeSectors; bpb.bpbSecPerClust <<= 1) continue; if (fat != 32 && bpb.bpbBigFATsecs > MAXU16) { warnx("too many sectors/FAT for FAT12/16"); goto done; } x1 = bpb.bpbResSectors + rds; x = bpb.bpbBigFATsecs ? bpb.bpbBigFATsecs : 1; if (x1 + (u_int64_t)x * bpb.bpbFATs > bpb.bpbHugeSectors) { warnx("meta data exceeds file system size"); goto done; } x1 += x * bpb.bpbFATs; x = (u_int64_t)(bpb.bpbHugeSectors - x1) * bpb.bpbBytesPerSec * NPB / (bpb.bpbSecPerClust * bpb.bpbBytesPerSec * NPB + fat / BPN * bpb.bpbFATs); x2 = howmany((RESFTE + MIN(x, maxcls(fat))) * (fat / BPN), bpb.bpbBytesPerSec * NPB); if (!bpb.bpbBigFATsecs) { bpb.bpbBigFATsecs = x2; x1 += (bpb.bpbBigFATsecs - 1) * bpb.bpbFATs; } cls = (bpb.bpbHugeSectors - x1) / bpb.bpbSecPerClust; x = (u_int64_t)bpb.bpbBigFATsecs * bpb.bpbBytesPerSec * NPB / (fat / BPN) - RESFTE; if (cls > x) cls = x; if (bpb.bpbBigFATsecs < x2) warnx("warning: sectors/FAT limits file system to %u clusters", cls); if (cls < mincls(fat)) { warnx("%u clusters too few clusters for FAT%u, need %u", cls, fat, mincls(fat)); goto done; } if (cls > maxcls(fat)) { cls = maxcls(fat); bpb.bpbHugeSectors = x1 + (cls + 1) * bpb.bpbSecPerClust - 1; warnx("warning: FAT type limits file system to %u sectors", bpb.bpbHugeSectors); } printf("%s: %u sector%s in %u FAT%u cluster%s " "(%u bytes/cluster)\n", fname, cls * bpb.bpbSecPerClust, cls * bpb.bpbSecPerClust == 1 ? "" : "s", cls, fat, cls == 1 ? "" : "s", bpb.bpbBytesPerSec * bpb.bpbSecPerClust); if (!bpb.bpbMedia) bpb.bpbMedia = !bpb.bpbHiddenSecs ? 0xf0 : 0xf8; if (fat == 32) bpb.bpbRootClust = RESFTE; - if (bpb.bpbHiddenSecs + bpb.bpbHugeSectors <= MAXU16) { + if (bpb.bpbHugeSectors <= MAXU16) { bpb.bpbSectors = bpb.bpbHugeSectors; bpb.bpbHugeSectors = 0; } if (fat != 32) { bpb.bpbFATsecs = bpb.bpbBigFATsecs; bpb.bpbBigFATsecs = 0; } print_bpb(&bpb); if (!o.no_create) { gettimeofday(&tv, NULL); now = tv.tv_sec; tm = localtime(&now); if (!(img = malloc(bpb.bpbBytesPerSec))) { warn(NULL); goto done; } dir = bpb.bpbResSectors + (bpb.bpbFATsecs ? bpb.bpbFATsecs : bpb.bpbBigFATsecs) * bpb.bpbFATs; memset(&si_sa, 0, sizeof(si_sa)); si_sa.sa_handler = infohandler; if (sigaction(SIGINFO, &si_sa, NULL) == -1) { warn("sigaction SIGINFO"); goto done; } for (lsn = 0; lsn < dir + (fat == 32 ? bpb.bpbSecPerClust : rds); lsn++) { if (got_siginfo) { fprintf(stderr,"%s: writing sector %u of %u (%u%%)\n", fname, lsn, (dir + (fat == 32 ? bpb.bpbSecPerClust: rds)), (lsn * 100) / (dir + (fat == 32 ? bpb.bpbSecPerClust: rds))); got_siginfo = 0; } x = lsn; if (o.bootstrap && fat == 32 && bpb.bpbBackup != MAXU16 && bss <= bpb.bpbBackup && x >= bpb.bpbBackup) { x -= bpb.bpbBackup; if (!x && lseek(fd1, o.offset, SEEK_SET)) { warn("%s", bname); goto done; } } if (o.bootstrap && x < bss) { if ((n = read(fd1, img, bpb.bpbBytesPerSec)) == -1) { warn("%s", bname); goto done; } if ((unsigned)n != bpb.bpbBytesPerSec) { warnx("%s: can't read sector %u", bname, x); goto done; } } else memset(img, 0, bpb.bpbBytesPerSec); if (!lsn || (fat == 32 && bpb.bpbBackup != MAXU16 && lsn == bpb.bpbBackup)) { x1 = sizeof(struct bs); bsbpb = (struct bsbpb *)(img + x1); mk2(bsbpb->bpbBytesPerSec, bpb.bpbBytesPerSec); mk1(bsbpb->bpbSecPerClust, bpb.bpbSecPerClust); mk2(bsbpb->bpbResSectors, bpb.bpbResSectors); mk1(bsbpb->bpbFATs, bpb.bpbFATs); mk2(bsbpb->bpbRootDirEnts, bpb.bpbRootDirEnts); mk2(bsbpb->bpbSectors, bpb.bpbSectors); mk1(bsbpb->bpbMedia, bpb.bpbMedia); mk2(bsbpb->bpbFATsecs, bpb.bpbFATsecs); mk2(bsbpb->bpbSecPerTrack, bpb.bpbSecPerTrack); mk2(bsbpb->bpbHeads, bpb.bpbHeads); mk4(bsbpb->bpbHiddenSecs, bpb.bpbHiddenSecs); mk4(bsbpb->bpbHugeSectors, bpb.bpbHugeSectors); x1 += sizeof(struct bsbpb); if (fat == 32) { bsxbpb = (struct bsxbpb *)(img + x1); mk4(bsxbpb->bpbBigFATsecs, bpb.bpbBigFATsecs); mk2(bsxbpb->bpbExtFlags, 0); mk2(bsxbpb->bpbFSVers, 0); mk4(bsxbpb->bpbRootClust, bpb.bpbRootClust); mk2(bsxbpb->bpbFSInfo, bpb.bpbFSInfo); mk2(bsxbpb->bpbBackup, bpb.bpbBackup); x1 += sizeof(struct bsxbpb); } bsx = (struct bsx *)(img + x1); mk1(bsx->exBootSignature, 0x29); if (o.volume_id_set) x = o.volume_id; else x = (((u_int)(1 + tm->tm_mon) << 8 | (u_int)tm->tm_mday) + ((u_int)tm->tm_sec << 8 | (u_int)(tv.tv_usec / 10))) << 16 | ((u_int)(1900 + tm->tm_year) + ((u_int)tm->tm_hour << 8 | (u_int)tm->tm_min)); mk4(bsx->exVolumeID, x); mklabel(bsx->exVolumeLabel, o.volume_label ? o.volume_label : "NO NAME"); snprintf(buf, sizeof(buf), "FAT%u", fat); setstr(bsx->exFileSysType, buf, sizeof(bsx->exFileSysType)); if (!o.bootstrap) { x1 += sizeof(struct bsx); bs = (struct bs *)img; mk1(bs->bsJump[0], 0xeb); mk1(bs->bsJump[1], x1 - 2); mk1(bs->bsJump[2], 0x90); setstr(bs->bsOemName, o.OEM_string ? o.OEM_string : "BSD4.4 ", sizeof(bs->bsOemName)); memcpy(img + x1, bootcode, sizeof(bootcode)); mk2(img + MINBPS - 2, DOSMAGIC); } } else if (fat == 32 && bpb.bpbFSInfo != MAXU16 && (lsn == bpb.bpbFSInfo || (bpb.bpbBackup != MAXU16 && lsn == bpb.bpbBackup + bpb.bpbFSInfo))) { mk4(img, 0x41615252); mk4(img + MINBPS - 28, 0x61417272); mk4(img + MINBPS - 24, 0xffffffff); mk4(img + MINBPS - 20, bpb.bpbRootClust); mk2(img + MINBPS - 2, DOSMAGIC); } else if (lsn >= bpb.bpbResSectors && lsn < dir && !((lsn - bpb.bpbResSectors) % (bpb.bpbFATsecs ? bpb.bpbFATsecs : bpb.bpbBigFATsecs))) { mk1(img[0], bpb.bpbMedia); for (x = 1; x < fat * (fat == 32 ? 3 : 2) / 8; x++) mk1(img[x], fat == 32 && x % 4 == 3 ? 0x0f : 0xff); } else if (lsn == dir && o.volume_label) { de = (struct de *)img; mklabel(de->deName, o.volume_label); mk1(de->deAttributes, 050); x = (u_int)tm->tm_hour << 11 | (u_int)tm->tm_min << 5 | (u_int)tm->tm_sec >> 1; mk2(de->deMTime, x); x = (u_int)(tm->tm_year - 80) << 9 | (u_int)(tm->tm_mon + 1) << 5 | (u_int)tm->tm_mday; mk2(de->deMDate, x); } if ((n = write(fd, img, bpb.bpbBytesPerSec)) == -1) { warn("%s", fname); goto done; } if ((unsigned)n != bpb.bpbBytesPerSec) { warnx("%s: can't write sector %u", fname, lsn); goto done; } } } rv = 0; done: free(img); return rv; } /* * return -1 with error if file system is mounted. */ static int check_mounted(const char *fname, mode_t mode) { struct statfs *mp; const char *s1, *s2; size_t len; int n, r; if (!(n = getmntinfo(&mp, MNT_NOWAIT))) { warn("getmntinfo"); return -1; } len = strlen(_PATH_DEV); s1 = fname; if (!strncmp(s1, _PATH_DEV, len)) s1 += len; r = S_ISCHR(mode) && s1 != fname && *s1 == 'r'; for (; n--; mp++) { s2 = mp->f_mntfromname; if (!strncmp(s2, _PATH_DEV, len)) s2 += len; if ((r && s2 != mp->f_mntfromname && !strcmp(s1 + 1, s2)) || !strcmp(s1, s2)) { warnx("%s is mounted on %s", fname, mp->f_mntonname); return -1; } } return 0; } /* * Get a standard format. */ static int getstdfmt(const char *fmt, struct bpb *bpb) { u_int x, i; x = nitems(stdfmt); for (i = 0; i < x && strcmp(fmt, stdfmt[i].name); i++); if (i == x) { warnx("%s: unknown standard format", fmt); return -1; } *bpb = stdfmt[i].bpb; return 0; } /* * Get disk slice, partition, and geometry information. */ static int getdiskinfo(int fd, const char *fname, const char *dtype, __unused int oflag, struct bpb *bpb) { struct disklabel *lp, dlp; struct fd_type type; off_t ms, hs = 0; lp = NULL; /* If the user specified a disk type, try to use that */ if (dtype != NULL) { lp = getdiskbyname(dtype); } /* Maybe it's a floppy drive */ if (lp == NULL) { if (ioctl(fd, DIOCGMEDIASIZE, &ms) == -1) { struct stat st; if (fstat(fd, &st)) err(1, "cannot get disk size"); /* create a fake geometry for a file image */ ms = st.st_size; dlp.d_secsize = 512; dlp.d_nsectors = 63; dlp.d_ntracks = 255; dlp.d_secperunit = ms / dlp.d_secsize; lp = &dlp; } else if (ioctl(fd, FD_GTYPE, &type) != -1) { dlp.d_secsize = 128 << type.secsize; dlp.d_nsectors = type.sectrac; dlp.d_ntracks = type.heads; dlp.d_secperunit = ms / dlp.d_secsize; lp = &dlp; } } /* Maybe it's a fixed drive */ if (lp == NULL) { if (bpb->bpbBytesPerSec) dlp.d_secsize = bpb->bpbBytesPerSec; if (bpb->bpbBytesPerSec == 0 && ioctl(fd, DIOCGSECTORSIZE, &dlp.d_secsize) == -1) err(1, "cannot get sector size"); dlp.d_secperunit = ms / dlp.d_secsize; if (bpb->bpbSecPerTrack == 0 && ioctl(fd, DIOCGFWSECTORS, &dlp.d_nsectors) == -1) { warn("cannot get number of sectors per track"); dlp.d_nsectors = 63; } if (bpb->bpbHeads == 0 && ioctl(fd, DIOCGFWHEADS, &dlp.d_ntracks) == -1) { warn("cannot get number of heads"); if (dlp.d_secperunit <= 63*1*1024) dlp.d_ntracks = 1; else if (dlp.d_secperunit <= 63*16*1024) dlp.d_ntracks = 16; else dlp.d_ntracks = 255; } hs = (ms / dlp.d_secsize) - dlp.d_secperunit; lp = &dlp; } if (bpb->bpbBytesPerSec == 0) { if (ckgeom(fname, lp->d_secsize, "bytes/sector") == -1) return -1; bpb->bpbBytesPerSec = lp->d_secsize; } if (bpb->bpbSecPerTrack == 0) { if (ckgeom(fname, lp->d_nsectors, "sectors/track") == -1) return -1; bpb->bpbSecPerTrack = lp->d_nsectors; } if (bpb->bpbHeads == 0) { if (ckgeom(fname, lp->d_ntracks, "drive heads") == -1) return -1; bpb->bpbHeads = lp->d_ntracks; } if (bpb->bpbHugeSectors == 0) bpb->bpbHugeSectors = lp->d_secperunit; if (bpb->bpbHiddenSecs == 0) bpb->bpbHiddenSecs = hs; return 0; } /* * Print out BPB values. */ static void print_bpb(struct bpb *bpb) { printf("BytesPerSec=%u SecPerClust=%u ResSectors=%u FATs=%u", bpb->bpbBytesPerSec, bpb->bpbSecPerClust, bpb->bpbResSectors, bpb->bpbFATs); if (bpb->bpbRootDirEnts) printf(" RootDirEnts=%u", bpb->bpbRootDirEnts); if (bpb->bpbSectors) printf(" Sectors=%u", bpb->bpbSectors); printf(" Media=%#x", bpb->bpbMedia); if (bpb->bpbFATsecs) printf(" FATsecs=%u", bpb->bpbFATsecs); printf(" SecPerTrack=%u Heads=%u HiddenSecs=%u", bpb->bpbSecPerTrack, bpb->bpbHeads, bpb->bpbHiddenSecs); if (bpb->bpbHugeSectors) printf(" HugeSectors=%u", bpb->bpbHugeSectors); if (!bpb->bpbFATsecs) { printf(" FATsecs=%u RootCluster=%u", bpb->bpbBigFATsecs, bpb->bpbRootClust); printf(" FSInfo="); printf(bpb->bpbFSInfo == MAXU16 ? "%#x" : "%u", bpb->bpbFSInfo); printf(" Backup="); printf(bpb->bpbBackup == MAXU16 ? "%#x" : "%u", bpb->bpbBackup); } printf("\n"); } /* * Check a disk geometry value. */ static int ckgeom(const char *fname, u_int val, const char *msg) { if (!val) { warnx("%s: no default %s", fname, msg); return -1; } if (val > MAXU16) { warnx("%s: illegal %s %d", fname, msg, val); return -1; } return 0; } /* * Check a volume label. */ static int oklabel(const char *src) { int c, i; for (i = 0; i <= 11; i++) { c = (u_char)*src++; if (c < ' ' + !i || strchr("\"*+,./:;<=>?[\\]|", c)) break; } return i && !c; } /* * Make a volume label. */ static void mklabel(u_int8_t *dest, const char *src) { int c, i; for (i = 0; i < 11; i++) { c = *src ? toupper(*src++) : ' '; *dest++ = !i && c == '\xe5' ? 5 : c; } } /* * Copy string, padding with spaces. */ static void setstr(u_int8_t *dest, const char *src, size_t len) { while (len--) *dest++ = *src ? *src++ : ' '; } static void infohandler(int sig __unused) { got_siginfo = 1; } Index: projects/clang390-import/secure/usr.sbin/sshd/Makefile =================================================================== --- projects/clang390-import/secure/usr.sbin/sshd/Makefile (revision 305080) +++ projects/clang390-import/secure/usr.sbin/sshd/Makefile (revision 305081) @@ -1,57 +1,64 @@ # $FreeBSD$ .include PROG= sshd SRCS= sshd.c auth-rhosts.c auth-passwd.c auth-rsa.c auth-rh-rsa.c \ audit.c audit-bsm.c audit-linux.c platform.c \ sshpty.c sshlogin.c servconf.c serverloop.c \ auth.c auth1.c auth2.c auth-options.c session.c \ auth-chall.c auth2-chall.c groupaccess.c \ auth-skey.c auth-bsdauth.c auth2-hostbased.c auth2-kbdint.c \ auth2-none.c auth2-passwd.c auth2-pubkey.c \ monitor_mm.c monitor.c monitor_wrap.c auth-krb5.c \ auth2-gss.c gss-serv.c gss-serv-krb5.c \ loginrec.c auth-pam.c auth-shadow.c auth-sia.c md5crypt.c \ sftp-server.c sftp-common.c \ sandbox-null.c sandbox-rlimit.c sandbox-systrace.c sandbox-darwin.c \ sandbox-seccomp-filter.c sandbox-capsicum.c sandbox-pledge.c \ sandbox-solaris.c PACKAGE= ssh # gss-genr.c really belongs in libssh; see src/secure/lib/libssh/Makefile SRCS+= gss-genr.c MAN= sshd.8 sshd_config.5 CFLAGS+=-I${SSHDIR} -include ssh_namespace.h SRCS+= ssh_namespace.h # pam should always happen before ssh here for static linking LIBADD= pam ssh util wrap .if ${MK_LDNS} != "no" CFLAGS+= -DHAVE_LDNS=1 #DPADD+= ${LIBLDNS} #LDADD+= -lldns .endif .if ${MK_AUDIT} != "no" CFLAGS+= -DUSE_BSM_AUDIT -DHAVE_GETAUDIT_ADDR LIBADD+= bsm .endif +.if ${MK_BLACKLIST_SUPPORT} != "no" +CFLAGS+= -DUSE_BLACKLIST -I${SRCTOP}/contrib/blacklist/include +SRCS+= blacklist.c +LIBADD+= blacklist +LDFLAGS+=-L${LIBBLACKLISTDIR} +.endif + .if ${MK_KERBEROS_SUPPORT} != "no" CFLAGS+= -include krb5_config.h SRCS+= krb5_config.h LIBADD+= gssapi_krb5 gssapi krb5 .endif LIBADD+= crypto .if defined(LOCALBASE) CFLAGS+= -DXAUTH_PATH=\"${LOCALBASE}/bin/xauth\" .endif .include .PATH: ${SSHDIR} Index: projects/clang390-import/secure/usr.sbin/sshd/Makefile.depend =================================================================== --- projects/clang390-import/secure/usr.sbin/sshd/Makefile.depend (revision 305080) +++ projects/clang390-import/secure/usr.sbin/sshd/Makefile.depend (revision 305081) @@ -1,40 +1,41 @@ # $FreeBSD$ # Autogenerated - do NOT edit! DIRDEPS = \ gnu/lib/csu \ gnu/lib/libgcc \ include \ include/arpa \ include/gssapi \ include/xlocale \ kerberos5/lib/libasn1 \ kerberos5/lib/libgssapi_krb5 \ kerberos5/lib/libheimbase \ kerberos5/lib/libheimipcc \ kerberos5/lib/libhx509 \ kerberos5/lib/libkrb5 \ kerberos5/lib/libroken \ kerberos5/lib/libwind \ lib/${CSU_DIR} \ + lib/libblacklist \ lib/libbsm \ lib/libc \ lib/libcom_err \ lib/libcompiler_rt \ lib/libcrypt \ lib/libgssapi \ lib/libldns \ lib/libpam/libpam \ lib/libthr \ lib/libutil \ lib/libwrap \ lib/libz \ secure/lib/libcrypto \ secure/lib/libssh \ .include .if ${DEP_RELDIR} == ${_DEP_RELDIR} # local dependencies - needed for -jN in clean tree .endif Index: projects/clang390-import/share/man/man9/mbuf.9 =================================================================== --- projects/clang390-import/share/man/man9/mbuf.9 (revision 305080) +++ projects/clang390-import/share/man/man9/mbuf.9 (revision 305081) @@ -1,1225 +1,1225 @@ .\" Copyright (c) 2000 FreeBSD Inc. .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL [your name] OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd February 29, 2016 +.Dd August 30, 2016 .Dt MBUF 9 .Os .\" .Sh NAME .Nm mbuf .Nd "memory management in the kernel IPC subsystem" .\" .Sh SYNOPSIS .In sys/param.h .In sys/systm.h .In sys/mbuf.h .\" .Ss Mbuf allocation macros .Fn MGET "struct mbuf *mbuf" "int how" "short type" .Fn MGETHDR "struct mbuf *mbuf" "int how" "short type" .Ft int .Fn MCLGET "struct mbuf *mbuf" "int how" .Fo MEXTADD .Fa "struct mbuf *mbuf" .Fa "caddr_t buf" .Fa "u_int size" .Fa "void (*free)(void *opt_arg1, void *opt_arg2)" .Fa "void *opt_arg1" .Fa "void *opt_arg2" .Fa "short flags" .Fa "int type" .Fc .\" .Ss Mbuf utility macros .Fn mtod "struct mbuf *mbuf" "type" .Fn M_ALIGN "struct mbuf *mbuf" "u_int len" .Fn MH_ALIGN "struct mbuf *mbuf" "u_int len" .Ft int .Fn M_LEADINGSPACE "struct mbuf *mbuf" .Ft int .Fn M_TRAILINGSPACE "struct mbuf *mbuf" .Fn M_MOVE_PKTHDR "struct mbuf *to" "struct mbuf *from" .Fn M_PREPEND "struct mbuf *mbuf" "int len" "int how" .Fn MCHTYPE "struct mbuf *mbuf" "short type" .Ft int .Fn M_WRITABLE "struct mbuf *mbuf" .\" .Ss Mbuf allocation functions .Ft struct mbuf * .Fn m_get "int how" "short type" .Ft struct mbuf * .Fn m_get2 "int size" "int how" "short type" "int flags" .Ft struct mbuf * .Fn m_getm "struct mbuf *orig" "int len" "int how" "short type" .Ft struct mbuf * .Fn m_getjcl "int how" "short type" "int flags" "int size" .Ft struct mbuf * .Fn m_getcl "int how" "short type" "int flags" .Ft struct mbuf * .Fn m_getclr "int how" "short type" .Ft struct mbuf * .Fn m_gethdr "int how" "short type" .Ft struct mbuf * .Fn m_free "struct mbuf *mbuf" .Ft void .Fn m_freem "struct mbuf *mbuf" .\" .Ss Mbuf utility functions .Ft void .Fn m_adj "struct mbuf *mbuf" "int len" .Ft void .Fn m_align "struct mbuf *mbuf" "int len" .Ft int .Fn m_append "struct mbuf *mbuf" "int len" "c_caddr_t cp" .Ft struct mbuf * .Fn m_prepend "struct mbuf *mbuf" "int len" "int how" .Ft struct mbuf * .Fn m_copyup "struct mbuf *mbuf" "int len" "int dstoff" .Ft struct mbuf * .Fn m_pullup "struct mbuf *mbuf" "int len" .Ft struct mbuf * .Fn m_pulldown "struct mbuf *mbuf" "int offset" "int len" "int *offsetp" .Ft struct mbuf * .Fn m_copym "struct mbuf *mbuf" "int offset" "int len" "int how" .Ft struct mbuf * .Fn m_copypacket "struct mbuf *mbuf" "int how" .Ft struct mbuf * .Fn m_dup "struct mbuf *mbuf" "int how" .Ft void .Fn m_copydata "const struct mbuf *mbuf" "int offset" "int len" "caddr_t buf" .Ft void .Fn m_copyback "struct mbuf *mbuf" "int offset" "int len" "caddr_t buf" .Ft struct mbuf * .Fo m_devget .Fa "char *buf" .Fa "int len" .Fa "int offset" .Fa "struct ifnet *ifp" .Fa "void (*copy)(char *from, caddr_t to, u_int len)" .Fc .Ft void .Fn m_cat "struct mbuf *m" "struct mbuf *n" .Ft void .Fn m_catpkt "struct mbuf *m" "struct mbuf *n" .Ft u_int .Fn m_fixhdr "struct mbuf *mbuf" .Ft void .Fn m_dup_pkthdr "struct mbuf *to" "struct mbuf *from" .Ft void .Fn m_move_pkthdr "struct mbuf *to" "struct mbuf *from" .Ft u_int .Fn m_length "struct mbuf *mbuf" "struct mbuf **last" .Ft struct mbuf * .Fn m_split "struct mbuf *mbuf" "int len" "int how" .Ft int .Fn m_apply "struct mbuf *mbuf" "int off" "int len" "int (*f)(void *arg, void *data, u_int len)" "void *arg" .Ft struct mbuf * .Fn m_getptr "struct mbuf *mbuf" "int loc" "int *off" .Ft struct mbuf * .Fn m_defrag "struct mbuf *m0" "int how" .Ft struct mbuf * .Fn m_collapse "struct mbuf *m0" "int how" "int maxfrags" .Ft struct mbuf * .Fn m_unshare "struct mbuf *m0" "int how" .\" .Sh DESCRIPTION An .Vt mbuf is a basic unit of memory management in the kernel IPC subsystem. Network packets and socket buffers are stored in .Vt mbufs . A network packet may span multiple .Vt mbufs arranged into a .Vt mbuf chain (linked list), which allows adding or trimming network headers with little overhead. .Pp While a developer should not bother with .Vt mbuf internals without serious reason in order to avoid incompatibilities with future changes, it is useful to understand the general structure of an .Vt mbuf . .Pp An .Vt mbuf consists of a variable-sized header and a small internal buffer for data. The total size of an .Vt mbuf , .Dv MSIZE , is a constant defined in .In sys/param.h . The .Vt mbuf header includes: .Bl -tag -width "m_nextpkt" -offset indent .It Va m_next .Pq Vt struct mbuf * A pointer to the next .Vt mbuf in the .Vt mbuf chain . .It Va m_nextpkt .Pq Vt struct mbuf * A pointer to the next .Vt mbuf chain in the queue. .It Va m_data .Pq Vt caddr_t A pointer to data attached to this .Vt mbuf . .It Va m_len .Pq Vt int The length of the data. .It Va m_type .Pq Vt short The type of the data. .It Va m_flags .Pq Vt int The .Vt mbuf flags. .El .Pp The .Vt mbuf flag bits are defined as follows: .Bd -literal /* mbuf flags */ #define M_EXT 0x00000001 /* has associated external storage */ #define M_PKTHDR 0x00000002 /* start of record */ #define M_EOR 0x00000004 /* end of record */ #define M_RDONLY 0x00000008 /* associated data marked read-only */ #define M_PROTO1 0x00001000 /* protocol-specific */ #define M_PROTO2 0x00002000 /* protocol-specific */ #define M_PROTO3 0x00004000 /* protocol-specific */ #define M_PROTO4 0x00008000 /* protocol-specific */ #define M_PROTO5 0x00010000 /* protocol-specific */ #define M_PROTO6 0x00020000 /* protocol-specific */ #define M_PROTO7 0x00040000 /* protocol-specific */ #define M_PROTO8 0x00080000 /* protocol-specific */ #define M_PROTO9 0x00100000 /* protocol-specific */ #define M_PROTO10 0x00200000 /* protocol-specific */ #define M_PROTO11 0x00400000 /* protocol-specific */ #define M_PROTO12 0x00800000 /* protocol-specific */ /* mbuf pkthdr flags (also stored in m_flags) */ #define M_BCAST 0x00000010 /* send/received as link-level broadcast */ #define M_MCAST 0x00000020 /* send/received as link-level multicast */ .Ed .Pp The available .Vt mbuf types are defined as follows: .Bd -literal /* mbuf types */ #define MT_DATA 1 /* dynamic (data) allocation */ #define MT_HEADER MT_DATA /* packet header */ #define MT_SONAME 8 /* socket name */ #define MT_CONTROL 14 /* extra-data protocol message */ #define MT_OOBDATA 15 /* expedited data */ .Ed .Pp The available external buffer types are defined as follows: .Bd -literal /* external buffer types */ #define EXT_CLUSTER 1 /* mbuf cluster */ #define EXT_SFBUF 2 /* sendfile(2)'s sf_bufs */ #define EXT_JUMBOP 3 /* jumbo cluster 4096 bytes */ #define EXT_JUMBO9 4 /* jumbo cluster 9216 bytes */ #define EXT_JUMBO16 5 /* jumbo cluster 16184 bytes */ #define EXT_PACKET 6 /* mbuf+cluster from packet zone */ -#define EXT_MBUF 7 /* external mbuf reference (M_IOVEC) */ +#define EXT_MBUF 7 /* external mbuf reference */ #define EXT_NET_DRV 252 /* custom ext_buf provided by net driver(s) */ #define EXT_MOD_TYPE 253 /* custom module's ext_buf type */ #define EXT_DISPOSABLE 254 /* can throw this buffer away w/page flipping */ #define EXT_EXTREF 255 /* has externally maintained ref_cnt ptr */ .Ed .Pp If the .Dv M_PKTHDR flag is set, a .Vt struct pkthdr Va m_pkthdr is added to the .Vt mbuf header. It contains a pointer to the interface the packet has been received from .Pq Vt struct ifnet Va *rcvif , and the total packet length .Pq Vt int Va len . Optionally, it may also contain an attached list of packet tags .Pq Vt "struct m_tag" . See .Xr mbuf_tags 9 for details. Fields used in offloading checksum calculation to the hardware are kept in .Va m_pkthdr as well. See .Sx HARDWARE-ASSISTED CHECKSUM CALCULATION for details. .Pp If small enough, data is stored in the internal data buffer of an .Vt mbuf . If the data is sufficiently large, another .Vt mbuf may be added to the .Vt mbuf chain , or external storage may be associated with the .Vt mbuf . .Dv MHLEN bytes of data can fit into an .Vt mbuf with the .Dv M_PKTHDR flag set, .Dv MLEN bytes can otherwise. .Pp If external storage is being associated with an .Vt mbuf , the .Va m_ext header is added at the cost of losing the internal data buffer. It includes a pointer to external storage, the size of the storage, a pointer to a function used for freeing the storage, a pointer to an optional argument that can be passed to the function, and a pointer to a reference counter. An .Vt mbuf using external storage has the .Dv M_EXT flag set. .Pp The system supplies a macro for allocating the desired external storage buffer, .Dv MEXTADD . .Pp The allocation and management of the reference counter is handled by the subsystem. .Pp The system also supplies a default type of external storage buffer called an .Vt mbuf cluster . .Vt Mbuf clusters can be allocated and configured with the use of the .Dv MCLGET macro. Each .Vt mbuf cluster is .Dv MCLBYTES in size, where MCLBYTES is a machine-dependent constant. The system defines an advisory macro .Dv MINCLSIZE , which is the smallest amount of data to put into an .Vt mbuf cluster . It is equal to .Dv MHLEN plus one. It is typically preferable to store data into the data region of an .Vt mbuf , if size permits, as opposed to allocating a separate .Vt mbuf cluster to hold the same data. .\" .Ss Macros and Functions There are numerous predefined macros and functions that provide the developer with common utilities. .\" .Bl -ohang -offset indent .It Fn mtod mbuf type Convert an .Fa mbuf pointer to a data pointer. The macro expands to the data pointer cast to the specified .Fa type . .Sy Note : It is advisable to ensure that there is enough contiguous data in .Fa mbuf . See .Fn m_pullup for details. .It Fn MGET mbuf how type Allocate an .Vt mbuf and initialize it to contain internal data. .Fa mbuf will point to the allocated .Vt mbuf on success, or be set to .Dv NULL on failure. The .Fa how argument is to be set to .Dv M_WAITOK or .Dv M_NOWAIT . It specifies whether the caller is willing to block if necessary. A number of other functions and macros related to .Vt mbufs have the same argument because they may at some point need to allocate new .Vt mbufs . .It Fn MGETHDR mbuf how type Allocate an .Vt mbuf and initialize it to contain a packet header and internal data. See .Fn MGET for details. .It Fn MEXTADD mbuf buf size free opt_arg1 opt_arg2 flags type Associate externally managed data with .Fa mbuf . Any internal data contained in the mbuf will be discarded, and the .Dv M_EXT flag will be set. The .Fa buf and .Fa size arguments are the address and length, respectively, of the data. The .Fa free argument points to a function which will be called to free the data when the mbuf is freed; it is only used if .Fa type is .Dv EXT_EXTREF . The .Fa opt_arg1 and .Fa opt_arg2 arguments will be passed unmodified to .Fa free . The .Fa flags argument specifies additional .Vt mbuf flags; it is not necessary to specify .Dv M_EXT . Finally, the .Fa type argument specifies the type of external data, which controls how it will be disposed of when the .Vt mbuf is freed. In most cases, the correct value is .Dv EXT_EXTREF . .It Fn MCLGET mbuf how Allocate and attach an .Vt mbuf cluster to .Fa mbuf . On success, a non-zero value returned; otherwise, 0. Historically, consumers would check for success by testing the .Dv M_EXT flag on the mbuf, but this is now discouraged to avoid unnecessary awareness of the implementation of external storage in protocol stacks and device drivers. .It Fn M_ALIGN mbuf len Set the pointer .Fa mbuf->m_data to place an object of the size .Fa len at the end of the internal data area of .Fa mbuf , long word aligned. Applicable only if .Fa mbuf is newly allocated with .Fn MGET or .Fn m_get . .It Fn MH_ALIGN mbuf len Serves the same purpose as .Fn M_ALIGN does, but only for .Fa mbuf newly allocated with .Fn MGETHDR or .Fn m_gethdr , or initialized by .Fn m_dup_pkthdr or .Fn m_move_pkthdr . .It Fn m_align mbuf len Services the same purpose as .Fn M_ALIGN but handles any type of mbuf. .It Fn M_LEADINGSPACE mbuf Returns the number of bytes available before the beginning of data in .Fa mbuf . .It Fn M_TRAILINGSPACE mbuf Returns the number of bytes available after the end of data in .Fa mbuf . .It Fn M_PREPEND mbuf len how This macro operates on an .Vt mbuf chain . It is an optimized wrapper for .Fn m_prepend that can make use of possible empty space before data (e.g.\& left after trimming of a link-layer header). The new .Vt mbuf chain pointer or .Dv NULL is in .Fa mbuf after the call. .It Fn M_MOVE_PKTHDR to from Using this macro is equivalent to calling .Fn m_move_pkthdr to from . .It Fn M_WRITABLE mbuf This macro will evaluate true if .Fa mbuf is not marked .Dv M_RDONLY and if either .Fa mbuf does not contain external storage or, if it does, then if the reference count of the storage is not greater than 1. The .Dv M_RDONLY flag can be set in .Fa mbuf->m_flags . This can be achieved during setup of the external storage, by passing the .Dv M_RDONLY bit as a .Fa flags argument to the .Fn MEXTADD macro, or can be directly set in individual .Vt mbufs . .It Fn MCHTYPE mbuf type Change the type of .Fa mbuf to .Fa type . This is a relatively expensive operation and should be avoided. .El .Pp The functions are: .Bl -ohang -offset indent .It Fn m_get how type A function version of .Fn MGET for non-critical paths. .It Fn m_get2 size how type flags Allocate an .Vt mbuf with enough space to hold specified amount of data. .It Fn m_getm orig len how type Allocate .Fa len bytes worth of .Vt mbufs and .Vt mbuf clusters if necessary and append the resulting allocated .Vt mbuf chain to the .Vt mbuf chain .Fa orig , if it is .No non- Ns Dv NULL . If the allocation fails at any point, free whatever was allocated and return .Dv NULL . If .Fa orig is .No non- Ns Dv NULL , it will not be freed. It is possible to use .Fn m_getm to either append .Fa len bytes to an existing .Vt mbuf or .Vt mbuf chain (for example, one which may be sitting in a pre-allocated ring) or to simply perform an all-or-nothing .Vt mbuf and .Vt mbuf cluster allocation. .It Fn m_gethdr how type A function version of .Fn MGETHDR for non-critical paths. .It Fn m_getcl how type flags Fetch an .Vt mbuf with a .Vt mbuf cluster attached to it. If one of the allocations fails, the entire allocation fails. This routine is the preferred way of fetching both the .Vt mbuf and .Vt mbuf cluster together, as it avoids having to unlock/relock between allocations. Returns .Dv NULL on failure. .It Fn m_getjcl how type flags size This is like .Fn m_getcl but it the size of the cluster allocated will be large enough for .Fa size bytes. .It Fn m_getclr how type Allocate an .Vt mbuf and zero out the data region. .It Fn m_free mbuf Frees .Vt mbuf . Returns .Va m_next of the freed .Vt mbuf . .El .Pp The functions below operate on .Vt mbuf chains . .Bl -ohang -offset indent .It Fn m_freem mbuf Free an entire .Vt mbuf chain , including any external storage. .\" .It Fn m_adj mbuf len Trim .Fa len bytes from the head of an .Vt mbuf chain if .Fa len is positive, from the tail otherwise. .\" .It Fn m_append mbuf len cp Append .Vt len bytes of data .Vt cp to the .Vt mbuf chain . Extend the mbuf chain if the new data does not fit in existing space. .\" .It Fn m_prepend mbuf len how Allocate a new .Vt mbuf and prepend it to the .Vt mbuf chain , handle .Dv M_PKTHDR properly. .Sy Note : It does not allocate any .Vt mbuf clusters , so .Fa len must be less than .Dv MLEN or .Dv MHLEN , depending on the .Dv M_PKTHDR flag setting. .\" .It Fn m_copyup mbuf len dstoff Similar to .Fn m_pullup but copies .Fa len bytes of data into a new mbuf at .Fa dstoff bytes into the mbuf. The .Fa dstoff argument aligns the data and leaves room for a link layer header. Returns the new .Vt mbuf chain on success, and frees the .Vt mbuf chain and returns .Dv NULL on failure. .Sy Note : The function does not allocate .Vt mbuf clusters , so .Fa len + dstoff must be less than .Dv MHLEN . .\" .It Fn m_pullup mbuf len Arrange that the first .Fa len bytes of an .Vt mbuf chain are contiguous and lay in the data area of .Fa mbuf , so they are accessible with .Fn mtod mbuf type . It is important to remember that this may involve reallocating some mbufs and moving data so all pointers referencing data within the old mbuf chain must be recalculated or made invalid. Return the new .Vt mbuf chain on success, .Dv NULL on failure (the .Vt mbuf chain is freed in this case). .Sy Note : It does not allocate any .Vt mbuf clusters , so .Fa len must be less than or equal to .Dv MHLEN . .\" .It Fn m_pulldown mbuf offset len offsetp Arrange that .Fa len bytes between .Fa offset and .Fa offset + len in the .Vt mbuf chain are contiguous and lay in the data area of .Fa mbuf , so they are accessible with .Fn mtod mbuf type . .Fa len must be smaller than, or equal to, the size of an .Vt mbuf cluster . Return a pointer to an intermediate .Vt mbuf in the chain containing the requested region; the offset in the data region of the .Vt mbuf chain to the data contained in the returned mbuf is stored in .Fa *offsetp . If .Fa offsetp is NULL, the region may be accessed using .Fn mtod mbuf type . If .Fa offsetp is non-NULL, the region may be accessed using .Fn mtod mbuf uint8_t + *offsetp. The region of the mbuf chain between its beginning and .Fa offset is not modified, therefore it is safe to hold pointers to data within this region before calling .Fn m_pulldown . .\" .It Fn m_copym mbuf offset len how Make a copy of an .Vt mbuf chain starting .Fa offset bytes from the beginning, continuing for .Fa len bytes. If .Fa len is .Dv M_COPYALL , copy to the end of the .Vt mbuf chain . .Sy Note : The copy is read-only, because the .Vt mbuf clusters are not copied, only their reference counts are incremented. .\" .It Fn m_copypacket mbuf how Copy an entire packet including header, which must be present. This is an optimized version of the common case .Fn m_copym mbuf 0 M_COPYALL how . .Sy Note : the copy is read-only, because the .Vt mbuf clusters are not copied, only their reference counts are incremented. .\" .It Fn m_dup mbuf how Copy a packet header .Vt mbuf chain into a completely new .Vt mbuf chain , including copying any .Vt mbuf clusters . Use this instead of .Fn m_copypacket when you need a writable copy of an .Vt mbuf chain . .\" .It Fn m_copydata mbuf offset len buf Copy data from an .Vt mbuf chain starting .Fa off bytes from the beginning, continuing for .Fa len bytes, into the indicated buffer .Fa buf . .\" .It Fn m_copyback mbuf offset len buf Copy .Fa len bytes from the buffer .Fa buf back into the indicated .Vt mbuf chain , starting at .Fa offset bytes from the beginning of the .Vt mbuf chain , extending the .Vt mbuf chain if necessary. .Sy Note : It does not allocate any .Vt mbuf clusters , just adds .Vt mbufs to the .Vt mbuf chain . It is safe to set .Fa offset beyond the current .Vt mbuf chain end: zeroed .Vt mbufs will be allocated to fill the space. .\" .It Fn m_length mbuf last Return the length of the .Vt mbuf chain , and optionally a pointer to the last .Vt mbuf . .\" .It Fn m_dup_pkthdr to from how Upon the function's completion, the .Vt mbuf .Fa to will contain an identical copy of .Fa from->m_pkthdr and the per-packet attributes found in the .Vt mbuf chain .Fa from . The .Vt mbuf .Fa from must have the flag .Dv M_PKTHDR initially set, and .Fa to must be empty on entry. .\" .It Fn m_move_pkthdr to from Move .Va m_pkthdr and the per-packet attributes from the .Vt mbuf chain .Fa from to the .Vt mbuf .Fa to . The .Vt mbuf .Fa from must have the flag .Dv M_PKTHDR initially set, and .Fa to must be empty on entry. Upon the function's completion, .Fa from will have the flag .Dv M_PKTHDR and the per-packet attributes cleared. .\" .It Fn m_fixhdr mbuf Set the packet-header length to the length of the .Vt mbuf chain . .\" .It Fn m_devget buf len offset ifp copy Copy data from a device local memory pointed to by .Fa buf to an .Vt mbuf chain . The copy is done using a specified copy routine .Fa copy , or .Fn bcopy if .Fa copy is .Dv NULL . .\" .It Fn m_cat m n Concatenate .Fa n to .Fa m . Both .Vt mbuf chains must be of the same type. .Fa n is not guaranteed to be valid after .Fn m_cat returns. .Fn m_cat does not update any packet header fields or free mbuf tags. .\" .It Fn m_catpkt m n A variant of .Fn m_cat that operates on packets. Both .Fa m and .Fa n must contain packet headers. .Fa n is not guaranteed to be valid after .Fn m_catpkt returns. .\" .It Fn m_split mbuf len how Partition an .Vt mbuf chain in two pieces, returning the tail: all but the first .Fa len bytes. In case of failure, it returns .Dv NULL and attempts to restore the .Vt mbuf chain to its original state. .\" .It Fn m_apply mbuf off len f arg Apply a function to an .Vt mbuf chain , at offset .Fa off , for length .Fa len bytes. Typically used to avoid calls to .Fn m_pullup which would otherwise be unnecessary or undesirable. .Fa arg is a convenience argument which is passed to the callback function .Fa f . .Pp Each time .Fn f is called, it will be passed .Fa arg , a pointer to the .Fa data in the current mbuf, and the length .Fa len of the data in this mbuf to which the function should be applied. .Pp The function should return zero to indicate success; otherwise, if an error is indicated, then .Fn m_apply will return the error and stop iterating through the .Vt mbuf chain . .\" .It Fn m_getptr mbuf loc off Return a pointer to the mbuf containing the data located at .Fa loc bytes from the beginning of the .Vt mbuf chain . The corresponding offset into the mbuf will be stored in .Fa *off . .It Fn m_defrag m0 how Defragment an mbuf chain, returning the shortest possible chain of mbufs and clusters. If allocation fails and this can not be completed, .Dv NULL will be returned and the original chain will be unchanged. Upon success, the original chain will be freed and the new chain will be returned. .Fa how should be either .Dv M_WAITOK or .Dv M_NOWAIT , depending on the caller's preference. .Pp This function is especially useful in network drivers, where certain long mbuf chains must be shortened before being added to TX descriptor lists. .It Fn m_collapse m0 how maxfrags Defragment an mbuf chain, returning a chain of at most .Fa maxfrags mbufs and clusters. If allocation fails or the chain cannot be collapsed as requested, .Dv NULL will be returned, with the original chain possibly modified. As with .Fn m_defrag , .Fa how should be one of .Dv M_WAITOK or .Dv M_NOWAIT . .It Fn m_unshare m0 how Create a version of the specified mbuf chain whose contents can be safely modified without affecting other users. If allocation fails and this operation can not be completed, .Dv NULL will be returned. The original mbuf chain is always reclaimed and the reference count of any shared mbuf clusters is decremented. .Fa how should be either .Dv M_WAITOK or .Dv M_NOWAIT , depending on the caller's preference. As a side-effect of this process the returned mbuf chain may be compacted. .Pp This function is especially useful in the transmit path of network code, when data must be encrypted or otherwise altered prior to transmission. .El .Sh HARDWARE-ASSISTED CHECKSUM CALCULATION This section currently applies to TCP/IP only. In order to save the host CPU resources, computing checksums is offloaded to the network interface hardware if possible. The .Va m_pkthdr member of the leading .Vt mbuf of a packet contains two fields used for that purpose, .Vt int Va csum_flags and .Vt int Va csum_data . The meaning of those fields depends on the direction a packet flows in, and on whether the packet is fragmented. Henceforth, .Va csum_flags or .Va csum_data of a packet will denote the corresponding field of the .Va m_pkthdr member of the leading .Vt mbuf in the .Vt mbuf chain containing the packet. .Pp On output, checksum offloading is attempted after the outgoing interface has been determined for a packet. The interface-specific field .Va ifnet.if_data.ifi_hwassist (see .Xr ifnet 9 ) is consulted for the capabilities of the interface to assist in computing checksums. The .Va csum_flags field of the packet header is set to indicate which actions the interface is supposed to perform on it. The actions unsupported by the network interface are done in the software prior to passing the packet down to the interface driver; such actions will never be requested through .Va csum_flags . .Pp The flags demanding a particular action from an interface are as follows: .Bl -tag -width ".Dv CSUM_TCP" -offset indent .It Dv CSUM_IP The IP header checksum is to be computed and stored in the corresponding field of the packet. The hardware is expected to know the format of an IP header to determine the offset of the IP checksum field. .It Dv CSUM_TCP The TCP checksum is to be computed. (See below.) .It Dv CSUM_UDP The UDP checksum is to be computed. (See below.) .El .Pp Should a TCP or UDP checksum be offloaded to the hardware, the field .Va csum_data will contain the byte offset of the checksum field relative to the end of the IP header. In this case, the checksum field will be initially set by the TCP/IP module to the checksum of the pseudo header defined by the TCP and UDP specifications. .Pp On input, an interface indicates the actions it has performed on a packet by setting one or more of the following flags in .Va csum_flags associated with the packet: .Bl -tag -width ".Dv CSUM_IP_CHECKED" -offset indent .It Dv CSUM_IP_CHECKED The IP header checksum has been computed. .It Dv CSUM_IP_VALID The IP header has a valid checksum. This flag can appear only in combination with .Dv CSUM_IP_CHECKED . .It Dv CSUM_DATA_VALID The checksum of the data portion of the IP packet has been computed and stored in the field .Va csum_data in network byte order. .It Dv CSUM_PSEUDO_HDR Can be set only along with .Dv CSUM_DATA_VALID to indicate that the IP data checksum found in .Va csum_data allows for the pseudo header defined by the TCP and UDP specifications. Otherwise the checksum of the pseudo header must be calculated by the host CPU and added to .Va csum_data to obtain the final checksum to be used for TCP or UDP validation purposes. .El .Pp If a particular network interface just indicates success or failure of TCP or UDP checksum validation without returning the exact value of the checksum to the host CPU, its driver can mark .Dv CSUM_DATA_VALID and .Dv CSUM_PSEUDO_HDR in .Va csum_flags , and set .Va csum_data to .Li 0xFFFF hexadecimal to indicate a valid checksum. It is a peculiarity of the algorithm used that the Internet checksum calculated over any valid packet will be .Li 0xFFFF as long as the original checksum field is included. .Sh STRESS TESTING When running a kernel compiled with the option .Dv MBUF_STRESS_TEST , the following .Xr sysctl 8 Ns -controlled options may be used to create various failure/extreme cases for testing of network drivers and other parts of the kernel that rely on .Vt mbufs . .Bl -tag -width ident .It Va net.inet.ip.mbuf_frag_size Causes .Fn ip_output to fragment outgoing .Vt mbuf chains into fragments of the specified size. Setting this variable to 1 is an excellent way to test the long .Vt mbuf chain handling ability of network drivers. .It Va kern.ipc.m_defragrandomfailures Causes the function .Fn m_defrag to randomly fail, returning .Dv NULL . Any piece of code which uses .Fn m_defrag should be tested with this feature. .El .Sh RETURN VALUES See above. .Sh SEE ALSO .Xr ifnet 9 , .Xr mbuf_tags 9 .Sh HISTORY .\" Please correct me if I'm wrong .Vt Mbufs appeared in an early version of .Bx . Besides being used for network packets, they were used to store various dynamic structures, such as routing table entries, interface addresses, protocol control blocks, etc. In more recent .Fx use of .Vt mbufs is almost entirely limited to packet storage, with .Xr uma 9 zones being used directly to store other network-related memory. .Pp Historically, the .Vt mbuf allocator has been a special-purpose memory allocator able to run in interrupt contexts and allocating from a special kernel address space map. As of .Fx 5.3 , the .Vt mbuf allocator is a wrapper around .Xr uma 9 , allowing caching of .Vt mbufs , clusters, and .Vt mbuf + cluster pairs in per-CPU caches, as well as bringing other benefits of slab allocation. .Sh AUTHORS The original .Nm manual page was written by .An Yar Tikhiy . The .Xr uma 9 .Vt mbuf allocator was written by .An Bosko Milekic . Index: projects/clang390-import/sys/arm/allwinner/aw_wdog.c =================================================================== --- projects/clang390-import/sys/arm/allwinner/aw_wdog.c (revision 305080) +++ projects/clang390-import/sys/arm/allwinner/aw_wdog.c (revision 305081) @@ -1,259 +1,277 @@ /*- * Copyright (c) 2013 Oleksandr Tymoshenko * Copyright (c) 2016 Emmanuel Vadot * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define READ(_sc, _r) bus_read_4((_sc)->res, (_r)) #define WRITE(_sc, _r, _v) bus_write_4((_sc)->res, (_r), (_v)) #define A10_WDOG_CTRL 0x00 #define A31_WDOG_CTRL 0x10 #define WDOG_CTRL_RESTART (1 << 0) +#define A31_WDOG_CTRL_KEY (0xa57 << 1) #define A10_WDOG_MODE 0x04 #define A31_WDOG_MODE 0x18 #define A10_WDOG_MODE_INTVL_SHIFT 3 #define A31_WDOG_MODE_INTVL_SHIFT 4 #define A10_WDOG_MODE_RST_EN (1 << 1) #define WDOG_MODE_EN (1 << 0) #define A31_WDOG_CONFIG 0x14 #define A31_WDOG_CONFIG_RST_EN_SYSTEM (1 << 0) #define A31_WDOG_CONFIG_RST_EN_INT (2 << 0) struct aw_wdog_interval { uint64_t milliseconds; unsigned int value; }; struct aw_wdog_interval wd_intervals[] = { { 500, 0 }, { 1000, 1 }, { 2000, 2 }, { 3000, 3 }, { 4000, 4 }, { 5000, 5 }, { 6000, 6 }, { 8000, 7 }, { 10000, 8 }, { 12000, 9 }, { 14000, 10 }, { 16000, 11 }, { 0, 0 } /* sentinel */ }; static struct aw_wdog_softc *aw_wdog_sc = NULL; struct aw_wdog_softc { device_t dev; struct resource * res; struct mtx mtx; uint8_t wdog_ctrl; + uint32_t wdog_ctrl_key; uint8_t wdog_mode; uint8_t wdog_mode_intvl_shift; uint8_t wdog_mode_en; uint8_t wdog_config; uint8_t wdog_config_value; }; #define A10_WATCHDOG 1 #define A31_WATCHDOG 2 static struct ofw_compat_data compat_data[] = { {"allwinner,sun4i-a10-wdt", A10_WATCHDOG}, {"allwinner,sun6i-a31-wdt", A31_WATCHDOG}, {NULL, 0} }; -static void aw_wdog_watchdog_fn(void *private, u_int cmd, int *error); +static void aw_wdog_watchdog_fn(void *, u_int, int *); +static void aw_wdog_shutdown_fn(void *, int); static int aw_wdog_probe(device_t dev) { struct aw_wdog_softc *sc; sc = device_get_softc(dev); if (!ofw_bus_status_okay(dev)) return (ENXIO); switch (ofw_bus_search_compatible(dev, compat_data)->ocd_data) { case A10_WATCHDOG: device_set_desc(dev, "Allwinner A10 Watchdog"); return (BUS_PROBE_DEFAULT); case A31_WATCHDOG: device_set_desc(dev, "Allwinner A31 Watchdog"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int aw_wdog_attach(device_t dev) { struct aw_wdog_softc *sc; int rid; if (aw_wdog_sc != NULL) return (ENXIO); sc = device_get_softc(dev); sc->dev = dev; rid = 0; sc->res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->res == NULL) { device_printf(dev, "could not allocate memory resource\n"); return (ENXIO); } aw_wdog_sc = sc; switch (ofw_bus_search_compatible(dev, compat_data)->ocd_data) { case A10_WATCHDOG: sc->wdog_ctrl = A10_WDOG_CTRL; sc->wdog_mode = A10_WDOG_MODE; sc->wdog_mode_intvl_shift = A10_WDOG_MODE_INTVL_SHIFT; sc->wdog_mode_en = A10_WDOG_MODE_RST_EN | WDOG_MODE_EN; break; case A31_WATCHDOG: sc->wdog_ctrl = A31_WDOG_CTRL; + sc->wdog_ctrl_key = A31_WDOG_CTRL_KEY; sc->wdog_mode = A31_WDOG_MODE; sc->wdog_mode_intvl_shift = A31_WDOG_MODE_INTVL_SHIFT; sc->wdog_mode_en = WDOG_MODE_EN; sc->wdog_config = A31_WDOG_CONFIG; sc->wdog_config_value = A31_WDOG_CONFIG_RST_EN_SYSTEM; break; default: bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res); return (ENXIO); } mtx_init(&sc->mtx, "AW Watchdog", "aw_wdog", MTX_DEF); EVENTHANDLER_REGISTER(watchdog_list, aw_wdog_watchdog_fn, sc, 0); + EVENTHANDLER_REGISTER(shutdown_final, aw_wdog_shutdown_fn, sc, + SHUTDOWN_PRI_LAST - 1); + return (0); } static void aw_wdog_watchdog_fn(void *private, u_int cmd, int *error) { struct aw_wdog_softc *sc; uint64_t ms; int i; sc = private; mtx_lock(&sc->mtx); cmd &= WD_INTERVAL; if (cmd > 0) { ms = ((uint64_t)1 << (cmd & WD_INTERVAL)) / 1000000; i = 0; while (wd_intervals[i].milliseconds && (ms > wd_intervals[i].milliseconds)) i++; if (wd_intervals[i].milliseconds) { WRITE(sc, sc->wdog_mode, (wd_intervals[i].value << sc->wdog_mode_intvl_shift) | sc->wdog_mode_en); - WRITE(sc, sc->wdog_ctrl, WDOG_CTRL_RESTART); + WRITE(sc, sc->wdog_ctrl, + WDOG_CTRL_RESTART | sc->wdog_ctrl_key); if (sc->wdog_config) WRITE(sc, sc->wdog_config, sc->wdog_config_value); *error = 0; } else { /* * Can't arm * disable watchdog as watchdog(9) requires */ device_printf(sc->dev, "Can't arm, timeout is more than 16 sec\n"); mtx_unlock(&sc->mtx); WRITE(sc, sc->wdog_mode, 0); return; } } else WRITE(sc, sc->wdog_mode, 0); mtx_unlock(&sc->mtx); } +static void +aw_wdog_shutdown_fn(void *private, int howto) +{ + if ((howto & (RB_POWEROFF|RB_HALT)) == 0) + aw_wdog_watchdog_reset(); +} + void aw_wdog_watchdog_reset() { if (aw_wdog_sc == NULL) { printf("Reset: watchdog device has not been initialized\n"); return; } WRITE(aw_wdog_sc, aw_wdog_sc->wdog_mode, (wd_intervals[0].value << aw_wdog_sc->wdog_mode_intvl_shift) | aw_wdog_sc->wdog_mode_en); if (aw_wdog_sc->wdog_config) WRITE(aw_wdog_sc, aw_wdog_sc->wdog_config, aw_wdog_sc->wdog_config_value); + WRITE(aw_wdog_sc, aw_wdog_sc->wdog_ctrl, + WDOG_CTRL_RESTART | aw_wdog_sc->wdog_ctrl_key); while(1) ; } static device_method_t aw_wdog_methods[] = { DEVMETHOD(device_probe, aw_wdog_probe), DEVMETHOD(device_attach, aw_wdog_attach), DEVMETHOD_END }; static driver_t aw_wdog_driver = { "aw_wdog", aw_wdog_methods, sizeof(struct aw_wdog_softc), }; static devclass_t aw_wdog_devclass; DRIVER_MODULE(aw_wdog, simplebus, aw_wdog_driver, aw_wdog_devclass, 0, 0); Index: projects/clang390-import/sys/arm/ti/ti_pruss.c =================================================================== --- projects/clang390-import/sys/arm/ti/ti_pruss.c (revision 305080) +++ projects/clang390-import/sys/arm/ti/ti_pruss.c (revision 305081) @@ -1,320 +1,322 @@ /*- * Copyright (c) 2013 Rui Paulo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEBUG #define DPRINTF(fmt, ...) do { \ printf("%s: ", __func__); \ printf(fmt, __VA_ARGS__); \ } while (0) #else #define DPRINTF(fmt, ...) #endif static device_probe_t ti_pruss_probe; static device_attach_t ti_pruss_attach; static device_detach_t ti_pruss_detach; static void ti_pruss_intr(void *); static d_open_t ti_pruss_open; static d_mmap_t ti_pruss_mmap; static void ti_pruss_kq_read_detach(struct knote *); static int ti_pruss_kq_read_event(struct knote *, long); static d_kqfilter_t ti_pruss_kqfilter; #define TI_PRUSS_IRQS 8 struct ti_pruss_softc { struct mtx sc_mtx; struct resource *sc_mem_res; struct resource *sc_irq_res[TI_PRUSS_IRQS]; void *sc_intr[TI_PRUSS_IRQS]; bus_space_tag_t sc_bt; bus_space_handle_t sc_bh; struct cdev *sc_pdev; struct selinfo sc_selinfo; }; static struct cdevsw ti_pruss_cdevsw = { .d_version = D_VERSION, .d_name = "ti_pruss", .d_open = ti_pruss_open, .d_mmap = ti_pruss_mmap, .d_kqfilter = ti_pruss_kqfilter, }; static device_method_t ti_pruss_methods[] = { DEVMETHOD(device_probe, ti_pruss_probe), DEVMETHOD(device_attach, ti_pruss_attach), DEVMETHOD(device_detach, ti_pruss_detach), DEVMETHOD_END }; static driver_t ti_pruss_driver = { "ti_pruss", ti_pruss_methods, sizeof(struct ti_pruss_softc) }; static devclass_t ti_pruss_devclass; DRIVER_MODULE(ti_pruss, simplebus, ti_pruss_driver, ti_pruss_devclass, 0, 0); static struct resource_spec ti_pruss_irq_spec[] = { { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE }, { SYS_RES_IRQ, 2, RF_ACTIVE }, { SYS_RES_IRQ, 3, RF_ACTIVE }, { SYS_RES_IRQ, 4, RF_ACTIVE }, { SYS_RES_IRQ, 5, RF_ACTIVE }, { SYS_RES_IRQ, 6, RF_ACTIVE }, { SYS_RES_IRQ, 7, RF_ACTIVE }, { -1, 0, 0 } }; CTASSERT(TI_PRUSS_IRQS == nitems(ti_pruss_irq_spec) - 1); static struct ti_pruss_irq_arg { int irq; struct ti_pruss_softc *sc; } ti_pruss_irq_args[TI_PRUSS_IRQS]; static __inline uint32_t ti_pruss_reg_read(struct ti_pruss_softc *sc, uint32_t reg) { return (bus_space_read_4(sc->sc_bt, sc->sc_bh, reg)); } static __inline void ti_pruss_reg_write(struct ti_pruss_softc *sc, uint32_t reg, uint32_t val) { bus_space_write_4(sc->sc_bt, sc->sc_bh, reg, val); } static int ti_pruss_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "ti,pruss-v1") || ofw_bus_is_compatible(dev, "ti,pruss-v2")) { device_set_desc(dev, "TI Programmable Realtime Unit Subsystem"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int ti_pruss_attach(device_t dev) { struct ti_pruss_softc *sc; int rid, i; if (ti_prcm_clk_enable(PRUSS_CLK) != 0) { device_printf(dev, "could not enable PRUSS clock\n"); return (ENXIO); } sc = device_get_softc(dev); rid = 0; mtx_init(&sc->sc_mtx, "TI PRUSS", NULL, MTX_DEF); knlist_init_mtx(&sc->sc_selinfo.si_note, &sc->sc_mtx); sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem_res == NULL) { device_printf(dev, "could not allocate memory resource\n"); return (ENXIO); } sc->sc_bt = rman_get_bustag(sc->sc_mem_res); sc->sc_bh = rman_get_bushandle(sc->sc_mem_res); if (bus_alloc_resources(dev, ti_pruss_irq_spec, sc->sc_irq_res) != 0) { device_printf(dev, "could not allocate interrupt resource\n"); ti_pruss_detach(dev); return (ENXIO); } for (i = 0; i < TI_PRUSS_IRQS; i++) { ti_pruss_irq_args[i].irq = i; ti_pruss_irq_args[i].sc = sc; if (bus_setup_intr(dev, sc->sc_irq_res[i], INTR_MPSAFE | INTR_TYPE_MISC, NULL, ti_pruss_intr, &ti_pruss_irq_args[i], &sc->sc_intr[i]) != 0) { device_printf(dev, "unable to setup the interrupt handler\n"); ti_pruss_detach(dev); return (ENXIO); } } if (ti_pruss_reg_read(sc, PRUSS_AM18XX_INTC) == PRUSS_AM18XX_REV) device_printf(dev, "AM18xx PRU-ICSS\n"); else if (ti_pruss_reg_read(sc, PRUSS_AM33XX_INTC) == PRUSS_AM33XX_REV) device_printf(dev, "AM33xx PRU-ICSS\n"); sc->sc_pdev = make_dev(&ti_pruss_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "pruss%d", device_get_unit(dev)); sc->sc_pdev->si_drv1 = dev; return (0); } static int ti_pruss_detach(device_t dev) { struct ti_pruss_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < TI_PRUSS_IRQS; i++) { if (sc->sc_intr[i]) bus_teardown_intr(dev, sc->sc_irq_res[i], sc->sc_intr[i]); if (sc->sc_irq_res[i]) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->sc_irq_res[i]), sc->sc_irq_res[i]); } knlist_clear(&sc->sc_selinfo.si_note, 0); knlist_destroy(&sc->sc_selinfo.si_note); mtx_destroy(&sc->sc_mtx); if (sc->sc_mem_res) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_mem_res), sc->sc_mem_res); if (sc->sc_pdev) destroy_dev(sc->sc_pdev); return (0); } static void ti_pruss_intr(void *arg) { int val; struct ti_pruss_irq_arg *iap = arg; struct ti_pruss_softc *sc = iap->sc; /* * Interrupts pr1_host_intr[0:7] are mapped to * Host-2 to Host-9 of PRU-ICSS IRQ-controller. */ const int pru_int = iap->irq + 2; const int pru_int_mask = (1 << pru_int); val = ti_pruss_reg_read(sc, PRUSS_AM33XX_INTC + PRUSS_INTC_HIER); DPRINTF("interrupt %p, %d", sc, pru_int); if (!(val & pru_int_mask)) return; ti_pruss_reg_write(sc, PRUSS_AM33XX_INTC + PRUSS_INTC_HIDISR, pru_int); KNOTE_UNLOCKED(&sc->sc_selinfo.si_note, pru_int); } static int ti_pruss_open(struct cdev *cdev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) { return (0); } static int ti_pruss_mmap(struct cdev *cdev, vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr) { device_t dev = cdev->si_drv1; struct ti_pruss_softc *sc = device_get_softc(dev); if (offset > rman_get_size(sc->sc_mem_res)) return (-1); *paddr = rman_get_start(sc->sc_mem_res) + offset; *memattr = VM_MEMATTR_UNCACHEABLE; return (0); } static struct filterops ti_pruss_kq_read = { .f_isfd = 1, .f_detach = ti_pruss_kq_read_detach, .f_event = ti_pruss_kq_read_event, }; static void ti_pruss_kq_read_detach(struct knote *kn) { struct ti_pruss_softc *sc = kn->kn_hook; knlist_remove(&sc->sc_selinfo.si_note, kn, 0); } static int ti_pruss_kq_read_event(struct knote *kn, long hint) { kn->kn_data = hint; return (hint); } static int ti_pruss_kqfilter(struct cdev *cdev, struct knote *kn) { device_t dev = cdev->si_drv1; struct ti_pruss_softc *sc = device_get_softc(dev); switch (kn->kn_filter) { case EVFILT_READ: kn->kn_hook = sc; kn->kn_fop = &ti_pruss_kq_read; + mtx_lock(&sc->sc_mtx); knlist_add(&sc->sc_selinfo.si_note, kn, 1); + mtx_unlock(&sc->sc_mtx); break; default: return (EINVAL); } return (0); } Index: projects/clang390-import/sys/arm64/arm64/pmap.c =================================================================== --- projects/clang390-import/sys/arm64/arm64/pmap.c (revision 305080) +++ projects/clang390-import/sys/arm64/arm64/pmap.c (revision 305081) @@ -1,4741 +1,4744 @@ /*- * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * Copyright (c) 2003 Peter Wemm * All rights reserved. * Copyright (c) 2005-2010 Alan L. Cox * All rights reserved. * Copyright (c) 2014 Andrew Turner * All rights reserved. * Copyright (c) 2014-2016 The FreeBSD Foundation * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * This software was developed by Andrew Turner under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ /*- * Copyright (c) 2003 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by Jake Burkholder, * Safeport Network Services, and Network Associates Laboratories, the * Security Research Division of Network Associates, Inc. under * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA * CHATS research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * Since the information managed by this module is * also stored by the logical address mapping module, * this module may throw away valid virtual-to-physical * mappings at almost any time. However, invalidations * of virtual-to-physical mappings must be done as * requested. * * In order to cope with hardware architectures which * make virtual-to-physical map invalidates expensive, * this module may delay invalidate or reduced protection * operations until such time as they are actually * necessary. This module is given full information as * to which processors are currently using which maps, * and to when physical maps must be made correct. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define NL0PG (PAGE_SIZE/(sizeof (pd_entry_t))) #define NL1PG (PAGE_SIZE/(sizeof (pd_entry_t))) #define NL2PG (PAGE_SIZE/(sizeof (pd_entry_t))) #define NL3PG (PAGE_SIZE/(sizeof (pt_entry_t))) #define NUL0E L0_ENTRIES #define NUL1E (NUL0E * NL1PG) #define NUL2E (NUL1E * NL2PG) #if !defined(DIAGNOSTIC) #ifdef __GNUC_GNU_INLINE__ #define PMAP_INLINE __attribute__((__gnu_inline__)) inline #else #define PMAP_INLINE extern inline #endif #else #define PMAP_INLINE #endif /* * These are configured by the mair_el1 register. This is set up in locore.S */ #define DEVICE_MEMORY 0 #define UNCACHED_MEMORY 1 #define CACHED_MEMORY 2 #ifdef PV_STATS #define PV_STAT(x) do { x ; } while (0) #else #define PV_STAT(x) do { } while (0) #endif #define pmap_l2_pindex(v) ((v) >> L2_SHIFT) #define pa_to_pvh(pa) (&pv_table[pmap_l2_pindex(pa)]) #define NPV_LIST_LOCKS MAXCPU #define PHYS_TO_PV_LIST_LOCK(pa) \ (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ struct rwlock **_lockp = (lockp); \ struct rwlock *_new_lock; \ \ _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ if (_new_lock != *_lockp) { \ if (*_lockp != NULL) \ rw_wunlock(*_lockp); \ *_lockp = _new_lock; \ rw_wlock(*_lockp); \ } \ } while (0) #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) #define RELEASE_PV_LIST_LOCK(lockp) do { \ struct rwlock **_lockp = (lockp); \ \ if (*_lockp != NULL) { \ rw_wunlock(*_lockp); \ *_lockp = NULL; \ } \ } while (0) #define VM_PAGE_TO_PV_LIST_LOCK(m) \ PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) struct pmap kernel_pmap_store; vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ vm_offset_t kernel_vm_end = 0; struct msgbuf *msgbufp = NULL; /* * Data for the pv entry allocation mechanism. * Updates to pv_invl_gen are protected by the pv_list_locks[] * elements, but reads are not. */ static struct md_page *pv_table; static struct md_page pv_dummy; vm_paddr_t dmap_phys_base; /* The start of the dmap region */ vm_paddr_t dmap_phys_max; /* The limit of the dmap region */ vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */ /* This code assumes all L1 DMAP entries will be used */ CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS); CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS); #define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) extern pt_entry_t pagetable_dmap[]; static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); static int superpages_enabled = 0; SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0, "Are large page mappings enabled?"); /* * Data for the pv entry allocation mechanism */ static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); static struct mtx pv_chunks_mutex; static struct rwlock pv_list_locks[NPV_LIST_LOCKS]; static void free_pv_chunk(struct pv_chunk *pc); static void free_pv_entry(pmap_t pmap, pv_entry_t pv); static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va); static int pmap_change_attr(vm_offset_t va, vm_size_t size, int mode); static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode); static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va); static pt_entry_t *pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va, struct rwlock **lockp); static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va); static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva, pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, struct rwlock **lockp); static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); static int pmap_unuse_l3(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); /* * These load the old table data and store the new value. * They need to be atomic as the System MMU may write to the table at * the same time as the CPU. */ #define pmap_load_store(table, entry) atomic_swap_64(table, entry) #define pmap_set(table, mask) atomic_set_64(table, mask) #define pmap_load_clear(table) atomic_swap_64(table, 0) #define pmap_load(table) (*table) /********************/ /* Inline functions */ /********************/ static __inline void pagecopy(void *s, void *d) { memcpy(d, s, PAGE_SIZE); } #define pmap_l0_index(va) (((va) >> L0_SHIFT) & L0_ADDR_MASK) #define pmap_l1_index(va) (((va) >> L1_SHIFT) & Ln_ADDR_MASK) #define pmap_l2_index(va) (((va) >> L2_SHIFT) & Ln_ADDR_MASK) #define pmap_l3_index(va) (((va) >> L3_SHIFT) & Ln_ADDR_MASK) static __inline pd_entry_t * pmap_l0(pmap_t pmap, vm_offset_t va) { return (&pmap->pm_l0[pmap_l0_index(va)]); } static __inline pd_entry_t * pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) { pd_entry_t *l1; l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); return (&l1[pmap_l1_index(va)]); } static __inline pd_entry_t * pmap_l1(pmap_t pmap, vm_offset_t va) { pd_entry_t *l0; l0 = pmap_l0(pmap, va); if ((pmap_load(l0) & ATTR_DESCR_MASK) != L0_TABLE) return (NULL); return (pmap_l0_to_l1(l0, va)); } static __inline pd_entry_t * pmap_l1_to_l2(pd_entry_t *l1, vm_offset_t va) { pd_entry_t *l2; l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); return (&l2[pmap_l2_index(va)]); } static __inline pd_entry_t * pmap_l2(pmap_t pmap, vm_offset_t va) { pd_entry_t *l1; l1 = pmap_l1(pmap, va); if ((pmap_load(l1) & ATTR_DESCR_MASK) != L1_TABLE) return (NULL); return (pmap_l1_to_l2(l1, va)); } static __inline pt_entry_t * pmap_l2_to_l3(pd_entry_t *l2, vm_offset_t va) { pt_entry_t *l3; l3 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l2) & ~ATTR_MASK); return (&l3[pmap_l3_index(va)]); } /* * Returns the lowest valid pde for a given virtual address. * The next level may or may not point to a valid page or block. */ static __inline pd_entry_t * pmap_pde(pmap_t pmap, vm_offset_t va, int *level) { pd_entry_t *l0, *l1, *l2, desc; l0 = pmap_l0(pmap, va); desc = pmap_load(l0) & ATTR_DESCR_MASK; if (desc != L0_TABLE) { *level = -1; return (NULL); } l1 = pmap_l0_to_l1(l0, va); desc = pmap_load(l1) & ATTR_DESCR_MASK; if (desc != L1_TABLE) { *level = 0; return (l0); } l2 = pmap_l1_to_l2(l1, va); desc = pmap_load(l2) & ATTR_DESCR_MASK; if (desc != L2_TABLE) { *level = 1; return (l1); } *level = 2; return (l2); } /* * Returns the lowest valid pte block or table entry for a given virtual * address. If there are no valid entries return NULL and set the level to * the first invalid level. */ static __inline pt_entry_t * pmap_pte(pmap_t pmap, vm_offset_t va, int *level) { pd_entry_t *l1, *l2, desc; pt_entry_t *l3; l1 = pmap_l1(pmap, va); if (l1 == NULL) { *level = 0; return (NULL); } desc = pmap_load(l1) & ATTR_DESCR_MASK; if (desc == L1_BLOCK) { *level = 1; return (l1); } if (desc != L1_TABLE) { *level = 1; return (NULL); } l2 = pmap_l1_to_l2(l1, va); desc = pmap_load(l2) & ATTR_DESCR_MASK; if (desc == L2_BLOCK) { *level = 2; return (l2); } if (desc != L2_TABLE) { *level = 2; return (NULL); } *level = 3; l3 = pmap_l2_to_l3(l2, va); if ((pmap_load(l3) & ATTR_DESCR_MASK) != L3_PAGE) return (NULL); return (l3); } static inline bool pmap_superpages_enabled(void) { return (superpages_enabled != 0); } bool pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1, pd_entry_t **l2, pt_entry_t **l3) { pd_entry_t *l0p, *l1p, *l2p; if (pmap->pm_l0 == NULL) return (false); l0p = pmap_l0(pmap, va); *l0 = l0p; if ((pmap_load(l0p) & ATTR_DESCR_MASK) != L0_TABLE) return (false); l1p = pmap_l0_to_l1(l0p, va); *l1 = l1p; if ((pmap_load(l1p) & ATTR_DESCR_MASK) == L1_BLOCK) { *l2 = NULL; *l3 = NULL; return (true); } if ((pmap_load(l1p) & ATTR_DESCR_MASK) != L1_TABLE) return (false); l2p = pmap_l1_to_l2(l1p, va); *l2 = l2p; if ((pmap_load(l2p) & ATTR_DESCR_MASK) == L2_BLOCK) { *l3 = NULL; return (true); } *l3 = pmap_l2_to_l3(l2p, va); return (true); } static __inline int pmap_is_current(pmap_t pmap) { return ((pmap == pmap_kernel()) || (pmap == curthread->td_proc->p_vmspace->vm_map.pmap)); } static __inline int pmap_l3_valid(pt_entry_t l3) { return ((l3 & ATTR_DESCR_MASK) == L3_PAGE); } /* Is a level 1 or 2entry a valid block and cacheable */ CTASSERT(L1_BLOCK == L2_BLOCK); static __inline int pmap_pte_valid_cacheable(pt_entry_t pte) { return (((pte & ATTR_DESCR_MASK) == L1_BLOCK) && ((pte & ATTR_IDX_MASK) == ATTR_IDX(CACHED_MEMORY))); } static __inline int pmap_l3_valid_cacheable(pt_entry_t l3) { return (((l3 & ATTR_DESCR_MASK) == L3_PAGE) && ((l3 & ATTR_IDX_MASK) == ATTR_IDX(CACHED_MEMORY))); } #define PTE_SYNC(pte) cpu_dcache_wb_range((vm_offset_t)pte, sizeof(*pte)) /* * Checks if the page is dirty. We currently lack proper tracking of this on * arm64 so for now assume is a page mapped as rw was accessed it is. */ static inline int pmap_page_dirty(pt_entry_t pte) { return ((pte & (ATTR_AF | ATTR_AP_RW_BIT)) == (ATTR_AF | ATTR_AP(ATTR_AP_RW))); } static __inline void pmap_resident_count_inc(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); pmap->pm_stats.resident_count += count; } static __inline void pmap_resident_count_dec(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(pmap->pm_stats.resident_count >= count, ("pmap %p resident count underflow %ld %d", pmap, pmap->pm_stats.resident_count, count)); pmap->pm_stats.resident_count -= count; } static pt_entry_t * pmap_early_page_idx(vm_offset_t l1pt, vm_offset_t va, u_int *l1_slot, u_int *l2_slot) { pt_entry_t *l2; pd_entry_t *l1; l1 = (pd_entry_t *)l1pt; *l1_slot = (va >> L1_SHIFT) & Ln_ADDR_MASK; /* Check locore has used a table L1 map */ KASSERT((l1[*l1_slot] & ATTR_DESCR_MASK) == L1_TABLE, ("Invalid bootstrap L1 table")); /* Find the address of the L2 table */ l2 = (pt_entry_t *)init_pt_va; *l2_slot = pmap_l2_index(va); return (l2); } static vm_paddr_t pmap_early_vtophys(vm_offset_t l1pt, vm_offset_t va) { u_int l1_slot, l2_slot; pt_entry_t *l2; l2 = pmap_early_page_idx(l1pt, va, &l1_slot, &l2_slot); return ((l2[l2_slot] & ~ATTR_MASK) + (va & L2_OFFSET)); } static void pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa, vm_paddr_t max_pa) { vm_offset_t va; vm_paddr_t pa; u_int l1_slot; pa = dmap_phys_base = min_pa & ~L1_OFFSET; va = DMAP_MIN_ADDRESS; for (; va < DMAP_MAX_ADDRESS && pa < max_pa; pa += L1_SIZE, va += L1_SIZE, l1_slot++) { l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT); pmap_load_store(&pagetable_dmap[l1_slot], (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_IDX(CACHED_MEMORY) | L1_BLOCK); } /* Set the upper limit of the DMAP region */ dmap_phys_max = pa; dmap_max_addr = va; cpu_dcache_wb_range((vm_offset_t)pagetable_dmap, PAGE_SIZE * DMAP_TABLES); cpu_tlb_flushID(); } static vm_offset_t pmap_bootstrap_l2(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l2_start) { vm_offset_t l2pt; vm_paddr_t pa; pd_entry_t *l1; u_int l1_slot; KASSERT((va & L1_OFFSET) == 0, ("Invalid virtual address")); l1 = (pd_entry_t *)l1pt; l1_slot = pmap_l1_index(va); l2pt = l2_start; for (; va < VM_MAX_KERNEL_ADDRESS; l1_slot++, va += L1_SIZE) { KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index")); pa = pmap_early_vtophys(l1pt, l2pt); pmap_load_store(&l1[l1_slot], (pa & ~Ln_TABLE_MASK) | L1_TABLE); l2pt += PAGE_SIZE; } /* Clean the L2 page table */ memset((void *)l2_start, 0, l2pt - l2_start); cpu_dcache_wb_range(l2_start, l2pt - l2_start); /* Flush the l1 table to ram */ cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); return l2pt; } static vm_offset_t pmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l3_start) { vm_offset_t l2pt, l3pt; vm_paddr_t pa; pd_entry_t *l2; u_int l2_slot; KASSERT((va & L2_OFFSET) == 0, ("Invalid virtual address")); l2 = pmap_l2(kernel_pmap, va); l2 = (pd_entry_t *)rounddown2((uintptr_t)l2, PAGE_SIZE); l2pt = (vm_offset_t)l2; l2_slot = pmap_l2_index(va); l3pt = l3_start; for (; va < VM_MAX_KERNEL_ADDRESS; l2_slot++, va += L2_SIZE) { KASSERT(l2_slot < Ln_ENTRIES, ("Invalid L2 index")); pa = pmap_early_vtophys(l1pt, l3pt); pmap_load_store(&l2[l2_slot], (pa & ~Ln_TABLE_MASK) | L2_TABLE); l3pt += PAGE_SIZE; } /* Clean the L2 page table */ memset((void *)l3_start, 0, l3pt - l3_start); cpu_dcache_wb_range(l3_start, l3pt - l3_start); cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); return l3pt; } /* * Bootstrap the system enough to run with virtual memory. */ void pmap_bootstrap(vm_offset_t l0pt, vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen) { u_int l1_slot, l2_slot, avail_slot, map_slot, used_map_slot; uint64_t kern_delta; pt_entry_t *l2; vm_offset_t va, freemempos; vm_offset_t dpcpu, msgbufpv; vm_paddr_t pa, max_pa, min_pa; int i; kern_delta = KERNBASE - kernstart; physmem = 0; printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen); printf("%lx\n", l1pt); printf("%lx\n", (KERNBASE >> L1_SHIFT) & Ln_ADDR_MASK); /* Set this early so we can use the pagetable walking functions */ kernel_pmap_store.pm_l0 = (pd_entry_t *)l0pt; PMAP_LOCK_INIT(kernel_pmap); /* Assume the address we were loaded to is a valid physical address */ min_pa = max_pa = KERNBASE - kern_delta; /* * Find the minimum physical address. physmap is sorted, * but may contain empty ranges. */ for (i = 0; i < (physmap_idx * 2); i += 2) { if (physmap[i] == physmap[i + 1]) continue; if (physmap[i] <= min_pa) min_pa = physmap[i]; if (physmap[i + 1] > max_pa) max_pa = physmap[i + 1]; } /* Create a direct map region early so we can use it for pa -> va */ pmap_bootstrap_dmap(l1pt, min_pa, max_pa); va = KERNBASE; pa = KERNBASE - kern_delta; /* * Start to initialise phys_avail by copying from physmap * up to the physical address KERNBASE points at. */ map_slot = avail_slot = 0; for (; map_slot < (physmap_idx * 2) && avail_slot < (PHYS_AVAIL_SIZE - 2); map_slot += 2) { if (physmap[map_slot] == physmap[map_slot + 1]) continue; if (physmap[map_slot] <= pa && physmap[map_slot + 1] > pa) break; phys_avail[avail_slot] = physmap[map_slot]; phys_avail[avail_slot + 1] = physmap[map_slot + 1]; physmem += (phys_avail[avail_slot + 1] - phys_avail[avail_slot]) >> PAGE_SHIFT; avail_slot += 2; } /* Add the memory before the kernel */ if (physmap[avail_slot] < pa && avail_slot < (PHYS_AVAIL_SIZE - 2)) { phys_avail[avail_slot] = physmap[map_slot]; phys_avail[avail_slot + 1] = pa; physmem += (phys_avail[avail_slot + 1] - phys_avail[avail_slot]) >> PAGE_SHIFT; avail_slot += 2; } used_map_slot = map_slot; /* * Read the page table to find out what is already mapped. * This assumes we have mapped a block of memory from KERNBASE * using a single L1 entry. */ l2 = pmap_early_page_idx(l1pt, KERNBASE, &l1_slot, &l2_slot); /* Sanity check the index, KERNBASE should be the first VA */ KASSERT(l2_slot == 0, ("The L2 index is non-zero")); /* Find how many pages we have mapped */ for (; l2_slot < Ln_ENTRIES; l2_slot++) { if ((l2[l2_slot] & ATTR_DESCR_MASK) == 0) break; /* Check locore used L2 blocks */ KASSERT((l2[l2_slot] & ATTR_DESCR_MASK) == L2_BLOCK, ("Invalid bootstrap L2 table")); KASSERT((l2[l2_slot] & ~ATTR_MASK) == pa, ("Incorrect PA in L2 table")); va += L2_SIZE; pa += L2_SIZE; } va = roundup2(va, L1_SIZE); freemempos = KERNBASE + kernlen; freemempos = roundup2(freemempos, PAGE_SIZE); /* Create the l2 tables up to VM_MAX_KERNEL_ADDRESS */ freemempos = pmap_bootstrap_l2(l1pt, va, freemempos); /* And the l3 tables for the early devmap */ freemempos = pmap_bootstrap_l3(l1pt, VM_MAX_KERNEL_ADDRESS - L2_SIZE, freemempos); cpu_tlb_flushID(); #define alloc_pages(var, np) \ (var) = freemempos; \ freemempos += (np * PAGE_SIZE); \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); /* Allocate dynamic per-cpu area. */ alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu, 0); /* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */ alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); msgbufp = (void *)msgbufpv; virtual_avail = roundup2(freemempos, L1_SIZE); virtual_end = VM_MAX_KERNEL_ADDRESS - L2_SIZE; kernel_vm_end = virtual_avail; pa = pmap_early_vtophys(l1pt, freemempos); /* Finish initialising physmap */ map_slot = used_map_slot; for (; avail_slot < (PHYS_AVAIL_SIZE - 2) && map_slot < (physmap_idx * 2); map_slot += 2) { if (physmap[map_slot] == physmap[map_slot + 1]) continue; /* Have we used the current range? */ if (physmap[map_slot + 1] <= pa) continue; /* Do we need to split the entry? */ if (physmap[map_slot] < pa) { phys_avail[avail_slot] = pa; phys_avail[avail_slot + 1] = physmap[map_slot + 1]; } else { phys_avail[avail_slot] = physmap[map_slot]; phys_avail[avail_slot + 1] = physmap[map_slot + 1]; } physmem += (phys_avail[avail_slot + 1] - phys_avail[avail_slot]) >> PAGE_SHIFT; avail_slot += 2; } phys_avail[avail_slot] = 0; phys_avail[avail_slot + 1] = 0; /* * Maxmem isn't the "maximum memory", it's one larger than the * highest page of the physical address space. It should be * called something like "Maxphyspage". */ Maxmem = atop(phys_avail[avail_slot - 1]); cpu_tlb_flushID(); } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.pv_list); m->md.pv_memattr = VM_MEMATTR_WRITE_BACK; } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ void pmap_init(void) { vm_size_t s; int i, pv_npg; /* * Are large page mappings enabled? */ TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); /* * Initialize the pv chunk list mutex. */ mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); /* * Initialize the pool of pv list locks. */ for (i = 0; i < NPV_LIST_LOCKS; i++) rw_init(&pv_list_locks[i], "pmap pv list"); /* * Calculate the size of the pv head table for superpages. */ pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L2_SIZE); /* * Allocate memory for the pv head table for superpages. */ s = (vm_size_t)(pv_npg * sizeof(struct md_page)); s = round_page(s); pv_table = (struct md_page *)kmem_malloc(kernel_arena, s, M_WAITOK | M_ZERO); for (i = 0; i < pv_npg; i++) TAILQ_INIT(&pv_table[i].pv_list); TAILQ_INIT(&pv_dummy.pv_list); } static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2, CTLFLAG_RD, 0, "2MB page mapping counters"); static u_long pmap_l2_demotions; SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, demotions, CTLFLAG_RD, &pmap_l2_demotions, 0, "2MB page demotions"); static u_long pmap_l2_p_failures; SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, p_failures, CTLFLAG_RD, &pmap_l2_p_failures, 0, "2MB page promotion failures"); static u_long pmap_l2_promotions; SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, promotions, CTLFLAG_RD, &pmap_l2_promotions, 0, "2MB page promotions"); /* * Invalidate a single TLB entry. */ PMAP_INLINE void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { sched_pin(); __asm __volatile( "dsb ishst \n" "tlbi vaae1is, %0 \n" "dsb ish \n" "isb \n" : : "r"(va >> PAGE_SHIFT)); sched_unpin(); } PMAP_INLINE void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t addr; sched_pin(); dsb(ishst); for (addr = sva; addr < eva; addr += PAGE_SIZE) { __asm __volatile( "tlbi vaae1is, %0" : : "r"(addr >> PAGE_SHIFT)); } __asm __volatile( "dsb ish \n" "isb \n"); sched_unpin(); } PMAP_INLINE void pmap_invalidate_all(pmap_t pmap) { sched_pin(); __asm __volatile( "dsb ishst \n" "tlbi vmalle1is \n" "dsb ish \n" "isb \n"); sched_unpin(); } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va) { pt_entry_t *pte, tpte; vm_paddr_t pa; int lvl; pa = 0; PMAP_LOCK(pmap); /* * Find the block or page map for this virtual address. pmap_pte * will return either a valid block/page entry, or NULL. */ pte = pmap_pte(pmap, va, &lvl); if (pte != NULL) { tpte = pmap_load(pte); pa = tpte & ~ATTR_MASK; switch(lvl) { case 1: KASSERT((tpte & ATTR_DESCR_MASK) == L1_BLOCK, ("pmap_extract: Invalid L1 pte found: %lx", tpte & ATTR_DESCR_MASK)); pa |= (va & L1_OFFSET); break; case 2: KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK, ("pmap_extract: Invalid L2 pte found: %lx", tpte & ATTR_DESCR_MASK)); pa |= (va & L2_OFFSET); break; case 3: KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE, ("pmap_extract: Invalid L3 pte found: %lx", tpte & ATTR_DESCR_MASK)); pa |= (va & L3_OFFSET); break; } } PMAP_UNLOCK(pmap); return (pa); } /* * Routine: pmap_extract_and_hold * Function: * Atomically extract and hold the physical page * with the given pmap and virtual address pair * if that mapping permits the given protection. */ vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) { pt_entry_t *pte, tpte; vm_offset_t off; vm_paddr_t pa; vm_page_t m; int lvl; pa = 0; m = NULL; PMAP_LOCK(pmap); retry: pte = pmap_pte(pmap, va, &lvl); if (pte != NULL) { tpte = pmap_load(pte); KASSERT(lvl > 0 && lvl <= 3, ("pmap_extract_and_hold: Invalid level %d", lvl)); CTASSERT(L1_BLOCK == L2_BLOCK); KASSERT((lvl == 3 && (tpte & ATTR_DESCR_MASK) == L3_PAGE) || (lvl < 3 && (tpte & ATTR_DESCR_MASK) == L1_BLOCK), ("pmap_extract_and_hold: Invalid pte at L%d: %lx", lvl, tpte & ATTR_DESCR_MASK)); if (((tpte & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) || ((prot & VM_PROT_WRITE) == 0)) { switch(lvl) { case 1: off = va & L1_OFFSET; break; case 2: off = va & L2_OFFSET; break; case 3: default: off = 0; } if (vm_page_pa_tryrelock(pmap, (tpte & ~ATTR_MASK) | off, &pa)) goto retry; m = PHYS_TO_VM_PAGE((tpte & ~ATTR_MASK) | off); vm_page_hold(m); } } PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } vm_paddr_t pmap_kextract(vm_offset_t va) { pt_entry_t *pte, tpte; vm_paddr_t pa; int lvl; if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { pa = DMAP_TO_PHYS(va); } else { pa = 0; pte = pmap_pte(kernel_pmap, va, &lvl); if (pte != NULL) { tpte = pmap_load(pte); pa = tpte & ~ATTR_MASK; switch(lvl) { case 1: KASSERT((tpte & ATTR_DESCR_MASK) == L1_BLOCK, ("pmap_kextract: Invalid L1 pte found: %lx", tpte & ATTR_DESCR_MASK)); pa |= (va & L1_OFFSET); break; case 2: KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK, ("pmap_kextract: Invalid L2 pte found: %lx", tpte & ATTR_DESCR_MASK)); pa |= (va & L2_OFFSET); break; case 3: KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE, ("pmap_kextract: Invalid L3 pte found: %lx", tpte & ATTR_DESCR_MASK)); pa |= (va & L3_OFFSET); break; } } } return (pa); } /*************************************************** * Low level mapping routines..... ***************************************************/ static void pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode) { pd_entry_t *pde; pt_entry_t *pte; vm_offset_t va; int lvl; KASSERT((pa & L3_OFFSET) == 0, ("pmap_kenter: Invalid physical address")); KASSERT((sva & L3_OFFSET) == 0, ("pmap_kenter: Invalid virtual address")); KASSERT((size & PAGE_MASK) == 0, ("pmap_kenter: Mapping is not page-sized")); va = sva; while (size != 0) { pde = pmap_pde(kernel_pmap, va, &lvl); KASSERT(pde != NULL, ("pmap_kenter: Invalid page entry, va: 0x%lx", va)); KASSERT(lvl == 2, ("pmap_kenter: Invalid level %d", lvl)); pte = pmap_l2_to_l3(pde, va); pmap_load_store(pte, (pa & ~L3_OFFSET) | ATTR_DEFAULT | ATTR_IDX(mode) | L3_PAGE); PTE_SYNC(pte); va += PAGE_SIZE; pa += PAGE_SIZE; size -= PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } void pmap_kenter_device(vm_offset_t sva, vm_size_t size, vm_paddr_t pa) { pmap_kenter(sva, size, pa, DEVICE_MEMORY); } /* * Remove a page from the kernel pagetables. */ PMAP_INLINE void pmap_kremove(vm_offset_t va) { pt_entry_t *pte; int lvl; pte = pmap_pte(kernel_pmap, va, &lvl); KASSERT(pte != NULL, ("pmap_kremove: Invalid address")); KASSERT(lvl == 3, ("pmap_kremove: Invalid pte level %d", lvl)); if (pmap_l3_valid_cacheable(pmap_load(pte))) cpu_dcache_wb_range(va, L3_SIZE); pmap_load_clear(pte); PTE_SYNC(pte); pmap_invalidate_page(kernel_pmap, va); } void pmap_kremove_device(vm_offset_t sva, vm_size_t size) { pt_entry_t *pte; vm_offset_t va; int lvl; KASSERT((sva & L3_OFFSET) == 0, ("pmap_kremove_device: Invalid virtual address")); KASSERT((size & PAGE_MASK) == 0, ("pmap_kremove_device: Mapping is not page-sized")); va = sva; while (size != 0) { pte = pmap_pte(kernel_pmap, va, &lvl); KASSERT(pte != NULL, ("Invalid page table, va: 0x%lx", va)); KASSERT(lvl == 3, ("Invalid device pagetable level: %d != 3", lvl)); pmap_load_clear(pte); PTE_SYNC(pte); va += PAGE_SIZE; size -= PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /* * Used to map a range of physical addresses into kernel * virtual address space. * * The value passed in '*virt' is a suggested virtual address for * the mapping. Architectures which can support a direct-mapped * physical to virtual region can return the appropriate address * within that region, leaving '*virt' unchanged. Other * architectures should map the pages starting at '*virt' and * update '*virt' with the first usable address after the mapped * region. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { return PHYS_TO_DMAP(start); } /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) { pd_entry_t *pde; pt_entry_t *pte, pa; vm_offset_t va; vm_page_t m; int i, lvl; va = sva; for (i = 0; i < count; i++) { pde = pmap_pde(kernel_pmap, va, &lvl); KASSERT(pde != NULL, ("pmap_qenter: Invalid page entry, va: 0x%lx", va)); KASSERT(lvl == 2, ("pmap_qenter: Invalid level %d", lvl)); m = ma[i]; pa = VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | ATTR_AP(ATTR_AP_RW) | ATTR_IDX(m->md.pv_memattr) | L3_PAGE; pte = pmap_l2_to_l3(pde, va); pmap_load_store(pte, pa); PTE_SYNC(pte); va += L3_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /* * This routine tears out page mappings from the * kernel -- it is meant only for temporary mappings. */ void pmap_qremove(vm_offset_t sva, int count) { pt_entry_t *pte; vm_offset_t va; int lvl; KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", sva)); va = sva; while (count-- > 0) { pte = pmap_pte(kernel_pmap, va, &lvl); KASSERT(lvl == 3, ("Invalid device pagetable level: %d != 3", lvl)); if (pte != NULL) { if (pmap_l3_valid_cacheable(pmap_load(pte))) cpu_dcache_wb_range(va, L3_SIZE); pmap_load_clear(pte); PTE_SYNC(pte); } va += PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /*************************************************** * Page table page management routines..... ***************************************************/ static __inline void pmap_free_zero_pages(struct spglist *free) { vm_page_t m; while ((m = SLIST_FIRST(free)) != NULL) { SLIST_REMOVE_HEAD(free, plinks.s.ss); /* Preserve the page's PG_ZERO setting. */ vm_page_free_toq(m); } } /* * Schedule the specified unused page table page to be freed. Specifically, * add the page to the specified list of pages that will be released to the * physical memory manager after the TLB has been updated. */ static __inline void pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, boolean_t set_PG_ZERO) { if (set_PG_ZERO) m->flags |= PG_ZERO; else m->flags &= ~PG_ZERO; SLIST_INSERT_HEAD(free, m, plinks.s.ss); } /* * Decrements a page table page's wire count, which is used to record the * number of valid page table entries within the page. If the wire count * drops to zero, then the page table page is unmapped. Returns TRUE if the * page table page was unmapped and FALSE otherwise. */ static inline boolean_t pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { --m->wire_count; if (m->wire_count == 0) { _pmap_unwire_l3(pmap, va, m, free); return (TRUE); } else return (FALSE); } static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * unmap the page table page */ if (m->pindex >= (NUL2E + NUL1E)) { /* l1 page */ pd_entry_t *l0; l0 = pmap_l0(pmap, va); pmap_load_clear(l0); PTE_SYNC(l0); } else if (m->pindex >= NUL2E) { /* l2 page */ pd_entry_t *l1; l1 = pmap_l1(pmap, va); pmap_load_clear(l1); PTE_SYNC(l1); } else { /* l3 page */ pd_entry_t *l2; l2 = pmap_l2(pmap, va); pmap_load_clear(l2); PTE_SYNC(l2); } pmap_resident_count_dec(pmap, 1); if (m->pindex < NUL2E) { /* We just released an l3, unhold the matching l2 */ pd_entry_t *l1, tl1; vm_page_t l2pg; l1 = pmap_l1(pmap, va); tl1 = pmap_load(l1); l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); pmap_unwire_l3(pmap, va, l2pg, free); } else if (m->pindex < (NUL2E + NUL1E)) { /* We just released an l2, unhold the matching l1 */ pd_entry_t *l0, tl0; vm_page_t l1pg; l0 = pmap_l0(pmap, va); tl0 = pmap_load(l0); l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); pmap_unwire_l3(pmap, va, l1pg, free); } pmap_invalidate_page(pmap, va); /* * This is a release store so that the ordinary store unmapping * the page table page is globally performed before TLB shoot- * down is begun. */ atomic_subtract_rel_int(&vm_cnt.v_wire_count, 1); /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ pmap_add_delayed_free_list(m, free, TRUE); } /* * After removing an l3 entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ static int pmap_unuse_l3(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, struct spglist *free) { vm_page_t mpte; if (va >= VM_MAXUSER_ADDRESS) return (0); KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); mpte = PHYS_TO_VM_PAGE(ptepde & ~ATTR_MASK); return (pmap_unwire_l3(pmap, va, mpte, free)); } void pmap_pinit0(pmap_t pmap) { PMAP_LOCK_INIT(pmap); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); pmap->pm_l0 = kernel_pmap->pm_l0; pmap->pm_root.rt_root = 0; } int pmap_pinit(pmap_t pmap) { vm_paddr_t l0phys; vm_page_t l0pt; /* * allocate the l0 page */ while ((l0pt = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) VM_WAIT; l0phys = VM_PAGE_TO_PHYS(l0pt); pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(l0phys); if ((l0pt->flags & PG_ZERO) == 0) pagezero(pmap->pm_l0); pmap->pm_root.rt_root = 0; bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); return (1); } /* * This routine is called if the desired page table page does not exist. * * If page table page allocation fails, this routine may sleep before * returning NULL. It sleeps only if a lock pointer was given. * * Note: If a page allocation fails at page table level two or three, * one or two pages may be held during the wait, only to be released * afterwards. This conservative approach is easily argued to avoid * race conditions. */ static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) { vm_page_t m, l1pg, l2pg; PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Allocate a page table page. */ if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { if (lockp != NULL) { RELEASE_PV_LIST_LOCK(lockp); PMAP_UNLOCK(pmap); VM_WAIT; PMAP_LOCK(pmap); } /* * Indicate the need to retry. While waiting, the page table * page may have been allocated. */ return (NULL); } if ((m->flags & PG_ZERO) == 0) pmap_zero_page(m); /* * Map the pagetable page into the process address space, if * it isn't already there. */ if (ptepindex >= (NUL2E + NUL1E)) { pd_entry_t *l0; vm_pindex_t l0index; l0index = ptepindex - (NUL2E + NUL1E); l0 = &pmap->pm_l0[l0index]; pmap_load_store(l0, VM_PAGE_TO_PHYS(m) | L0_TABLE); PTE_SYNC(l0); } else if (ptepindex >= NUL2E) { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1; pd_entry_t tl0; l1index = ptepindex - NUL2E; l0index = l1index >> L0_ENTRIES_SHIFT; l0 = &pmap->pm_l0[l0index]; tl0 = pmap_load(l0); if (tl0 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index, lockp) == NULL) { --m->wire_count; /* XXX: release mem barrier? */ atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } } else { l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); l1pg->wire_count++; } l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); l1 = &l1[ptepindex & Ln_ADDR_MASK]; pmap_load_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE); PTE_SYNC(l1); } else { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1, *l2; pd_entry_t tl0, tl1; l1index = ptepindex >> Ln_ENTRIES_SHIFT; l0index = l1index >> L0_ENTRIES_SHIFT; l0 = &pmap->pm_l0[l0index]; tl0 = pmap_load(l0); if (tl0 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + l1index, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } tl0 = pmap_load(l0); l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); l1 = &l1[l1index & Ln_ADDR_MASK]; } else { l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); l1 = &l1[l1index & Ln_ADDR_MASK]; tl1 = pmap_load(l1); if (tl1 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + l1index, lockp) == NULL) { --m->wire_count; /* XXX: release mem barrier? */ atomic_subtract_int( &vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } } else { l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); l2pg->wire_count++; } } l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); l2 = &l2[ptepindex & Ln_ADDR_MASK]; pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE); PTE_SYNC(l2); } pmap_resident_count_inc(pmap, 1); return (m); } static vm_page_t pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) { vm_pindex_t ptepindex; pd_entry_t *pde, tpde; #ifdef INVARIANTS pt_entry_t *pte; #endif vm_page_t m; int lvl; /* * Calculate pagetable page index */ ptepindex = pmap_l2_pindex(va); retry: /* * Get the page directory entry */ pde = pmap_pde(pmap, va, &lvl); /* * If the page table page is mapped, we just increment the hold count, * and activate it. If we get a level 2 pde it will point to a level 3 * table. */ switch (lvl) { case -1: break; case 0: #ifdef INVARIANTS pte = pmap_l0_to_l1(pde, va); KASSERT(pmap_load(pte) == 0, ("pmap_alloc_l3: TODO: l0 superpages")); #endif break; case 1: #ifdef INVARIANTS pte = pmap_l1_to_l2(pde, va); KASSERT(pmap_load(pte) == 0, ("pmap_alloc_l3: TODO: l1 superpages")); #endif break; case 2: tpde = pmap_load(pde); if (tpde != 0) { m = PHYS_TO_VM_PAGE(tpde & ~ATTR_MASK); m->wire_count++; return (m); } break; default: panic("pmap_alloc_l3: Invalid level %d", lvl); } /* * Here if the pte page isn't mapped, or if it has been deallocated. */ m = _pmap_alloc_l3(pmap, ptepindex, lockp); if (m == NULL && lockp != NULL) goto retry; return (m); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pmap) { vm_page_t m; KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); KASSERT(vm_radix_is_empty(&pmap->pm_root), ("pmap_release: pmap has reserved page table page(s)")); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_l0)); m->wire_count--; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); } static int kvm_size(SYSCTL_HANDLER_ARGS) { unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; return sysctl_handle_long(oidp, &ksize, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_size, "LU", "Size of KVM"); static int kvm_free(SYSCTL_HANDLER_ARGS) { unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; return sysctl_handle_long(oidp, &kfree, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_free, "LU", "Amount of KVM free"); /* * grow the number of kernel page table entries, if needed */ void pmap_growkernel(vm_offset_t addr) { vm_paddr_t paddr; vm_page_t nkpg; pd_entry_t *l0, *l1, *l2; mtx_assert(&kernel_map->system_mtx, MA_OWNED); addr = roundup2(addr, L2_SIZE); if (addr - 1 >= kernel_map->max_offset) addr = kernel_map->max_offset; while (kernel_vm_end < addr) { l0 = pmap_l0(kernel_pmap, kernel_vm_end); KASSERT(pmap_load(l0) != 0, ("pmap_growkernel: No level 0 kernel entry")); l1 = pmap_l0_to_l1(l0, kernel_vm_end); if (pmap_load(l1) == 0) { /* We need a new PDP entry */ nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); if ((nkpg->flags & PG_ZERO) == 0) pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); pmap_load_store(l1, paddr | L1_TABLE); PTE_SYNC(l1); continue; /* try again */ } l2 = pmap_l1_to_l2(l1, kernel_vm_end); if ((pmap_load(l2) & ATTR_AF) != 0) { kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } continue; } nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); if ((nkpg->flags & PG_ZERO) == 0) pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); pmap_load_store(l2, paddr | L2_TABLE); PTE_SYNC(l2); pmap_invalidate_page(kernel_pmap, kernel_vm_end); kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } } } /*************************************************** * page management routines. ***************************************************/ CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); CTASSERT(_NPCM == 3); CTASSERT(_NPCPV == 168); static __inline struct pv_chunk * pv_to_chunk(pv_entry_t pv) { return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); } #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) #define PC_FREE0 0xfffffffffffffffful #define PC_FREE1 0xfffffffffffffffful #define PC_FREE2 0x000000fffffffffful static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; #if 0 #ifdef PV_STATS static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, "Current number of pv entry chunks"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, "Current number of pv entry chunks allocated"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, "Current number of pv entry chunks frees"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, "Number of times tried to get a chunk page but failed."); static long pv_entry_frees, pv_entry_allocs, pv_entry_count; static int pv_entry_spare; SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, "Current number of pv entry frees"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, "Current number of pv entry allocs"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, "Current number of pv entries"); SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, "Current number of spare pv entries"); #endif #endif /* 0 */ /* * We are in a serious low memory condition. Resort to * drastic measures to free some pages so we can allocate * another pv entry chunk. * * Returns NULL if PV entries were reclaimed from the specified pmap. * * We do not, however, unmap 2mpages because subsequent accesses will * allocate per-page pv entries until repromotion occurs, thereby * exacerbating the shortage of free pv entries. */ static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) { panic("ARM64TODO: reclaim_pv_chunk"); } /* * free the pv_entry back to the free list */ static void free_pv_entry(pmap_t pmap, pv_entry_t pv) { struct pv_chunk *pc; int idx, field, bit; PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(atomic_add_long(&pv_entry_frees, 1)); PV_STAT(atomic_add_int(&pv_entry_spare, 1)); PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 64; bit = idx % 64; pc->pc_map[field] |= 1ul << bit; if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 || pc->pc_map[2] != PC_FREE2) { /* 98% of the time, pc is already at the head of the list. */ if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); } return; } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } static void free_pv_chunk(struct pv_chunk *pc) { vm_page_t m; mtx_lock(&pv_chunks_mutex); TAILQ_REMOVE(&pv_chunks, pc, pc_lru); mtx_unlock(&pv_chunks_mutex); PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); dump_drop_page(m->phys_addr); vm_page_unwire(m, PQ_NONE); vm_page_free(m); } /* * Returns a new PV entry, allocating a new PV chunk from the system when * needed. If this PV chunk allocation fails and a PV list lock pointer was * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is * returned. * * The given PV list lock may be released. */ static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp) { int bit, field; pv_entry_t pv; struct pv_chunk *pc; vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); retry: pc = TAILQ_FIRST(&pmap->pm_pvchunk); if (pc != NULL) { for (field = 0; field < _NPCM; field++) { if (pc->pc_map[field]) { bit = ffsl(pc->pc_map[field]) - 1; break; } } if (field < _NPCM) { pv = &pc->pc_pventry[field * 64 + bit]; pc->pc_map[field] &= ~(1ul << bit); /* If this was the last item, move it to tail */ if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } PV_STAT(atomic_add_long(&pv_entry_count, 1)); PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); return (pv); } } /* No free items, allocate another chunk */ m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (m == NULL) { if (lockp == NULL) { PV_STAT(pc_chunk_tryfail++); return (NULL); } m = reclaim_pv_chunk(pmap, lockp); if (m == NULL) goto retry; } PV_STAT(atomic_add_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); dump_add_page(m->phys_addr); pc = (void *)PHYS_TO_DMAP(m->phys_addr); pc->pc_pmap = pmap; pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ pc->pc_map[1] = PC_FREE1; pc->pc_map[2] = PC_FREE2; mtx_lock(&pv_chunks_mutex); TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); mtx_unlock(&pv_chunks_mutex); pv = &pc->pc_pventry[0]; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(atomic_add_long(&pv_entry_count, 1)); PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); return (pv); } /* * Ensure that the number of spare PV entries in the specified pmap meets or * exceeds the given count, "needed". * * The given PV list lock may be released. */ static void reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) { struct pch new_tail; struct pv_chunk *pc; int avail, free; vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); /* * Newly allocated PV chunks must be stored in a private list until * the required number of PV chunks have been allocated. Otherwise, * reclaim_pv_chunk() could recycle one of these chunks. In * contrast, these chunks must be added to the pmap upon allocation. */ TAILQ_INIT(&new_tail); retry: avail = 0; TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { bit_count((bitstr_t *)pc->pc_map, 0, sizeof(pc->pc_map) * NBBY, &free); if (free == 0) break; avail += free; if (avail >= needed) break; } for (; avail < needed; avail += _NPCPV) { m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (m == NULL) { m = reclaim_pv_chunk(pmap, lockp); if (m == NULL) goto retry; } PV_STAT(atomic_add_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); dump_add_page(m->phys_addr); pc = (void *)PHYS_TO_DMAP(m->phys_addr); pc->pc_pmap = pmap; pc->pc_map[0] = PC_FREE0; pc->pc_map[1] = PC_FREE1; pc->pc_map[2] = PC_FREE2; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); } if (!TAILQ_EMPTY(&new_tail)) { mtx_lock(&pv_chunks_mutex); TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); mtx_unlock(&pv_chunks_mutex); } } /* * First find and then remove the pv entry for the specified pmap and virtual * address from the specified pv list. Returns the pv entry if found and NULL * otherwise. This operation can be performed on pv lists for either 4KB or * 2MB page mappings. */ static __inline pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; break; } } return (pv); } /* * After demotion from a 2MB page mapping to 512 4KB page mappings, * destroy the pv entry for the 2MB page mapping and reinstantiate the pv * entries for each of the 4KB page mappings. */ static void pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp) { struct md_page *pvh; struct pv_chunk *pc; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; int bit, field; PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((pa & L2_OFFSET) == 0, ("pmap_pv_demote_l2: pa is not 2mpage aligned")); CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); /* * Transfer the 2mpage's pv entry for this mapping to the first * page's pv list. Once this transfer begins, the pv list lock * must not be released until the last pv entry is reinstantiated. */ pvh = pa_to_pvh(pa); va = va & ~L2_OFFSET; pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pv_demote_l2: pv not found")); m = PHYS_TO_VM_PAGE(pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; /* Instantiate the remaining Ln_ENTRIES - 1 pv entries. */ PV_STAT(atomic_add_long(&pv_entry_allocs, Ln_ENTRIES - 1)); va_last = va + L2_SIZE - PAGE_SIZE; for (;;) { pc = TAILQ_FIRST(&pmap->pm_pvchunk); KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 || pc->pc_map[2] != 0, ("pmap_pv_demote_l2: missing spare")); for (field = 0; field < _NPCM; field++) { while (pc->pc_map[field]) { bit = ffsl(pc->pc_map[field]) - 1; pc->pc_map[field] &= ~(1ul << bit); pv = &pc->pc_pventry[field * 64 + bit]; va += PAGE_SIZE; pv->pv_va = va; m++; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_pv_demote_l2: page %p is not managed", m)); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if (va == va_last) goto out; } } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } out: if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } PV_STAT(atomic_add_long(&pv_entry_count, Ln_ENTRIES - 1)); PV_STAT(atomic_subtract_int(&pv_entry_spare, Ln_ENTRIES - 1)); } /* * First find and then destroy the pv entry for the specified pmap and virtual * address. This operation can be performed on pv lists for either 4KB or 2MB * page mappings. */ static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); free_pv_entry(pmap, pv); } /* * Conditionally create the PV entry for a 4KB page mapping if the required * memory can be allocated without resorting to reclamation. */ static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, struct rwlock **lockp) { pv_entry_t pv; PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* Pass NULL instead of the lock pointer to disable reclamation. */ if ((pv = get_pv_entry(pmap, NULL)) != NULL) { pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; return (TRUE); } else return (FALSE); } /* * pmap_remove_l3: do the things to unmap a page in a process */ static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va, pd_entry_t l2e, struct spglist *free, struct rwlock **lockp) { struct md_page *pvh; pt_entry_t old_l3; vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(pmap_load(l3))) cpu_dcache_wb_range(va, L3_SIZE); old_l3 = pmap_load_clear(l3); PTE_SYNC(l3); pmap_invalidate_page(pmap, va); if (old_l3 & ATTR_SW_WIRED) pmap->pm_stats.wired_count -= 1; pmap_resident_count_dec(pmap, 1); if (old_l3 & ATTR_SW_MANAGED) { m = PHYS_TO_VM_PAGE(old_l3 & ~ATTR_MASK); if (pmap_page_dirty(old_l3)) vm_page_dirty(m); if (old_l3 & ATTR_AF) vm_page_aflag_set(m, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); pmap_pvh_free(&m->md, pmap, va); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } return (pmap_unuse_l3(pmap, va, l2e, free)); } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ void pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { struct rwlock *lock; vm_offset_t va, va_next; pd_entry_t *l0, *l1, *l2; pt_entry_t l3_paddr, *l3; struct spglist free; int anyvalid; /* * Perform an unsynchronized read. This is, however, safe. */ if (pmap->pm_stats.resident_count == 0) return; anyvalid = 0; SLIST_INIT(&free); PMAP_LOCK(pmap); lock = NULL; for (; sva < eva; sva = va_next) { if (pmap->pm_stats.resident_count == 0) break; l0 = pmap_l0(pmap, sva); if (pmap_load(l0) == 0) { va_next = (sva + L0_SIZE) & ~L0_OFFSET; if (va_next < sva) va_next = eva; continue; } l1 = pmap_l0_to_l1(l0, sva); if (pmap_load(l1) == 0) { va_next = (sva + L1_SIZE) & ~L1_OFFSET; if (va_next < sva) va_next = eva; continue; } /* * Calculate index for next page table. */ va_next = (sva + L2_SIZE) & ~L2_OFFSET; if (va_next < sva) va_next = eva; l2 = pmap_l1_to_l2(l1, sva); if (l2 == NULL) continue; l3_paddr = pmap_load(l2); if ((l3_paddr & ATTR_DESCR_MASK) == L2_BLOCK) { /* TODO: Add pmap_remove_l2 */ if (pmap_demote_l2_locked(pmap, l2, sva & ~L2_OFFSET, &lock) == NULL) continue; l3_paddr = pmap_load(l2); } /* * Weed out invalid mappings. */ if ((l3_paddr & ATTR_DESCR_MASK) != L2_TABLE) continue; /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the * range being removed. */ if (va_next > eva) va_next = eva; va = va_next; for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, sva += L3_SIZE) { if (l3 == NULL) panic("l3 == NULL"); if (pmap_load(l3) == 0) { if (va != va_next) { pmap_invalidate_range(pmap, va, sva); va = va_next; } continue; } if (va == va_next) va = sva; if (pmap_remove_l3(pmap, l3, sva, l3_paddr, &free, &lock)) { sva += L3_SIZE; break; } } if (va != va_next) pmap_invalidate_range(pmap, va, sva); } if (lock != NULL) rw_wunlock(lock); if (anyvalid) pmap_invalidate_all(pmap); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ void pmap_remove_all(vm_page_t m) { struct md_page *pvh; pv_entry_t pv; pmap_t pmap; struct rwlock *lock; pd_entry_t *pde, tpde; pt_entry_t *pte, tpte; vm_offset_t va; struct spglist free; int lvl, pvh_gen, md_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_all: page %p is not managed", m)); SLIST_INIT(&free); lock = VM_PAGE_TO_PV_LIST_LOCK(m); pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(VM_PAGE_TO_PHYS(m)); retry: rw_wlock(lock); while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen) { rw_wunlock(lock); PMAP_UNLOCK(pmap); goto retry; } } va = pv->pv_va; pte = pmap_pte(pmap, va, &lvl); KASSERT(pte != NULL, ("pmap_remove_all: no page table entry found")); KASSERT(lvl == 2, ("pmap_remove_all: invalid pte level %d", lvl)); pmap_demote_l2_locked(pmap, pte, va, &lock); PMAP_UNLOCK(pmap); } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; md_gen = m->md.pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { rw_wunlock(lock); PMAP_UNLOCK(pmap); goto retry; } } pmap_resident_count_dec(pmap, 1); pde = pmap_pde(pmap, pv->pv_va, &lvl); KASSERT(pde != NULL, ("pmap_remove_all: no page directory entry found")); KASSERT(lvl == 2, ("pmap_remove_all: invalid pde level %d", lvl)); tpde = pmap_load(pde); pte = pmap_l2_to_l3(pde, pv->pv_va); tpte = pmap_load(pte); if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(tpte)) cpu_dcache_wb_range(pv->pv_va, L3_SIZE); pmap_load_clear(pte); PTE_SYNC(pte); pmap_invalidate_page(pmap, pv->pv_va); if (tpte & ATTR_SW_WIRED) pmap->pm_stats.wired_count--; if ((tpte & ATTR_AF) != 0) vm_page_aflag_set(m, PGA_REFERENCED); /* * Update the vm_page_t clean and reference bits. */ if (pmap_page_dirty(tpte)) vm_page_dirty(m); pmap_unuse_l3(pmap, pv->pv_va, tpde, &free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(lock); pmap_free_zero_pages(&free); } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va, va_next; pd_entry_t *l0, *l1, *l2; pt_entry_t *l3p, l3; if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; } if ((prot & VM_PROT_WRITE) == VM_PROT_WRITE) return; PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { l0 = pmap_l0(pmap, sva); if (pmap_load(l0) == 0) { va_next = (sva + L0_SIZE) & ~L0_OFFSET; if (va_next < sva) va_next = eva; continue; } l1 = pmap_l0_to_l1(l0, sva); if (pmap_load(l1) == 0) { va_next = (sva + L1_SIZE) & ~L1_OFFSET; if (va_next < sva) va_next = eva; continue; } va_next = (sva + L2_SIZE) & ~L2_OFFSET; if (va_next < sva) va_next = eva; l2 = pmap_l1_to_l2(l1, sva); if (pmap_load(l2) == 0) continue; if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) { l3p = pmap_demote_l2(pmap, l2, sva); if (l3p == NULL) continue; } KASSERT((pmap_load(l2) & ATTR_DESCR_MASK) == L2_TABLE, ("pmap_protect: Invalid L2 entry after demotion")); if (va_next > eva) va_next = eva; va = va_next; for (l3p = pmap_l2_to_l3(l2, sva); sva != va_next; l3p++, sva += L3_SIZE) { l3 = pmap_load(l3p); if (pmap_l3_valid(l3)) { pmap_set(l3p, ATTR_AP(ATTR_AP_RO)); PTE_SYNC(l3p); /* XXX: Use pmap_invalidate_range */ pmap_invalidate_page(pmap, va); } } } PMAP_UNLOCK(pmap); /* TODO: Only invalidate entries we are touching */ pmap_invalidate_all(pmap); } /* * Inserts the specified page table page into the specified pmap's collection * of idle page table pages. Each of a pmap's page table pages is responsible * for mapping a distinct range of virtual addresses. The pmap's collection is * ordered by this virtual address range. */ static __inline int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); return (vm_radix_insert(&pmap->pm_root, mpte)); } /* * Looks for a page table page mapping the specified virtual address in the * specified pmap's collection of idle page table pages. Returns NULL if there * is no page table page corresponding to the specified virtual address. */ static __inline vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); return (vm_radix_lookup(&pmap->pm_root, pmap_l2_pindex(va))); } /* * Removes the specified page table page from the specified pmap's collection * of idle page table pages. The specified page table page must be a member of * the pmap's collection. */ static __inline void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); vm_radix_remove(&pmap->pm_root, mpte->pindex); } /* * Performs a break-before-make update of a pmap entry. This is needed when * either promoting or demoting pages to ensure the TLB doesn't get into an * inconsistent state. */ static void pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte, vm_offset_t va, vm_size_t size) { register_t intr; PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Ensure we don't get switched out with the page table in an * inconsistent state. We also need to ensure no interrupts fire * as they may make use of an address we are about to invalidate. */ intr = intr_disable(); critical_enter(); /* Clear the old mapping */ pmap_load_clear(pte); PTE_SYNC(pte); pmap_invalidate_range(pmap, va, va + size); /* Create the new mapping */ pmap_load_store(pte, newpte); PTE_SYNC(pte); critical_exit(); intr_restore(intr); } /* * After promotion from 512 4KB page mappings to a single 2MB page mapping, * replace the many pv entries for the 4KB page mappings by a single pv entry * for the 2MB page mapping. */ static void pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp) { struct md_page *pvh; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; KASSERT((pa & L2_OFFSET) == 0, ("pmap_pv_promote_l2: pa is not 2mpage aligned")); CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); /* * Transfer the first page's pv entry for this mapping to the 2mpage's * pv list. Aside from avoiding the cost of a call to get_pv_entry(), * a transfer avoids the possibility that get_pv_entry() calls * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the * mappings that is being promoted. */ m = PHYS_TO_VM_PAGE(pa); va = va & ~L2_OFFSET; pv = pmap_pvh_remove(&m->md, pmap, va); KASSERT(pv != NULL, ("pmap_pv_promote_l2: pv not found")); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; /* Free the remaining NPTEPG - 1 pv entries. */ va_last = va + L2_SIZE - PAGE_SIZE; do { m++; va += PAGE_SIZE; pmap_pvh_free(&m->md, pmap, va); } while (va < va_last); } /* * Tries to promote the 512, contiguous 4KB page mappings that are within a * single level 2 table entry to a single 2MB page mapping. For promotion * to occur, two conditions must be met: (1) the 4KB page mappings must map * aligned, contiguous physical memory and (2) the 4KB page mappings must have * identical characteristics. */ static void pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, struct rwlock **lockp) { pt_entry_t *firstl3, *l3, newl2, oldl3, pa; vm_page_t mpte; vm_offset_t sva; PMAP_LOCK_ASSERT(pmap, MA_OWNED); sva = va & ~L2_OFFSET; firstl3 = pmap_l2_to_l3(l2, sva); newl2 = pmap_load(firstl3); /* Check the alingment is valid */ if (((newl2 & ~ATTR_MASK) & L2_OFFSET) != 0) { atomic_add_long(&pmap_l2_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" " in pmap %p", va, pmap); return; } pa = newl2 + L2_SIZE - PAGE_SIZE; for (l3 = firstl3 + NL3PG - 1; l3 > firstl3; l3--) { oldl3 = pmap_load(l3); if (oldl3 != pa) { atomic_add_long(&pmap_l2_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" " in pmap %p", va, pmap); return; } pa -= PAGE_SIZE; } /* * Save the page table page in its current state until the L2 * mapping the superpage is demoted by pmap_demote_l2() or * destroyed by pmap_remove_l3(). */ mpte = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK); KASSERT(mpte >= vm_page_array && mpte < &vm_page_array[vm_page_array_size], ("pmap_promote_l2: page table page is out of range")); KASSERT(mpte->pindex == pmap_l2_pindex(va), ("pmap_promote_l2: page table page's pindex is wrong")); if (pmap_insert_pt_page(pmap, mpte)) { atomic_add_long(&pmap_l2_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx in pmap %p", va, pmap); return; } if ((newl2 & ATTR_SW_MANAGED) != 0) pmap_pv_promote_l2(pmap, va, newl2 & ~ATTR_MASK, lockp); newl2 &= ~ATTR_DESCR_MASK; newl2 |= L2_BLOCK; pmap_update_entry(pmap, l2, newl2, sva, L2_SIZE); atomic_add_long(&pmap_l2_promotions, 1); CTR2(KTR_PMAP, "pmap_promote_l2: success for va %#lx in pmap %p", va, pmap); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind __unused) { struct rwlock *lock; pd_entry_t *pde; pt_entry_t new_l3, orig_l3; pt_entry_t *l2, *l3; pv_entry_t pv; vm_paddr_t opa, pa, l1_pa, l2_pa, l3_pa; vm_page_t mpte, om, l1_m, l2_m, l3_m; boolean_t nosleep; int lvl; va = trunc_page(va); if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); pa = VM_PAGE_TO_PHYS(m); new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | ATTR_IDX(m->md.pv_memattr) | L3_PAGE); if ((prot & VM_PROT_WRITE) == 0) new_l3 |= ATTR_AP(ATTR_AP_RO); if ((flags & PMAP_ENTER_WIRED) != 0) new_l3 |= ATTR_SW_WIRED; if ((va >> 63) == 0) new_l3 |= ATTR_AP(ATTR_AP_USER); CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa); mpte = NULL; lock = NULL; PMAP_LOCK(pmap); pde = pmap_pde(pmap, va, &lvl); if (pde != NULL && lvl == 1) { l2 = pmap_l1_to_l2(pde, va); if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK && (l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) { if (va < VM_MAXUSER_ADDRESS) { mpte = PHYS_TO_VM_PAGE( pmap_load(l2) & ~ATTR_MASK); mpte->wire_count++; } goto havel3; } } if (va < VM_MAXUSER_ADDRESS) { nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock); if (mpte == NULL && nosleep) { CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); if (lock != NULL) rw_wunlock(lock); PMAP_UNLOCK(pmap); return (KERN_RESOURCE_SHORTAGE); } pde = pmap_pde(pmap, va, &lvl); KASSERT(pde != NULL, ("pmap_enter: Invalid page entry, va: 0x%lx", va)); KASSERT(lvl == 2, ("pmap_enter: Invalid level %d", lvl)); l3 = pmap_l2_to_l3(pde, va); } else { /* * If we get a level 2 pde it must point to a level 3 entry * otherwise we will need to create the intermediate tables */ if (lvl < 2) { switch(lvl) { default: case -1: /* Get the l0 pde to update */ pde = pmap_l0(pmap, va); KASSERT(pde != NULL, ("...")); l1_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (l1_m == NULL) panic("pmap_enter: l1 pte_m == NULL"); if ((l1_m->flags & PG_ZERO) == 0) pmap_zero_page(l1_m); l1_pa = VM_PAGE_TO_PHYS(l1_m); pmap_load_store(pde, l1_pa | L0_TABLE); PTE_SYNC(pde); /* FALLTHROUGH */ case 0: /* Get the l1 pde to update */ pde = pmap_l1_to_l2(pde, va); KASSERT(pde != NULL, ("...")); l2_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (l2_m == NULL) panic("pmap_enter: l2 pte_m == NULL"); if ((l2_m->flags & PG_ZERO) == 0) pmap_zero_page(l2_m); l2_pa = VM_PAGE_TO_PHYS(l2_m); pmap_load_store(pde, l2_pa | L1_TABLE); PTE_SYNC(pde); /* FALLTHROUGH */ case 1: /* Get the l2 pde to update */ pde = pmap_l1_to_l2(pde, va); l3_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (l3_m == NULL) panic("pmap_enter: l3 pte_m == NULL"); if ((l3_m->flags & PG_ZERO) == 0) pmap_zero_page(l3_m); l3_pa = VM_PAGE_TO_PHYS(l3_m); pmap_load_store(pde, l3_pa | L2_TABLE); PTE_SYNC(pde); break; } } l3 = pmap_l2_to_l3(pde, va); pmap_invalidate_page(pmap, va); } havel3: om = NULL; orig_l3 = pmap_load(l3); opa = orig_l3 & ~ATTR_MASK; /* * Is the specified virtual address already mapped? */ if (pmap_l3_valid(orig_l3)) { /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there * are valid mappings in them. Hence, if a user page is wired, * the PT page will be also. */ if ((flags & PMAP_ENTER_WIRED) != 0 && (orig_l3 & ATTR_SW_WIRED) == 0) pmap->pm_stats.wired_count++; else if ((flags & PMAP_ENTER_WIRED) == 0 && (orig_l3 & ATTR_SW_WIRED) != 0) pmap->pm_stats.wired_count--; /* * Remove the extra PT page reference. */ if (mpte != NULL) { mpte->wire_count--; KASSERT(mpte->wire_count > 0, ("pmap_enter: missing reference to page table page," " va: 0x%lx", va)); } /* * Has the physical page changed? */ if (opa == pa) { /* * No, might be a protection or wiring change. */ if ((orig_l3 & ATTR_SW_MANAGED) != 0) { new_l3 |= ATTR_SW_MANAGED; if ((new_l3 & ATTR_AP(ATTR_AP_RW)) == ATTR_AP(ATTR_AP_RW)) { vm_page_aflag_set(m, PGA_WRITEABLE); } } goto validate; } /* Flush the cache, there might be uncommitted data in it */ if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(orig_l3)) cpu_dcache_wb_range(va, L3_SIZE); } else { /* * Increment the counters. */ if ((new_l3 & ATTR_SW_WIRED) != 0) pmap->pm_stats.wired_count++; pmap_resident_count_inc(pmap, 1); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0) { new_l3 |= ATTR_SW_MANAGED; pv = get_pv_entry(pmap, &lock); pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if ((new_l3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) vm_page_aflag_set(m, PGA_WRITEABLE); } /* * Update the L3 entry. */ if (orig_l3 != 0) { validate: orig_l3 = pmap_load(l3); opa = orig_l3 & ~ATTR_MASK; if (opa != pa) { pmap_update_entry(pmap, l3, new_l3, va, PAGE_SIZE); if ((orig_l3 & ATTR_SW_MANAGED) != 0) { om = PHYS_TO_VM_PAGE(opa); if (pmap_page_dirty(orig_l3)) vm_page_dirty(om); if ((orig_l3 & ATTR_AF) != 0) vm_page_aflag_set(om, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); pmap_pvh_free(&om->md, pmap, va); if ((om->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&om->md.pv_list) && ((om->flags & PG_FICTITIOUS) != 0 || TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) vm_page_aflag_clear(om, PGA_WRITEABLE); } } else { pmap_load_store(l3, new_l3); PTE_SYNC(l3); pmap_invalidate_page(pmap, va); if (pmap_page_dirty(orig_l3) && (orig_l3 & ATTR_SW_MANAGED) != 0) vm_page_dirty(m); } } else { pmap_load_store(l3, new_l3); } PTE_SYNC(l3); pmap_invalidate_page(pmap, va); - if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) - cpu_icache_sync_range(va, PAGE_SIZE); + if (pmap != pmap_kernel()) { + if (pmap == &curproc->p_vmspace->vm_pmap) + cpu_icache_sync_range(va, PAGE_SIZE); - if ((mpte == NULL || mpte->wire_count == NL3PG) && - pmap_superpages_enabled() && (m->flags & PG_FICTITIOUS) == 0 && - vm_reserv_level_iffullpop(m) == 0) { - KASSERT(lvl == 2, ("Invalid pde level %d", lvl)); - pmap_promote_l2(pmap, pde, va, &lock); + if ((mpte == NULL || mpte->wire_count == NL3PG) && + pmap_superpages_enabled() && + (m->flags & PG_FICTITIOUS) == 0 && + vm_reserv_level_iffullpop(m) == 0) { + KASSERT(lvl == 2, ("Invalid pde level %d", lvl)); + pmap_promote_l2(pmap, pde, va, &lock); + } } if (lock != NULL) rw_wunlock(lock); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { struct rwlock *lock; vm_offset_t va; vm_page_t m, mpte; vm_pindex_t diff, psize; VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); mpte = NULL; m = m_start; lock = NULL; PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { va = start + ptoa(diff); mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte, &lock); m = TAILQ_NEXT(m, listq); } if (lock != NULL) rw_wunlock(lock); PMAP_UNLOCK(pmap); } /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No page table pages. * but is *MUCH* faster than pmap_enter... */ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { struct rwlock *lock; lock = NULL; PMAP_LOCK(pmap); (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); if (lock != NULL) rw_wunlock(lock); PMAP_UNLOCK(pmap); } static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) { struct spglist free; pd_entry_t *pde; pt_entry_t *l2, *l3; vm_paddr_t pa; int lvl; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->oflags & VPO_UNMANAGED) != 0, ("pmap_enter_quick_locked: managed mapping within the clean submap")); PMAP_LOCK_ASSERT(pmap, MA_OWNED); CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va); /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { vm_pindex_t l2pindex; /* * Calculate pagetable page index */ l2pindex = pmap_l2_pindex(va); if (mpte && (mpte->pindex == l2pindex)) { mpte->wire_count++; } else { /* * Get the l2 entry */ pde = pmap_pde(pmap, va, &lvl); /* * If the page table page is mapped, we just increment * the hold count, and activate it. Otherwise, we * attempt to allocate a page table page. If this * attempt fails, we don't retry. Instead, we give up. */ if (lvl == 1) { l2 = pmap_l1_to_l2(pde, va); if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) return (NULL); } if (lvl == 2 && pmap_load(pde) != 0) { mpte = PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK); mpte->wire_count++; } else { /* * Pass NULL instead of the PV list lock * pointer, because we don't intend to sleep. */ mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); if (mpte == NULL) return (mpte); } } l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); l3 = &l3[pmap_l3_index(va)]; } else { mpte = NULL; pde = pmap_pde(kernel_pmap, va, &lvl); KASSERT(pde != NULL, ("pmap_enter_quick_locked: Invalid page entry, va: 0x%lx", va)); KASSERT(lvl == 2, ("pmap_enter_quick_locked: Invalid level %d", lvl)); l3 = pmap_l2_to_l3(pde, va); } if (pmap_load(l3) != 0) { if (mpte != NULL) { mpte->wire_count--; mpte = NULL; } return (mpte); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0 && !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { if (mpte != NULL) { SLIST_INIT(&free); if (pmap_unwire_l3(pmap, va, mpte, &free)) { pmap_invalidate_page(pmap, va); pmap_free_zero_pages(&free); } mpte = NULL; } return (mpte); } /* * Increment counters */ pmap_resident_count_inc(pmap, 1); pa = VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | ATTR_IDX(m->md.pv_memattr) | ATTR_AP(ATTR_AP_RO) | L3_PAGE; /* * Now validate mapping with RO protection */ if ((m->oflags & VPO_UNMANAGED) == 0) pa |= ATTR_SW_MANAGED; pmap_load_store(l3, pa); PTE_SYNC(l3); pmap_invalidate_page(pmap, va); return (mpte); } /* * This code maps large physical mmap regions into the * processor address space. Note that some shortcuts * are taken, but the code works. */ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range * must have the wired attribute set. In contrast, invalid mappings * cannot have the wired attribute set, so they are ignored. * * The wired attribute of the page table entry is not a hardware feature, * so there is no need to invalidate any TLB entries. */ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t va_next; pd_entry_t *l0, *l1, *l2; pt_entry_t *l3; PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { l0 = pmap_l0(pmap, sva); if (pmap_load(l0) == 0) { va_next = (sva + L0_SIZE) & ~L0_OFFSET; if (va_next < sva) va_next = eva; continue; } l1 = pmap_l0_to_l1(l0, sva); if (pmap_load(l1) == 0) { va_next = (sva + L1_SIZE) & ~L1_OFFSET; if (va_next < sva) va_next = eva; continue; } va_next = (sva + L2_SIZE) & ~L2_OFFSET; if (va_next < sva) va_next = eva; l2 = pmap_l1_to_l2(l1, sva); if (pmap_load(l2) == 0) continue; if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) { l3 = pmap_demote_l2(pmap, l2, sva); if (l3 == NULL) continue; } KASSERT((pmap_load(l2) & ATTR_DESCR_MASK) == L2_TABLE, ("pmap_unwire: Invalid l2 entry after demotion")); if (va_next > eva) va_next = eva; for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, sva += L3_SIZE) { if (pmap_load(l3) == 0) continue; if ((pmap_load(l3) & ATTR_SW_WIRED) == 0) panic("pmap_unwire: l3 %#jx is missing " "ATTR_SW_WIRED", (uintmax_t)pmap_load(l3)); /* * PG_W must be cleared atomically. Although the pmap * lock synchronizes access to PG_W, another processor * could be setting PG_M and/or PG_A concurrently. */ atomic_clear_long(l3, ATTR_SW_WIRED); pmap->pm_stats.wired_count--; } } PMAP_UNLOCK(pmap); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { } /* * pmap_zero_page zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. */ void pmap_zero_page(vm_page_t m) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); pagezero((void *)va); } /* * pmap_zero_page_area zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. * * off and size may not cover an area beyond a single hardware page. */ void pmap_zero_page_area(vm_page_t m, int off, int size) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); if (off == 0 && size == PAGE_SIZE) pagezero((void *)va); else bzero((char *)va + off, size); } /* * pmap_zero_page_idle zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. This * is intended to be called from the vm_pagezero process only and * outside of Giant. */ void pmap_zero_page_idle(vm_page_t m) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); pagezero((void *)va); } /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ void pmap_copy_page(vm_page_t msrc, vm_page_t mdst) { vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); pagecopy((void *)src, (void *)dst); } int unmapped_buf_allowed = 1; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { void *a_cp, *b_cp; vm_page_t m_a, m_b; vm_paddr_t p_a, p_b; vm_offset_t a_pg_offset, b_pg_offset; int cnt; while (xfersize > 0) { a_pg_offset = a_offset & PAGE_MASK; m_a = ma[a_offset >> PAGE_SHIFT]; p_a = m_a->phys_addr; b_pg_offset = b_offset & PAGE_MASK; m_b = mb[b_offset >> PAGE_SHIFT]; p_b = m_b->phys_addr; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); cnt = min(cnt, PAGE_SIZE - b_pg_offset); if (__predict_false(!PHYS_IN_DMAP(p_a))) { panic("!DMAP a %lx", p_a); } else { a_cp = (char *)PHYS_TO_DMAP(p_a) + a_pg_offset; } if (__predict_false(!PHYS_IN_DMAP(p_b))) { panic("!DMAP b %lx", p_b); } else { b_cp = (char *)PHYS_TO_DMAP(p_b) + b_pg_offset; } bcopy(a_cp, b_cp, cnt); a_offset += cnt; b_offset += cnt; xfersize -= cnt; } } vm_offset_t pmap_quick_enter_page(vm_page_t m) { return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m))); } void pmap_quick_remove_page(vm_offset_t addr) { } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { struct md_page *pvh; struct rwlock *lock; pv_entry_t pv; int loops = 0; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } } rw_runlock(lock); return (rv); } /* * pmap_page_wired_mappings: * * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { struct rwlock *lock; struct md_page *pvh; pmap_t pmap; pt_entry_t *pte; pv_entry_t pv; int count, lvl, md_gen, pvh_gen; if ((m->oflags & VPO_UNMANAGED) != 0) return (0); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: count = 0; TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pte(pmap, pv->pv_va, &lvl); if (pte != NULL && (pmap_load(pte) & ATTR_SW_WIRED) != 0) count++; PMAP_UNLOCK(pmap); } if ((m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; pvh_gen = pvh->pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen || pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pte(pmap, pv->pv_va, &lvl); if (pte != NULL && (pmap_load(pte) & ATTR_SW_WIRED) != 0) count++; PMAP_UNLOCK(pmap); } } rw_runlock(lock); return (count); } /* * Destroy all managed, non-wired mappings in the given user-space * pmap. This pmap cannot be active on any processor besides the * caller. * * This function cannot be applied to the kernel pmap. Moreover, it * is not intended for general use. It is only to be used during * process termination. Consequently, it can be implemented in ways * that make it faster than pmap_remove(). First, it can more quickly * destroy mappings by iterating over the pmap's collection of PV * entries, rather than searching the page table. Second, it doesn't * have to test and clear the page table entries atomically, because * no processor is currently accessing the user address space. In * particular, a page table entry's dirty bit won't change state once * this function starts. */ void pmap_remove_pages(pmap_t pmap) { pd_entry_t *pde; pt_entry_t *pte, tpte; struct spglist free; vm_page_t m, ml3, mt; pv_entry_t pv; struct md_page *pvh; struct pv_chunk *pc, *npc; struct rwlock *lock; int64_t bit; uint64_t inuse, bitmask; int allfree, field, freed, idx, lvl; vm_paddr_t pa; lock = NULL; SLIST_INIT(&free); PMAP_LOCK(pmap); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; freed = 0; for (field = 0; field < _NPCM; field++) { inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = ffsl(inuse) - 1; bitmask = 1UL << bit; idx = field * 64 + bit; pv = &pc->pc_pventry[idx]; inuse &= ~bitmask; pde = pmap_pde(pmap, pv->pv_va, &lvl); KASSERT(pde != NULL, ("Attempting to remove an unmapped page")); switch(lvl) { case 1: pte = pmap_l1_to_l2(pde, pv->pv_va); tpte = pmap_load(pte); KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK, ("Attempting to remove an invalid " "block: %lx", tpte)); tpte = pmap_load(pte); break; case 2: pte = pmap_l2_to_l3(pde, pv->pv_va); tpte = pmap_load(pte); KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE, ("Attempting to remove an invalid " "page: %lx", tpte)); break; default: panic( "Invalid page directory level: %d", lvl); } /* * We cannot remove wired pages from a process' mapping at this time */ if (tpte & ATTR_SW_WIRED) { allfree = 0; continue; } pa = tpte & ~ATTR_MASK; m = PHYS_TO_VM_PAGE(pa); KASSERT(m->phys_addr == pa, ("vm_page_t %p phys_addr mismatch %016jx %016jx", m, (uintmax_t)m->phys_addr, (uintmax_t)tpte)); KASSERT((m->flags & PG_FICTITIOUS) != 0 || m < &vm_page_array[vm_page_array_size], ("pmap_remove_pages: bad pte %#jx", (uintmax_t)tpte)); if (pmap_is_current(pmap)) { if (lvl == 2 && pmap_l3_valid_cacheable(tpte)) { cpu_dcache_wb_range(pv->pv_va, L3_SIZE); } else if (lvl == 1 && pmap_pte_valid_cacheable(tpte)) { cpu_dcache_wb_range(pv->pv_va, L2_SIZE); } } pmap_load_clear(pte); PTE_SYNC(pte); pmap_invalidate_page(pmap, pv->pv_va); /* * Update the vm_page_t clean/reference bits. */ if ((tpte & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) { switch (lvl) { case 1: for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) vm_page_dirty(m); break; case 2: vm_page_dirty(m); break; } } CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); /* Mark free */ pc->pc_map[field] |= bitmask; switch (lvl) { case 1: pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE); pvh = pa_to_pvh(tpte & ~ATTR_MASK); TAILQ_REMOVE(&pvh->pv_list, pv,pv_next); pvh->pv_gen++; if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) if ((mt->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&mt->md.pv_list)) vm_page_aflag_clear(mt, PGA_WRITEABLE); } ml3 = pmap_lookup_pt_page(pmap, pv->pv_va); if (ml3 != NULL) { pmap_remove_pt_page(pmap, ml3); pmap_resident_count_dec(pmap,1); KASSERT(ml3->wire_count == NL3PG, ("pmap_remove_pages: l3 page wire count error")); ml3->wire_count = 0; pmap_add_delayed_free_list(ml3, &free, FALSE); atomic_subtract_int( &vm_cnt.v_wire_count, 1); } break; case 2: pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if ((m->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh( VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } break; } pmap_unuse_l3(pmap, pv->pv_va, pmap_load(pde), &free); freed++; } } PV_STAT(atomic_add_long(&pv_entry_frees, freed)); PV_STAT(atomic_add_int(&pv_entry_spare, freed)); PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); if (allfree) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } } pmap_invalidate_all(pmap); if (lock != NULL) rw_wunlock(lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * This is used to check if a page has been accessed or modified. As we * don't have a bit to see if it has been modified we have to assume it * has been if the page is read/write. */ static boolean_t pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) { struct rwlock *lock; pv_entry_t pv; struct md_page *pvh; pt_entry_t *pte, mask, value; pmap_t pmap; int lvl, md_gen, pvh_gen; boolean_t rv; rv = FALSE; lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pte(pmap, pv->pv_va, &lvl); KASSERT(lvl == 3, ("pmap_page_test_mappings: Invalid level %d", lvl)); mask = 0; value = 0; if (modified) { mask |= ATTR_AP_RW_BIT; value |= ATTR_AP(ATTR_AP_RW); } if (accessed) { mask |= ATTR_AF | ATTR_DESCR_MASK; value |= ATTR_AF | L3_PAGE; } rv = (pmap_load(pte) & mask) == value; PMAP_UNLOCK(pmap); if (rv) goto out; } if ((m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; pvh_gen = pvh->pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen || pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pte(pmap, pv->pv_va, &lvl); KASSERT(lvl == 2, ("pmap_page_test_mappings: Invalid level %d", lvl)); mask = 0; value = 0; if (modified) { mask |= ATTR_AP_RW_BIT; value |= ATTR_AP(ATTR_AP_RW); } if (accessed) { mask |= ATTR_AF | ATTR_DESCR_MASK; value |= ATTR_AF | L2_BLOCK; } rv = (pmap_load(pte) & mask) == value; PMAP_UNLOCK(pmap); if (rv) goto out; } } out: rw_runlock(lock); return (rv); } /* * pmap_is_modified: * * Return whether or not the specified physical page was modified * in any physical maps. */ boolean_t pmap_is_modified(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_modified: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PG_M set. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); return (pmap_page_test_mappings(m, FALSE, TRUE)); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is eligible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { pt_entry_t *pte; boolean_t rv; int lvl; rv = FALSE; PMAP_LOCK(pmap); pte = pmap_pte(pmap, addr, &lvl); if (pte != NULL && pmap_load(pte) != 0) { rv = TRUE; } PMAP_UNLOCK(pmap); return (rv); } /* * pmap_is_referenced: * * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_referenced: page %p is not managed", m)); return (pmap_page_test_mappings(m, TRUE, FALSE)); } /* * Clear the write and modified bits in each of the given page's mappings. */ void pmap_remove_write(vm_page_t m) { struct md_page *pvh; pmap_t pmap; struct rwlock *lock; pv_entry_t next_pv, pv; pt_entry_t oldpte, *pte; vm_offset_t va; int lvl, md_gen, pvh_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_write: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; lock = VM_PAGE_TO_PV_LIST_LOCK(m); pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(VM_PAGE_TO_PHYS(m)); retry_pv_loop: rw_wlock(lock); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); rw_wunlock(lock); goto retry_pv_loop; } } va = pv->pv_va; pte = pmap_pte(pmap, pv->pv_va, &lvl); if ((pmap_load(pte) & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) pmap_demote_l2_locked(pmap, pte, va & ~L2_OFFSET, &lock); KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); PMAP_UNLOCK(pmap); } TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; md_gen = m->md.pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); rw_wunlock(lock); goto retry_pv_loop; } } pte = pmap_pte(pmap, pv->pv_va, &lvl); retry: oldpte = pmap_load(pte); if ((oldpte & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) { if (!atomic_cmpset_long(pte, oldpte, oldpte | ATTR_AP(ATTR_AP_RO))) goto retry; if ((oldpte & ATTR_AF) != 0) vm_page_dirty(m); pmap_invalidate_page(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } rw_wunlock(lock); vm_page_aflag_clear(m, PGA_WRITEABLE); } static __inline boolean_t safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) { return (FALSE); } #define PMAP_TS_REFERENCED_MAX 5 /* * pmap_ts_referenced: * * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct md_page *pvh; pv_entry_t pv, pvf; pmap_t pmap; struct rwlock *lock; pd_entry_t *pde, tpde; pt_entry_t *pte, tpte; pt_entry_t *l3; vm_offset_t va; vm_paddr_t pa; int cleared, md_gen, not_cleared, lvl, pvh_gen; struct spglist free; bool demoted; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); SLIST_INIT(&free); cleared = 0; pa = VM_PAGE_TO_PHYS(m); lock = PHYS_TO_PV_LIST_LOCK(pa); pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); rw_wlock(lock); retry: not_cleared = 0; if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) goto small_mappings; pv = pvf; do { if (pvf == NULL) pvf = pv; pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); goto retry; } } va = pv->pv_va; pde = pmap_pde(pmap, pv->pv_va, &lvl); KASSERT(pde != NULL, ("pmap_ts_referenced: no l1 table found")); KASSERT(lvl == 1, ("pmap_ts_referenced: invalid pde level %d", lvl)); tpde = pmap_load(pde); KASSERT((tpde & ATTR_DESCR_MASK) == L1_TABLE, ("pmap_ts_referenced: found an invalid l1 table")); pte = pmap_l1_to_l2(pde, pv->pv_va); tpte = pmap_load(pte); if ((tpte & ATTR_AF) != 0) { /* * Since this reference bit is shared by 512 4KB * pages, it should not be cleared every time it is * tested. Apply a simple "hash" function on the * physical page number, the virtual superpage number, * and the pmap address to select one 4KB page out of * the 512 on which testing the reference bit will * result in clearing that reference bit. This * function is designed to avoid the selection of the * same 4KB page for every 2MB page mapping. * * On demotion, a mapping that hasn't been referenced * is simply destroyed. To avoid the possibility of a * subsequent page fault on a demoted wired mapping, * always leave its reference bit set. Moreover, * since the superpage is wired, the current state of * its reference bit won't affect page replacement. */ if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L2_SHIFT) ^ (uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 && (tpte & ATTR_SW_WIRED) == 0) { if (safe_to_clear_referenced(pmap, tpte)) { /* * TODO: We don't handle the access * flag at all. We need to be able * to set it in the exception handler. */ panic("ARM64TODO: " "safe_to_clear_referenced\n"); } else if (pmap_demote_l2_locked(pmap, pte, pv->pv_va, &lock) != NULL) { demoted = true; va += VM_PAGE_TO_PHYS(m) - (tpte & ~ATTR_MASK); l3 = pmap_l2_to_l3(pte, va); pmap_remove_l3(pmap, l3, va, pmap_load(pte), NULL, &lock); } else demoted = true; if (demoted) { /* * The superpage mapping was removed * entirely and therefore 'pv' is no * longer valid. */ if (pvf == pv) pvf = NULL; pv = NULL; } cleared++; KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } else not_cleared++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; } if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) goto out; } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); small_mappings: if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) goto out; pv = pvf; do { if (pvf == NULL) pvf = pv; pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; md_gen = m->md.pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto retry; } } pde = pmap_pde(pmap, pv->pv_va, &lvl); KASSERT(pde != NULL, ("pmap_ts_referenced: no l2 table found")); KASSERT(lvl == 2, ("pmap_ts_referenced: invalid pde level %d", lvl)); tpde = pmap_load(pde); KASSERT((tpde & ATTR_DESCR_MASK) == L2_TABLE, ("pmap_ts_referenced: found an invalid l2 table")); pte = pmap_l2_to_l3(pde, pv->pv_va); tpte = pmap_load(pte); if ((tpte & ATTR_AF) != 0) { if (safe_to_clear_referenced(pmap, tpte)) { /* * TODO: We don't handle the access flag * at all. We need to be able to set it in * the exception handler. */ panic("ARM64TODO: safe_to_clear_referenced\n"); } else if ((tpte & ATTR_SW_WIRED) == 0) { /* * Wired pages cannot be paged out so * doing accessed bit emulation for * them is wasted effort. We do the * hard work for unwired pages only. */ pmap_remove_l3(pmap, pte, pv->pv_va, tpde, &free, &lock); pmap_invalidate_page(pmap, pv->pv_va); cleared++; if (pvf == pv) pvf = NULL; pv = NULL; KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } else not_cleared++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; } } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + not_cleared < PMAP_TS_REFERENCED_MAX); out: rw_wunlock(lock); pmap_free_zero_pages(&free); return (cleared + not_cleared); } /* * Apply the given advice to the specified range of addresses within the * given pmap. Depending on the advice, clear the referenced and/or * modified flags in each mapping and set the mapped page's dirty field. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("pmap_clear_modify: page %p is exclusive busied", m)); /* * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. * If the object containing the page is locked and the page is not * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; /* ARM64TODO: We lack support for tracking if a page is modified */ } void * pmap_mapbios(vm_paddr_t pa, vm_size_t size) { return ((void *)PHYS_TO_DMAP(pa)); } void pmap_unmapbios(vm_paddr_t pa, vm_size_t size) { } /* * Sets the memory attribute for the specified page. */ void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { m->md.pv_memattr = ma; /* * If "m" is a normal page, update its direct mapping. This update * can be relied upon to perform any cache operations that are * required for data coherence. */ if ((m->flags & PG_FICTITIOUS) == 0 && pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), PAGE_SIZE, m->md.pv_memattr) != 0) panic("memory attribute change on the direct map failed"); } /* * Changes the specified virtual address range's memory type to that given by * the parameter "mode". The specified virtual address range must be * completely contained within either the direct map or the kernel map. If * the virtual address range is contained within the kernel map, then the * memory type for each of the corresponding ranges of the direct map is also * changed. (The corresponding ranges of the direct map are those ranges that * map the same physical pages as the specified virtual address range.) These * changes to the direct map are necessary because Intel describes the * behavior of their processors as "undefined" if two or more mappings to the * same physical page have different memory types. * * Returns zero if the change completed successfully, and either EINVAL or * ENOMEM if the change failed. Specifically, EINVAL is returned if some part * of the virtual address range was not mapped, and ENOMEM is returned if * there was insufficient memory available to complete the change. In the * latter case, the memory type may have been changed on some part of the * virtual address range or the direct map. */ static int pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) { int error; PMAP_LOCK(kernel_pmap); error = pmap_change_attr_locked(va, size, mode); PMAP_UNLOCK(kernel_pmap); return (error); } static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode) { vm_offset_t base, offset, tmpva; pt_entry_t l3, *pte, *newpte; int lvl; PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); base = trunc_page(va); offset = va & PAGE_MASK; size = round_page(offset + size); if (!VIRT_IN_DMAP(base)) return (EINVAL); for (tmpva = base; tmpva < base + size; ) { pte = pmap_pte(kernel_pmap, va, &lvl); if (pte == NULL) return (EINVAL); if ((pmap_load(pte) & ATTR_IDX_MASK) == ATTR_IDX(mode)) { /* * We already have the correct attribute, * ignore this entry. */ switch (lvl) { default: panic("Invalid DMAP table level: %d\n", lvl); case 1: tmpva = (tmpva & ~L1_OFFSET) + L1_SIZE; break; case 2: tmpva = (tmpva & ~L2_OFFSET) + L2_SIZE; break; case 3: tmpva += PAGE_SIZE; break; } } else { /* * Split the entry to an level 3 table, then * set the new attribute. */ switch (lvl) { default: panic("Invalid DMAP table level: %d\n", lvl); case 1: newpte = pmap_demote_l1(kernel_pmap, pte, tmpva & ~L1_OFFSET); if (newpte == NULL) return (EINVAL); pte = pmap_l1_to_l2(pte, tmpva); case 2: newpte = pmap_demote_l2(kernel_pmap, pte, tmpva & ~L2_OFFSET); if (newpte == NULL) return (EINVAL); pte = pmap_l2_to_l3(pte, tmpva); case 3: /* Update the entry */ l3 = pmap_load(pte); l3 &= ~ATTR_IDX_MASK; l3 |= ATTR_IDX(mode); pmap_update_entry(kernel_pmap, pte, l3, tmpva, PAGE_SIZE); /* * If moving to a non-cacheable entry flush * the cache. */ if (mode == VM_MEMATTR_UNCACHEABLE) cpu_dcache_wbinv_range(tmpva, L3_SIZE); break; } tmpva += PAGE_SIZE; } } return (0); } /* * Create an L2 table to map all addresses within an L1 mapping. */ static pt_entry_t * pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va) { pt_entry_t *l2, newl2, oldl1; vm_offset_t tmpl1; vm_paddr_t l2phys, phys; vm_page_t ml2; int i; PMAP_LOCK_ASSERT(pmap, MA_OWNED); oldl1 = pmap_load(l1); KASSERT((oldl1 & ATTR_DESCR_MASK) == L1_BLOCK, ("pmap_demote_l1: Demoting a non-block entry")); KASSERT((va & L1_OFFSET) == 0, ("pmap_demote_l1: Invalid virtual address %#lx", va)); KASSERT((oldl1 & ATTR_SW_MANAGED) == 0, ("pmap_demote_l1: Level 1 table shouldn't be managed")); tmpl1 = 0; if (va <= (vm_offset_t)l1 && va + L1_SIZE > (vm_offset_t)l1) { tmpl1 = kva_alloc(PAGE_SIZE); if (tmpl1 == 0) return (NULL); } if ((ml2 = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { CTR2(KTR_PMAP, "pmap_demote_l1: failure for va %#lx" " in pmap %p", va, pmap); return (NULL); } l2phys = VM_PAGE_TO_PHYS(ml2); l2 = (pt_entry_t *)PHYS_TO_DMAP(l2phys); /* Address the range points at */ phys = oldl1 & ~ATTR_MASK; /* The attributed from the old l1 table to be copied */ newl2 = oldl1 & ATTR_MASK; /* Create the new entries */ for (i = 0; i < Ln_ENTRIES; i++) { l2[i] = newl2 | phys; phys += L2_SIZE; } cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); KASSERT(l2[0] == ((oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK), ("Invalid l2 page (%lx != %lx)", l2[0], (oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK)); if (tmpl1 != 0) { pmap_kenter(tmpl1, PAGE_SIZE, DMAP_TO_PHYS((vm_offset_t)l1) & ~L3_OFFSET, CACHED_MEMORY); l1 = (pt_entry_t *)(tmpl1 + ((vm_offset_t)l1 & PAGE_MASK)); } pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va, PAGE_SIZE); if (tmpl1 != 0) { pmap_kremove(tmpl1); kva_free(tmpl1, PAGE_SIZE); } return (l2); } /* * Create an L3 table to map all addresses within an L2 mapping. */ static pt_entry_t * pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va, struct rwlock **lockp) { pt_entry_t *l3, newl3, oldl2; vm_offset_t tmpl2; vm_paddr_t l3phys, phys; vm_page_t ml3; int i; PMAP_LOCK_ASSERT(pmap, MA_OWNED); l3 = NULL; oldl2 = pmap_load(l2); KASSERT((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK, ("pmap_demote_l2: Demoting a non-block entry")); KASSERT((va & L2_OFFSET) == 0, ("pmap_demote_l2: Invalid virtual address %#lx", va)); tmpl2 = 0; if (va <= (vm_offset_t)l2 && va + L2_SIZE > (vm_offset_t)l2) { tmpl2 = kva_alloc(PAGE_SIZE); if (tmpl2 == 0) return (NULL); } if ((ml3 = pmap_lookup_pt_page(pmap, va)) != NULL) { pmap_remove_pt_page(pmap, ml3); } else { ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va), (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (ml3 == NULL) { CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" " in pmap %p", va, pmap); goto fail; } if (va < VM_MAXUSER_ADDRESS) pmap_resident_count_inc(pmap, 1); } l3phys = VM_PAGE_TO_PHYS(ml3); l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); /* Address the range points at */ phys = oldl2 & ~ATTR_MASK; /* The attributed from the old l2 table to be copied */ newl3 = (oldl2 & (ATTR_MASK & ~ATTR_DESCR_MASK)) | L3_PAGE; /* * If the page table page is new, initialize it. */ if (ml3->wire_count == 1) { for (i = 0; i < Ln_ENTRIES; i++) { l3[i] = newl3 | phys; phys += L3_SIZE; } cpu_dcache_wb_range((vm_offset_t)l3, PAGE_SIZE); } KASSERT(l3[0] == ((oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE), ("Invalid l3 page (%lx != %lx)", l3[0], (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE)); /* * Map the temporary page so we don't lose access to the l2 table. */ if (tmpl2 != 0) { pmap_kenter(tmpl2, PAGE_SIZE, DMAP_TO_PHYS((vm_offset_t)l2) & ~L3_OFFSET, CACHED_MEMORY); l2 = (pt_entry_t *)(tmpl2 + ((vm_offset_t)l2 & PAGE_MASK)); } /* * The spare PV entries must be reserved prior to demoting the * mapping, that is, prior to changing the PDE. Otherwise, the state * of the L2 and the PV lists will be inconsistent, which can result * in reclaim_pv_chunk() attempting to remove a PV entry from the * wrong PV list and pmap_pv_demote_l2() failing to find the expected * PV entry for the 2MB page mapping that is being demoted. */ if ((oldl2 & ATTR_SW_MANAGED) != 0) reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp); pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va, PAGE_SIZE); /* * Demote the PV entry. */ if ((oldl2 & ATTR_SW_MANAGED) != 0) pmap_pv_demote_l2(pmap, va, oldl2 & ~ATTR_MASK, lockp); atomic_add_long(&pmap_l2_demotions, 1); CTR3(KTR_PMAP, "pmap_demote_l2: success for va %#lx" " in pmap %p %lx", va, pmap, l3[0]); fail: if (tmpl2 != 0) { pmap_kremove(tmpl2); kva_free(tmpl2, PAGE_SIZE); } return (l3); } static pt_entry_t * pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va) { struct rwlock *lock; pt_entry_t *l3; lock = NULL; l3 = pmap_demote_l2_locked(pmap, l2, va, &lock); if (lock != NULL) rw_wunlock(lock); return (l3); } /* * perform the pmap work for mincore */ int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { pd_entry_t *l1p, l1; pd_entry_t *l2p, l2; pt_entry_t *l3p, l3; vm_paddr_t pa; bool managed; int val; PMAP_LOCK(pmap); retry: pa = 0; val = 0; managed = false; l1p = pmap_l1(pmap, addr); if (l1p == NULL) /* No l1 */ goto done; l1 = pmap_load(l1p); if ((l1 & ATTR_DESCR_MASK) == L1_INVAL) goto done; if ((l1 & ATTR_DESCR_MASK) == L1_BLOCK) { pa = (l1 & ~ATTR_MASK) | (addr & L1_OFFSET); managed = (l1 & ATTR_SW_MANAGED) == ATTR_SW_MANAGED; val = MINCORE_SUPER | MINCORE_INCORE; if (pmap_page_dirty(l1)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if ((l1 & ATTR_AF) == ATTR_AF) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; goto done; } l2p = pmap_l1_to_l2(l1p, addr); if (l2p == NULL) /* No l2 */ goto done; l2 = pmap_load(l2p); if ((l2 & ATTR_DESCR_MASK) == L2_INVAL) goto done; if ((l2 & ATTR_DESCR_MASK) == L2_BLOCK) { pa = (l2 & ~ATTR_MASK) | (addr & L2_OFFSET); managed = (l2 & ATTR_SW_MANAGED) == ATTR_SW_MANAGED; val = MINCORE_SUPER | MINCORE_INCORE; if (pmap_page_dirty(l2)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if ((l2 & ATTR_AF) == ATTR_AF) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; goto done; } l3p = pmap_l2_to_l3(l2p, addr); if (l3p == NULL) /* No l3 */ goto done; l3 = pmap_load(l2p); if ((l3 & ATTR_DESCR_MASK) == L3_INVAL) goto done; if ((l3 & ATTR_DESCR_MASK) == L3_PAGE) { pa = (l3 & ~ATTR_MASK) | (addr & L3_OFFSET); managed = (l3 & ATTR_SW_MANAGED) == ATTR_SW_MANAGED; val = MINCORE_INCORE; if (pmap_page_dirty(l3)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if ((l3 & ATTR_AF) == ATTR_AF) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } done: if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) goto retry; } else PA_UNLOCK_COND(*locked_pa); PMAP_UNLOCK(pmap); return (val); } void pmap_activate(struct thread *td) { pmap_t pmap; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); td->td_pcb->pcb_l0addr = vtophys(pmap->pm_l0); __asm __volatile("msr ttbr0_el1, %0" : : "r"(td->td_pcb->pcb_l0addr)); pmap_invalidate_all(pmap); critical_exit(); } void pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz) { if (va >= VM_MIN_KERNEL_ADDRESS) { cpu_icache_sync_range(va, sz); } else { u_int len, offset; vm_paddr_t pa; /* Find the length of data in this page to flush */ offset = va & PAGE_MASK; len = imin(PAGE_SIZE - offset, sz); while (sz != 0) { /* Extract the physical address & find it in the DMAP */ pa = pmap_extract(pmap, va); if (pa != 0) cpu_icache_sync_range(PHYS_TO_DMAP(pa), len); /* Move to the next page */ sz -= len; va += len; /* Set the length for the next iteration */ len = imin(PAGE_SIZE, sz); } } } int pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far) { #ifdef SMP uint64_t par; #endif switch (ESR_ELx_EXCEPTION(esr)) { case EXCP_DATA_ABORT_L: case EXCP_DATA_ABORT: break; default: return (KERN_FAILURE); } #ifdef SMP PMAP_LOCK(pmap); switch (esr & ISS_DATA_DFSC_MASK) { case ISS_DATA_DFSC_TF_L0: case ISS_DATA_DFSC_TF_L1: case ISS_DATA_DFSC_TF_L2: case ISS_DATA_DFSC_TF_L3: /* Ask the MMU to check the address */ if (pmap == kernel_pmap) par = arm64_address_translate_s1e1r(far); else par = arm64_address_translate_s1e0r(far); /* * If the translation was successful the address was invalid * due to a break-before-make sequence. We can unlock and * return success to the trap handler. */ if (PAR_SUCCESS(par)) { PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } break; default: break; } PMAP_UNLOCK(pmap); #endif return (KERN_FAILURE); } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { vm_offset_t superpage_offset; if (size < L2_SIZE) return; if (object != NULL && (object->flags & OBJ_COLORED) != 0) offset += ptoa(object->pg_color); superpage_offset = offset & L2_OFFSET; if (size - ((L2_SIZE - superpage_offset) & L2_OFFSET) < L2_SIZE || (*addr & L2_OFFSET) == superpage_offset) return; if ((*addr & L2_OFFSET) < superpage_offset) *addr = (*addr & ~L2_OFFSET) + superpage_offset; else *addr = ((*addr + L2_OFFSET) & ~L2_OFFSET) + superpage_offset; } /** * Get the kernel virtual address of a set of physical pages. If there are * physical addresses not covered by the DMAP perform a transient mapping * that will be removed when calling pmap_unmap_io_transient. * * \param page The pages the caller wishes to obtain the virtual * address on the kernel memory map. * \param vaddr On return contains the kernel virtual memory address * of the pages passed in the page parameter. * \param count Number of pages passed in. * \param can_fault TRUE if the thread using the mapped pages can take * page faults, FALSE otherwise. * * \returns TRUE if the caller must call pmap_unmap_io_transient when * finished or FALSE otherwise. * */ boolean_t pmap_map_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, boolean_t can_fault) { vm_paddr_t paddr; boolean_t needs_mapping; int error, i; /* * Allocate any KVA space that we need, this is done in a separate * loop to prevent calling vmem_alloc while pinned. */ needs_mapping = FALSE; for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (__predict_false(!PHYS_IN_DMAP(paddr))) { error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, &vaddr[i]); KASSERT(error == 0, ("vmem_alloc failed: %d", error)); needs_mapping = TRUE; } else { vaddr[i] = PHYS_TO_DMAP(paddr); } } /* Exit early if everything is covered by the DMAP */ if (!needs_mapping) return (FALSE); if (!can_fault) sched_pin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (!PHYS_IN_DMAP(paddr)) { panic( "pmap_map_io_transient: TODO: Map out of DMAP data"); } } return (needs_mapping); } void pmap_unmap_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, boolean_t can_fault) { vm_paddr_t paddr; int i; if (!can_fault) sched_unpin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (!PHYS_IN_DMAP(paddr)) { panic("ARM64TODO: pmap_unmap_io_transient: Unmap data"); } } } Index: projects/clang390-import/sys/boot/powerpc/boot1.chrp/boot1.c =================================================================== --- projects/clang390-import/sys/boot/powerpc/boot1.chrp/boot1.c (revision 305080) +++ projects/clang390-import/sys/boot/powerpc/boot1.chrp/boot1.c (revision 305081) @@ -1,766 +1,777 @@ /*- * Copyright (c) 1998 Robert Nordier * All rights reserved. * Copyright (c) 2001 Robert Drehmel * All rights reserved. * * Redistribution and use in source and binary forms are freely * permitted provided that the above copyright notice and this * paragraph and the following disclaimer are duplicated in all * such forms. * * This software is provided "AS IS" and without any express or * implied warranties, including, without limitation, the implied * warranties of merchantability and fitness for a particular * purpose. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include "paths.h" #define BSIZEMAX 16384 typedef int putc_func_t(char c, void *arg); typedef int32_t ofwh_t; struct sp_data { char *sp_buf; u_int sp_len; u_int sp_size; }; static const char digits[] = "0123456789abcdef"; static char bootpath[128]; static char bootargs[128]; static ofwh_t bootdev; static struct fs fs; static char blkbuf[BSIZEMAX]; static unsigned int fsblks; static uint32_t fs_off; int main(int ac, char **av); static void exit(int) __dead2; static void load(const char *); static int dskread(void *, u_int64_t, int); static void usage(void); static void bcopy(const void *src, void *dst, size_t len); static void bzero(void *b, size_t len); static int domount(const char *device, int quiet); static void panic(const char *fmt, ...) __dead2; static int printf(const char *fmt, ...); static int putchar(char c, void *arg); static int vprintf(const char *fmt, va_list ap); static int vsnprintf(char *str, size_t sz, const char *fmt, va_list ap); static int __printf(const char *fmt, putc_func_t *putc, void *arg, va_list ap); static int __putc(char c, void *arg); static int __puts(const char *s, putc_func_t *putc, void *arg); static int __sputc(char c, void *arg); static char *__uitoa(char *buf, u_int val, int base); static char *__ultoa(char *buf, u_long val, int base); void __syncicache(void *, int); /* * Open Firmware interface functions */ typedef u_int32_t ofwcell_t; typedef u_int32_t u_ofwh_t; typedef int (*ofwfp_t)(void *); ofwfp_t ofw; /* the prom Open Firmware entry */ ofwh_t chosenh; void ofw_init(void *, int, int (*)(void *), char *, int); static ofwh_t ofw_finddevice(const char *); static ofwh_t ofw_open(const char *); static int ofw_close(ofwh_t); static int ofw_getprop(ofwh_t, const char *, void *, size_t); static int ofw_setprop(ofwh_t, const char *, void *, size_t); static int ofw_read(ofwh_t, void *, size_t); static int ofw_write(ofwh_t, const void *, size_t); static int ofw_claim(void *virt, size_t len, u_int align); static int ofw_seek(ofwh_t, u_int64_t); static void ofw_exit(void) __dead2; ofwh_t bootdevh; ofwh_t stdinh, stdouth; __asm(" \n\ .data \n\ .align 4 \n\ stack: \n\ .space 16384 \n\ \n\ .text \n\ .globl _start \n\ _start: \n\ lis %r1,stack@ha \n\ addi %r1,%r1,stack@l \n\ addi %r1,%r1,8192 \n\ \n\ b ofw_init \n\ "); void ofw_init(void *vpd, int res, int (*openfirm)(void *), char *arg, int argl) { char *av[16]; char *p; int ac; ofw = openfirm; chosenh = ofw_finddevice("/chosen"); ofw_getprop(chosenh, "stdin", &stdinh, sizeof(stdinh)); ofw_getprop(chosenh, "stdout", &stdouth, sizeof(stdouth)); ofw_getprop(chosenh, "bootargs", bootargs, sizeof(bootargs)); ofw_getprop(chosenh, "bootpath", bootpath, sizeof(bootpath)); bootargs[sizeof(bootargs) - 1] = '\0'; bootpath[sizeof(bootpath) - 1] = '\0'; p = bootpath; while (*p != '\0') { + /* Truncate partition ID */ if (*p == ':') { + ofw_close(bootdev); *(++p) = '\0'; break; } p++; } ac = 0; p = bootargs; for (;;) { while (*p == ' ' && *p != '\0') p++; if (*p == '\0' || ac >= 16) break; av[ac++] = p; while (*p != ' ' && *p != '\0') p++; if (*p != '\0') *p++ = '\0'; } exit(main(ac, av)); } static ofwh_t ofw_finddevice(const char *name) { ofwcell_t args[] = { (ofwcell_t)"finddevice", 1, 1, (ofwcell_t)name, 0 }; if ((*ofw)(args)) { printf("ofw_finddevice: name=\"%s\"\n", name); return (1); } return (args[4]); } static int ofw_getprop(ofwh_t ofwh, const char *name, void *buf, size_t len) { ofwcell_t args[] = { (ofwcell_t)"getprop", 4, 1, (u_ofwh_t)ofwh, (ofwcell_t)name, (ofwcell_t)buf, len, 0 }; if ((*ofw)(args)) { printf("ofw_getprop: ofwh=0x%x buf=%p len=%u\n", ofwh, buf, len); return (1); } return (0); } static int ofw_setprop(ofwh_t ofwh, const char *name, void *buf, size_t len) { ofwcell_t args[] = { (ofwcell_t)"setprop", 4, 1, (u_ofwh_t)ofwh, (ofwcell_t)name, (ofwcell_t)buf, len, 0 }; if ((*ofw)(args)) { printf("ofw_setprop: ofwh=0x%x buf=%p len=%u\n", ofwh, buf, len); return (1); } return (0); } static ofwh_t ofw_open(const char *path) { ofwcell_t args[] = { (ofwcell_t)"open", 1, 1, (ofwcell_t)path, 0 }; if ((*ofw)(args)) { printf("ofw_open: path=\"%s\"\n", path); return (-1); } return (args[4]); } static int ofw_close(ofwh_t devh) { ofwcell_t args[] = { (ofwcell_t)"close", 1, 0, (u_ofwh_t)devh }; if ((*ofw)(args)) { printf("ofw_close: devh=0x%x\n", devh); return (1); } return (0); } static int ofw_claim(void *virt, size_t len, u_int align) { ofwcell_t args[] = { (ofwcell_t)"claim", 3, 1, (ofwcell_t)virt, len, align, 0, 0 }; if ((*ofw)(args)) { printf("ofw_claim: virt=%p len=%u\n", virt, len); return (1); } return (0); } static int ofw_read(ofwh_t devh, void *buf, size_t len) { ofwcell_t args[] = { (ofwcell_t)"read", 3, 1, (u_ofwh_t)devh, (ofwcell_t)buf, len, 0 }; if ((*ofw)(args)) { printf("ofw_read: devh=0x%x buf=%p len=%u\n", devh, buf, len); return (1); } return (0); } static int ofw_write(ofwh_t devh, const void *buf, size_t len) { ofwcell_t args[] = { (ofwcell_t)"write", 3, 1, (u_ofwh_t)devh, (ofwcell_t)buf, len, 0 }; if ((*ofw)(args)) { printf("ofw_write: devh=0x%x buf=%p len=%u\n", devh, buf, len); return (1); } return (0); } static int ofw_seek(ofwh_t devh, u_int64_t off) { ofwcell_t args[] = { (ofwcell_t)"seek", 3, 1, (u_ofwh_t)devh, off >> 32, off, 0 }; if ((*ofw)(args)) { printf("ofw_seek: devh=0x%x off=0x%lx\n", devh, off); return (1); } return (0); } static void ofw_exit(void) { ofwcell_t args[3]; args[0] = (ofwcell_t)"exit"; args[1] = 0; args[2] = 0; for (;;) (*ofw)(args); } static void bcopy(const void *src, void *dst, size_t len) { const char *s = src; char *d = dst; while (len-- != 0) *d++ = *s++; } static void memcpy(void *dst, const void *src, size_t len) { bcopy(src, dst, len); } static void bzero(void *b, size_t len) { char *p = b; while (len-- != 0) *p++ = 0; } static int strcmp(const char *s1, const char *s2) { for (; *s1 == *s2 && *s1; s1++, s2++) ; return ((u_char)*s1 - (u_char)*s2); } #include "ufsread.c" int main(int ac, char **av) { const char *path; char bootpath_full[255]; int i, len; path = PATH_LOADER; for (i = 0; i < ac; i++) { switch (av[i][0]) { case '-': switch (av[i][1]) { default: usage(); } break; default: path = av[i]; break; } } printf(" \n>> FreeBSD/powerpc Open Firmware boot block\n" " Boot path: %s\n" " Boot loader: %s\n", bootpath, path); len = 0; while (bootpath[len] != '\0') len++; memcpy(bootpath_full,bootpath,len+1); - if (bootpath_full[len-1] == ':') { - for (i = 0; i < 16; i++) { - if (i < 10) { - bootpath_full[len] = i + '0'; - bootpath_full[len+1] = '\0'; - } else { - bootpath_full[len] = '1'; - bootpath_full[len+1] = i - 10 + '0'; - bootpath_full[len+2] = '\0'; - } - - if (domount(bootpath_full,1) >= 0) - break; + if (bootpath_full[len-1] != ':') { + /* First try full volume */ + if (domount(bootpath_full,1) == 0) + goto out; - if (bootdev > 0) - ofw_close(bootdev); + /* Add a : so that we try partitions if that fails */ + if (bootdev > 0) + ofw_close(bootdev); + bootpath_full[len] = ':'; + len += 1; + } + + /* Loop through first 16 partitions to find a UFS one */ + for (i = 0; i < 16; i++) { + if (i < 10) { + bootpath_full[len] = i + '0'; + bootpath_full[len+1] = '\0'; + } else { + bootpath_full[len] = '1'; + bootpath_full[len+1] = i - 10 + '0'; + bootpath_full[len+2] = '\0'; } + + if (domount(bootpath_full,1) >= 0) + break; - if (i >= 16) - panic("domount"); - } else { - if (domount(bootpath_full,0) == -1) - panic("domount"); + if (bootdev > 0) + ofw_close(bootdev); } + if (i >= 16) + panic("domount"); + +out: printf(" Boot volume: %s\n",bootpath_full); ofw_setprop(chosenh, "bootargs", bootpath_full, len+2); load(path); return (1); } static void usage(void) { printf("usage: boot device [/path/to/loader]\n"); exit(1); } static void exit(int code) { ofw_exit(); } static struct dmadat __dmadat; static int domount(const char *device, int quiet) { dmadat = &__dmadat; if ((bootdev = ofw_open(device)) == -1) { printf("domount: can't open device\n"); return (-1); } if (fsread(0, NULL, 0)) { if (!quiet) printf("domount: can't read superblock\n"); return (-1); } return (0); } static void load(const char *fname) { Elf32_Ehdr eh; Elf32_Phdr ph; caddr_t p; ufs_ino_t ino; int i; if ((ino = lookup(fname)) == 0) { printf("File %s not found\n", fname); return; } if (fsread(ino, &eh, sizeof(eh)) != sizeof(eh)) { printf("Can't read elf header\n"); return; } if (!IS_ELF(eh)) { printf("Not an ELF file\n"); return; } for (i = 0; i < eh.e_phnum; i++) { fs_off = eh.e_phoff + i * eh.e_phentsize; if (fsread(ino, &ph, sizeof(ph)) != sizeof(ph)) { printf("Can't read program header %d\n", i); return; } if (ph.p_type != PT_LOAD) continue; fs_off = ph.p_offset; p = (caddr_t)ph.p_vaddr; ofw_claim(p,(ph.p_filesz > ph.p_memsz) ? ph.p_filesz : ph.p_memsz,0); if (fsread(ino, p, ph.p_filesz) != ph.p_filesz) { printf("Can't read content of section %d\n", i); return; } if (ph.p_filesz != ph.p_memsz) bzero(p + ph.p_filesz, ph.p_memsz - ph.p_filesz); __syncicache(p, ph.p_memsz); } ofw_close(bootdev); (*(void (*)(void *, int, ofwfp_t, char *, int))eh.e_entry)(NULL, 0, ofw,NULL,0); } static int dskread(void *buf, u_int64_t lba, int nblk) { /* * The Open Firmware should open the correct partition for us. * That means, if we read from offset zero on an open instance handle, * we should read from offset zero of that partition. */ ofw_seek(bootdev, lba * DEV_BSIZE); ofw_read(bootdev, buf, nblk * DEV_BSIZE); return (0); } static void panic(const char *fmt, ...) { char buf[128]; va_list ap; va_start(ap, fmt); vsnprintf(buf, sizeof buf, fmt, ap); printf("panic: %s\n", buf); va_end(ap); exit(1); } static int printf(const char *fmt, ...) { va_list ap; int ret; va_start(ap, fmt); ret = vprintf(fmt, ap); va_end(ap); return (ret); } static int putchar(char c, void *arg) { char buf; if (c == '\n') { buf = '\r'; ofw_write(stdouth, &buf, 1); } buf = c; ofw_write(stdouth, &buf, 1); return (1); } static int vprintf(const char *fmt, va_list ap) { int ret; ret = __printf(fmt, putchar, 0, ap); return (ret); } static int vsnprintf(char *str, size_t sz, const char *fmt, va_list ap) { struct sp_data sp; int ret; sp.sp_buf = str; sp.sp_len = 0; sp.sp_size = sz; ret = __printf(fmt, __sputc, &sp, ap); return (ret); } static int __printf(const char *fmt, putc_func_t *putc, void *arg, va_list ap) { char buf[(sizeof(long) * 8) + 1]; char *nbuf; u_long ul; u_int ui; int lflag; int sflag; char *s; int pad; int ret; int c; nbuf = &buf[sizeof buf - 1]; ret = 0; while ((c = *fmt++) != 0) { if (c != '%') { ret += putc(c, arg); continue; } lflag = 0; sflag = 0; pad = 0; reswitch: c = *fmt++; switch (c) { case '#': sflag = 1; goto reswitch; case '%': ret += putc('%', arg); break; case 'c': c = va_arg(ap, int); ret += putc(c, arg); break; case 'd': if (lflag == 0) { ui = (u_int)va_arg(ap, int); if (ui < (int)ui) { ui = -ui; ret += putc('-', arg); } s = __uitoa(nbuf, ui, 10); } else { ul = (u_long)va_arg(ap, long); if (ul < (long)ul) { ul = -ul; ret += putc('-', arg); } s = __ultoa(nbuf, ul, 10); } ret += __puts(s, putc, arg); break; case 'l': lflag = 1; goto reswitch; case 'o': if (lflag == 0) { ui = (u_int)va_arg(ap, u_int); s = __uitoa(nbuf, ui, 8); } else { ul = (u_long)va_arg(ap, u_long); s = __ultoa(nbuf, ul, 8); } ret += __puts(s, putc, arg); break; case 'p': ul = (u_long)va_arg(ap, void *); s = __ultoa(nbuf, ul, 16); ret += __puts("0x", putc, arg); ret += __puts(s, putc, arg); break; case 's': s = va_arg(ap, char *); ret += __puts(s, putc, arg); break; case 'u': if (lflag == 0) { ui = va_arg(ap, u_int); s = __uitoa(nbuf, ui, 10); } else { ul = va_arg(ap, u_long); s = __ultoa(nbuf, ul, 10); } ret += __puts(s, putc, arg); break; case 'x': if (lflag == 0) { ui = va_arg(ap, u_int); s = __uitoa(nbuf, ui, 16); } else { ul = va_arg(ap, u_long); s = __ultoa(nbuf, ul, 16); } if (sflag) ret += __puts("0x", putc, arg); ret += __puts(s, putc, arg); break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': pad = pad * 10 + c - '0'; goto reswitch; default: break; } } return (ret); } static int __sputc(char c, void *arg) { struct sp_data *sp; sp = arg; if (sp->sp_len < sp->sp_size) sp->sp_buf[sp->sp_len++] = c; sp->sp_buf[sp->sp_len] = '\0'; return (1); } static int __puts(const char *s, putc_func_t *putc, void *arg) { const char *p; int ret; ret = 0; for (p = s; *p != '\0'; p++) ret += putc(*p, arg); return (ret); } static char * __uitoa(char *buf, u_int ui, int base) { char *p; p = buf; *p = '\0'; do *--p = digits[ui % base]; while ((ui /= base) != 0); return (p); } static char * __ultoa(char *buf, u_long ul, int base) { char *p; p = buf; *p = '\0'; do *--p = digits[ul % base]; while ((ul /= base) != 0); return (p); } Index: projects/clang390-import/sys/dev/cxgbe/t4_iov.c =================================================================== --- projects/clang390-import/sys/dev/cxgbe/t4_iov.c (revision 305080) +++ projects/clang390-import/sys/dev/cxgbe/t4_iov.c (revision 305081) @@ -1,293 +1,298 @@ /*- * Copyright (c) 2015-2016 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #ifdef PCI_IOV #include #include #include #endif #include "t4_if.h" struct t4iov_softc { device_t sc_dev; device_t sc_main; bool sc_attached; }; struct { uint16_t device; char *desc; } t4iov_pciids[] = { {0x4000, "Chelsio T440-dbg"}, {0x4001, "Chelsio T420-CR"}, {0x4002, "Chelsio T422-CR"}, {0x4003, "Chelsio T440-CR"}, {0x4004, "Chelsio T420-BCH"}, {0x4005, "Chelsio T440-BCH"}, {0x4006, "Chelsio T440-CH"}, {0x4007, "Chelsio T420-SO"}, {0x4008, "Chelsio T420-CX"}, {0x4009, "Chelsio T420-BT"}, {0x400a, "Chelsio T404-BT"}, {0x400e, "Chelsio T440-LP-CR"}, }, t5iov_pciids[] = { {0x5000, "Chelsio T580-dbg"}, {0x5001, "Chelsio T520-CR"}, /* 2 x 10G */ {0x5002, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ {0x5003, "Chelsio T540-CR"}, /* 4 x 10G */ {0x5007, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ {0x5009, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ {0x500a, "Chelsio T504-BT"}, /* 4 x 1G */ {0x500d, "Chelsio T580-CR"}, /* 2 x 40G */ {0x500e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ {0x5010, "Chelsio T580-LP-CR"}, /* 2 x 40G */ {0x5011, "Chelsio T520-LL-CR"}, /* 2 x 10G */ {0x5012, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ {0x5014, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ {0x5015, "Chelsio T502-BT"}, /* 2 x 1G */ #ifdef notyet {0x5004, "Chelsio T520-BCH"}, {0x5005, "Chelsio T540-BCH"}, {0x5006, "Chelsio T540-CH"}, {0x5008, "Chelsio T520-CX"}, {0x500b, "Chelsio B520-SR"}, {0x500c, "Chelsio B504-BT"}, {0x500f, "Chelsio Amsterdam"}, {0x5013, "Chelsio T580-CHR"}, #endif }; static int t4iov_attach_child(device_t dev); static int t4iov_probe(device_t dev) { uint16_t d; size_t i; d = pci_get_device(dev); for (i = 0; i < nitems(t4iov_pciids); i++) { if (d == t4iov_pciids[i].device) { device_set_desc(dev, t4iov_pciids[i].desc); device_quiet(dev); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t5iov_probe(device_t dev) { uint16_t d; size_t i; d = pci_get_device(dev); for (i = 0; i < nitems(t5iov_pciids); i++) { if (d == t5iov_pciids[i].device) { device_set_desc(dev, t5iov_pciids[i].desc); device_quiet(dev); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t4iov_attach(device_t dev) { struct t4iov_softc *sc; sc = device_get_softc(dev); sc->sc_dev = dev; sc->sc_main = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), 4); if (T4_IS_MAIN_READY(sc->sc_main) == 0) return (t4iov_attach_child(dev)); return (0); } static int t4iov_attach_child(device_t dev) { struct t4iov_softc *sc; #ifdef PCI_IOV nvlist_t *pf_schema, *vf_schema; #endif device_t pdev; int error; sc = device_get_softc(dev); MPASS(!sc->sc_attached); /* * PF0-3 are associated with a specific port on the NIC (PF0 * with port 0, etc.). Ask the PF4 driver for the device for * this function's associated port to determine if the port is * present. */ error = T4_READ_PORT_DEVICE(sc->sc_main, pci_get_function(dev), &pdev); if (error) return (0); #ifdef PCI_IOV pf_schema = pci_iov_schema_alloc_node(); vf_schema = pci_iov_schema_alloc_node(); error = pci_iov_attach_name(dev, pf_schema, vf_schema, "%s", device_get_nameunit(pdev)); if (error) { device_printf(dev, "Failed to initialize SR-IOV: %d\n", error); return (0); } #endif sc->sc_attached = true; return (0); } static int t4iov_detach_child(device_t dev) { struct t4iov_softc *sc; #ifdef PCI_IOV int error; #endif sc = device_get_softc(dev); if (!sc->sc_attached) return (0); #ifdef PCI_IOV error = pci_iov_detach(dev); if (error != 0) { device_printf(dev, "Failed to disable SR-IOV\n"); return (error); } #endif sc->sc_attached = false; return (0); } static int t4iov_detach(device_t dev) { struct t4iov_softc *sc; + int error; sc = device_get_softc(dev); - if (sc->sc_attached) - return (t4iov_detach_child(dev)); + if (sc->sc_attached) { + error = t4iov_detach_child(dev); + if (error) + return (error); + } + device_verbose(dev); return (0); } #ifdef PCI_IOV static int t4iov_iov_init(device_t dev, uint16_t num_vfs, const struct nvlist *config) { /* XXX: The Linux driver sets up a vf_monitor task on T4 adapters. */ return (0); } static void t4iov_iov_uninit(device_t dev) { } static int t4iov_add_vf(device_t dev, uint16_t vfnum, const struct nvlist *config) { return (0); } #endif static device_method_t t4iov_methods[] = { DEVMETHOD(device_probe, t4iov_probe), DEVMETHOD(device_attach, t4iov_attach), DEVMETHOD(device_detach, t4iov_detach), #ifdef PCI_IOV DEVMETHOD(pci_iov_init, t4iov_iov_init), DEVMETHOD(pci_iov_uninit, t4iov_iov_uninit), DEVMETHOD(pci_iov_add_vf, t4iov_add_vf), #endif DEVMETHOD(t4_attach_child, t4iov_attach_child), DEVMETHOD(t4_detach_child, t4iov_detach_child), DEVMETHOD_END }; static driver_t t4iov_driver = { "t4iov", t4iov_methods, sizeof(struct t4iov_softc) }; static device_method_t t5iov_methods[] = { DEVMETHOD(device_probe, t5iov_probe), DEVMETHOD(device_attach, t4iov_attach), DEVMETHOD(device_detach, t4iov_detach), #ifdef PCI_IOV DEVMETHOD(pci_iov_init, t4iov_iov_init), DEVMETHOD(pci_iov_uninit, t4iov_iov_uninit), DEVMETHOD(pci_iov_add_vf, t4iov_add_vf), #endif DEVMETHOD(t4_attach_child, t4iov_attach_child), DEVMETHOD(t4_detach_child, t4iov_detach_child), DEVMETHOD_END }; static driver_t t5iov_driver = { "t5iov", t5iov_methods, sizeof(struct t4iov_softc) }; static devclass_t t4iov_devclass, t5iov_devclass; DRIVER_MODULE(t4iov, pci, t4iov_driver, t4iov_devclass, 0, 0); MODULE_VERSION(t4iov, 1); DRIVER_MODULE(t5iov, pci, t5iov_driver, t5iov_devclass, 0, 0); MODULE_VERSION(t5iov, 1); Index: projects/clang390-import/sys/dev/hyperv/netvsc/hv_net_vsc.c =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/hv_net_vsc.c (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/hv_net_vsc.c (revision 305081) @@ -1,900 +1,898 @@ /*- * Copyright (c) 2009-2012,2016 Microsoft Corp. * Copyright (c) 2010-2012 Citrix Inc. * Copyright (c) 2012 NetApp Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /** * HyperV vmbus network VSC (virtual services client) module * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE(M_NETVSC, "netvsc", "Hyper-V netvsc driver"); /* * Forward declarations */ static void hv_nv_on_channel_callback(struct vmbus_channel *chan, void *xrxr); static int hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc); static int hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *, int); static int hv_nv_destroy_send_buffer(struct hn_softc *sc); static int hv_nv_destroy_rx_buffer(struct hn_softc *sc); static int hv_nv_connect_to_vsp(struct hn_softc *sc); static void hv_nv_on_send_completion(struct hn_softc *sc, struct vmbus_channel *, const struct vmbus_chanpkt_hdr *pkt); static void hv_nv_on_receive_completion(struct vmbus_channel *chan, uint64_t tid); static void hv_nv_on_receive(struct hn_softc *sc, struct hn_rx_ring *rxr, struct vmbus_channel *chan, const struct vmbus_chanpkt_hdr *pkt); static void hn_nvs_sent_none(struct hn_send_ctx *sndc, struct hn_softc *, struct vmbus_channel *chan, const void *, int); -static void hn_nvs_sent_xact(struct hn_send_ctx *, struct hn_softc *sc, - struct vmbus_channel *, const void *, int); struct hn_send_ctx hn_send_ctx_none = HN_SEND_CTX_INITIALIZER(hn_nvs_sent_none, NULL); uint32_t hn_chim_alloc(struct hn_softc *sc) { int i, bmap_cnt = sc->hn_chim_bmap_cnt; u_long *bmap = sc->hn_chim_bmap; uint32_t ret = HN_NVS_CHIM_IDX_INVALID; for (i = 0; i < bmap_cnt; ++i) { int idx; idx = ffsl(~bmap[i]); if (idx == 0) continue; --idx; /* ffsl is 1-based */ KASSERT(i * LONG_BIT + idx < sc->hn_chim_cnt, ("invalid i %d and idx %d", i, idx)); if (atomic_testandset_long(&bmap[i], idx)) continue; ret = i * LONG_BIT + idx; break; } return (ret); } const void * hn_nvs_xact_execute(struct hn_softc *sc, struct vmbus_xact *xact, void *req, int reqlen, size_t *resp_len) { struct hn_send_ctx sndc; int error; hn_send_ctx_init_simple(&sndc, hn_nvs_sent_xact, xact); vmbus_xact_activate(xact); error = hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_RC, req, reqlen, &sndc); if (error) { vmbus_xact_deactivate(xact); return NULL; } return (vmbus_xact_wait(xact, resp_len)); } static __inline int hn_nvs_req_send(struct hn_softc *sc, void *req, int reqlen) { return (hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_NONE, req, reqlen, &hn_send_ctx_none)); } /* * Net VSC initialize receive buffer with net VSP * * Net VSP: Network virtual services client, also known as the * Hyper-V extensible switch and the synthetic data path. */ static int hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc, int rxbuf_size) { struct vmbus_xact *xact = NULL; struct hn_nvs_rxbuf_conn *conn; const struct hn_nvs_rxbuf_connresp *resp; size_t resp_len; uint32_t status; int error; KASSERT(rxbuf_size <= NETVSC_RECEIVE_BUFFER_SIZE, ("invalid rxbuf size %d", rxbuf_size)); /* * Connect the RXBUF GPADL to the primary channel. * * NOTE: * Only primary channel has RXBUF connected to it. Sub-channels * just share this RXBUF. */ error = vmbus_chan_gpadl_connect(sc->hn_prichan, sc->hn_rxbuf_dma.hv_paddr, rxbuf_size, &sc->hn_rxbuf_gpadl); if (error) { if_printf(sc->hn_ifp, "rxbuf gpadl connect failed: %d\n", error); goto cleanup; } /* * Connect RXBUF to NVS. */ xact = vmbus_xact_get(sc->hn_xact, sizeof(*conn)); if (xact == NULL) { if_printf(sc->hn_ifp, "no xact for nvs rxbuf conn\n"); error = ENXIO; goto cleanup; } conn = vmbus_xact_req_data(xact); conn->nvs_type = HN_NVS_TYPE_RXBUF_CONN; conn->nvs_gpadl = sc->hn_rxbuf_gpadl; conn->nvs_sig = HN_NVS_RXBUF_SIG; resp = hn_nvs_xact_execute(sc, xact, conn, sizeof(*conn), &resp_len); if (resp == NULL) { if_printf(sc->hn_ifp, "exec rxbuf conn failed\n"); error = EIO; goto cleanup; } if (resp_len < sizeof(*resp)) { if_printf(sc->hn_ifp, "invalid rxbuf conn resp length %zu\n", resp_len); error = EINVAL; goto cleanup; } if (resp->nvs_type != HN_NVS_TYPE_RXBUF_CONNRESP) { if_printf(sc->hn_ifp, "not rxbuf conn resp, type %u\n", resp->nvs_type); error = EINVAL; goto cleanup; } status = resp->nvs_status; vmbus_xact_put(xact); xact = NULL; if (status != HN_NVS_STATUS_OK) { if_printf(sc->hn_ifp, "rxbuf conn failed: %x\n", status); error = EIO; goto cleanup; } sc->hn_flags |= HN_FLAG_RXBUF_CONNECTED; return (0); cleanup: if (xact != NULL) vmbus_xact_put(xact); hv_nv_destroy_rx_buffer(sc); return (error); } /* * Net VSC initialize send buffer with net VSP */ static int hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc) { struct vmbus_xact *xact = NULL; struct hn_nvs_chim_conn *chim; const struct hn_nvs_chim_connresp *resp; size_t resp_len; uint32_t status, sectsz; int error; /* * Connect chimney sending buffer GPADL to the primary channel. * * NOTE: * Only primary channel has chimney sending buffer connected to it. * Sub-channels just share this chimney sending buffer. */ error = vmbus_chan_gpadl_connect(sc->hn_prichan, sc->hn_chim_dma.hv_paddr, NETVSC_SEND_BUFFER_SIZE, &sc->hn_chim_gpadl); if (error) { if_printf(sc->hn_ifp, "chimney sending buffer gpadl " "connect failed: %d\n", error); goto cleanup; } /* * Connect chimney sending buffer to NVS */ xact = vmbus_xact_get(sc->hn_xact, sizeof(*chim)); if (xact == NULL) { if_printf(sc->hn_ifp, "no xact for nvs chim conn\n"); error = ENXIO; goto cleanup; } chim = vmbus_xact_req_data(xact); chim->nvs_type = HN_NVS_TYPE_CHIM_CONN; chim->nvs_gpadl = sc->hn_chim_gpadl; chim->nvs_sig = HN_NVS_CHIM_SIG; resp = hn_nvs_xact_execute(sc, xact, chim, sizeof(*chim), &resp_len); if (resp == NULL) { if_printf(sc->hn_ifp, "exec chim conn failed\n"); error = EIO; goto cleanup; } if (resp_len < sizeof(*resp)) { if_printf(sc->hn_ifp, "invalid chim conn resp length %zu\n", resp_len); error = EINVAL; goto cleanup; } if (resp->nvs_type != HN_NVS_TYPE_CHIM_CONNRESP) { if_printf(sc->hn_ifp, "not chim conn resp, type %u\n", resp->nvs_type); error = EINVAL; goto cleanup; } status = resp->nvs_status; sectsz = resp->nvs_sectsz; vmbus_xact_put(xact); xact = NULL; if (status != HN_NVS_STATUS_OK) { if_printf(sc->hn_ifp, "chim conn failed: %x\n", status); error = EIO; goto cleanup; } if (sectsz == 0) { if_printf(sc->hn_ifp, "zero chimney sending buffer " "section size\n"); return 0; } sc->hn_chim_szmax = sectsz; sc->hn_chim_cnt = NETVSC_SEND_BUFFER_SIZE / sc->hn_chim_szmax; if (NETVSC_SEND_BUFFER_SIZE % sc->hn_chim_szmax != 0) { if_printf(sc->hn_ifp, "chimney sending sections are " "not properly aligned\n"); } if (sc->hn_chim_cnt % LONG_BIT != 0) { if_printf(sc->hn_ifp, "discard %d chimney sending sections\n", sc->hn_chim_cnt % LONG_BIT); } sc->hn_chim_bmap_cnt = sc->hn_chim_cnt / LONG_BIT; sc->hn_chim_bmap = malloc(sc->hn_chim_bmap_cnt * sizeof(u_long), M_NETVSC, M_WAITOK | M_ZERO); /* Done! */ sc->hn_flags |= HN_FLAG_CHIM_CONNECTED; if (bootverbose) { if_printf(sc->hn_ifp, "chimney sending buffer %d/%d\n", sc->hn_chim_szmax, sc->hn_chim_cnt); } return 0; cleanup: if (xact != NULL) vmbus_xact_put(xact); hv_nv_destroy_send_buffer(sc); return (error); } /* * Net VSC destroy receive buffer */ static int hv_nv_destroy_rx_buffer(struct hn_softc *sc) { int ret = 0; if (sc->hn_flags & HN_FLAG_RXBUF_CONNECTED) { struct hn_nvs_rxbuf_disconn disconn; /* * Disconnect RXBUF from NVS. */ memset(&disconn, 0, sizeof(disconn)); disconn.nvs_type = HN_NVS_TYPE_RXBUF_DISCONN; disconn.nvs_sig = HN_NVS_RXBUF_SIG; /* NOTE: No response. */ ret = hn_nvs_req_send(sc, &disconn, sizeof(disconn)); if (ret != 0) { if_printf(sc->hn_ifp, "send rxbuf disconn failed: %d\n", ret); return (ret); } sc->hn_flags &= ~HN_FLAG_RXBUF_CONNECTED; } if (sc->hn_rxbuf_gpadl != 0) { /* * Disconnect RXBUF from primary channel. */ ret = vmbus_chan_gpadl_disconnect(sc->hn_prichan, sc->hn_rxbuf_gpadl); if (ret != 0) { if_printf(sc->hn_ifp, "rxbuf disconn failed: %d\n", ret); return (ret); } sc->hn_rxbuf_gpadl = 0; } return (ret); } /* * Net VSC destroy send buffer */ static int hv_nv_destroy_send_buffer(struct hn_softc *sc) { int ret = 0; if (sc->hn_flags & HN_FLAG_CHIM_CONNECTED) { struct hn_nvs_chim_disconn disconn; /* * Disconnect chimney sending buffer from NVS. */ memset(&disconn, 0, sizeof(disconn)); disconn.nvs_type = HN_NVS_TYPE_CHIM_DISCONN; disconn.nvs_sig = HN_NVS_CHIM_SIG; /* NOTE: No response. */ ret = hn_nvs_req_send(sc, &disconn, sizeof(disconn)); if (ret != 0) { if_printf(sc->hn_ifp, "send chim disconn failed: %d\n", ret); return (ret); } sc->hn_flags &= ~HN_FLAG_CHIM_CONNECTED; } if (sc->hn_chim_gpadl != 0) { /* * Disconnect chimney sending buffer from primary channel. */ ret = vmbus_chan_gpadl_disconnect(sc->hn_prichan, sc->hn_chim_gpadl); if (ret != 0) { if_printf(sc->hn_ifp, "chim disconn failed: %d\n", ret); return (ret); } sc->hn_chim_gpadl = 0; } if (sc->hn_chim_bmap != NULL) { free(sc->hn_chim_bmap, M_NETVSC); sc->hn_chim_bmap = NULL; } return (ret); } static int hv_nv_negotiate_nvsp_protocol(struct hn_softc *sc, uint32_t nvs_ver) { struct vmbus_xact *xact; struct hn_nvs_init *init; const struct hn_nvs_init_resp *resp; size_t resp_len; uint32_t status; xact = vmbus_xact_get(sc->hn_xact, sizeof(*init)); if (xact == NULL) { if_printf(sc->hn_ifp, "no xact for nvs init\n"); return (ENXIO); } init = vmbus_xact_req_data(xact); init->nvs_type = HN_NVS_TYPE_INIT; init->nvs_ver_min = nvs_ver; init->nvs_ver_max = nvs_ver; resp = hn_nvs_xact_execute(sc, xact, init, sizeof(*init), &resp_len); if (resp == NULL) { if_printf(sc->hn_ifp, "exec init failed\n"); vmbus_xact_put(xact); return (EIO); } if (resp_len < sizeof(*resp)) { if_printf(sc->hn_ifp, "invalid init resp length %zu\n", resp_len); vmbus_xact_put(xact); return (EINVAL); } if (resp->nvs_type != HN_NVS_TYPE_INIT_RESP) { if_printf(sc->hn_ifp, "not init resp, type %u\n", resp->nvs_type); vmbus_xact_put(xact); return (EINVAL); } status = resp->nvs_status; vmbus_xact_put(xact); if (status != HN_NVS_STATUS_OK) { if_printf(sc->hn_ifp, "nvs init failed for ver 0x%x\n", nvs_ver); return (EINVAL); } return (0); } /* * Send NDIS version 2 config packet containing MTU. * * Not valid for NDIS version 1. */ static int hv_nv_send_ndis_config(struct hn_softc *sc, uint32_t mtu) { struct hn_nvs_ndis_conf conf; int error; memset(&conf, 0, sizeof(conf)); conf.nvs_type = HN_NVS_TYPE_NDIS_CONF; conf.nvs_mtu = mtu; conf.nvs_caps = HN_NVS_NDIS_CONF_VLAN; /* NOTE: No response. */ error = hn_nvs_req_send(sc, &conf, sizeof(conf)); if (error) if_printf(sc->hn_ifp, "send nvs ndis conf failed: %d\n", error); return (error); } /* * Net VSC connect to VSP */ static int hv_nv_connect_to_vsp(struct hn_softc *sc) { uint32_t protocol_list[] = { NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 }; int i; int protocol_number = nitems(protocol_list); int ret = 0; device_t dev = sc->hn_dev; struct ifnet *ifp = sc->hn_ifp; struct hn_nvs_ndis_init ndis; int rxbuf_size; /* * Negotiate the NVSP version. Try the latest NVSP first. */ for (i = protocol_number - 1; i >= 0; i--) { if (hv_nv_negotiate_nvsp_protocol(sc, protocol_list[i]) == 0) { sc->hn_nvs_ver = protocol_list[i]; sc->hn_ndis_ver = NDIS_VERSION_6_30; if (sc->hn_nvs_ver <= NVSP_PROTOCOL_VERSION_4) sc->hn_ndis_ver = NDIS_VERSION_6_1; if (bootverbose) { if_printf(sc->hn_ifp, "NVS version 0x%x, " "NDIS version %u.%u\n", sc->hn_nvs_ver, NDIS_VERSION_MAJOR(sc->hn_ndis_ver), NDIS_VERSION_MINOR(sc->hn_ndis_ver)); } break; } } if (i < 0) { if (bootverbose) device_printf(dev, "failed to negotiate a valid " "protocol.\n"); return (EPROTO); } /* * Set the MTU if supported by this NVSP protocol version * This needs to be right after the NVSP init message per Haiyang */ if (sc->hn_nvs_ver >= NVSP_PROTOCOL_VERSION_2) ret = hv_nv_send_ndis_config(sc, ifp->if_mtu); /* * Initialize NDIS. */ memset(&ndis, 0, sizeof(ndis)); ndis.nvs_type = HN_NVS_TYPE_NDIS_INIT; ndis.nvs_ndis_major = NDIS_VERSION_MAJOR(sc->hn_ndis_ver); ndis.nvs_ndis_minor = NDIS_VERSION_MINOR(sc->hn_ndis_ver); /* NOTE: No response. */ ret = hn_nvs_req_send(sc, &ndis, sizeof(ndis)); if (ret != 0) { if_printf(sc->hn_ifp, "send nvs ndis init failed: %d\n", ret); goto cleanup; } /* Post the big receive buffer to NetVSP */ if (sc->hn_nvs_ver <= NVSP_PROTOCOL_VERSION_2) rxbuf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY; else rxbuf_size = NETVSC_RECEIVE_BUFFER_SIZE; ret = hv_nv_init_rx_buffer_with_net_vsp(sc, rxbuf_size); if (ret == 0) ret = hv_nv_init_send_buffer_with_net_vsp(sc); cleanup: return (ret); } /* * Net VSC disconnect from VSP */ static void hv_nv_disconnect_from_vsp(struct hn_softc *sc) { hv_nv_destroy_rx_buffer(sc); hv_nv_destroy_send_buffer(sc); } void hv_nv_subchan_attach(struct vmbus_channel *chan, struct hn_rx_ring *rxr) { KASSERT(rxr->hn_rx_idx == vmbus_chan_subidx(chan), ("chan%u subidx %u, rxr%d mismatch", vmbus_chan_id(chan), vmbus_chan_subidx(chan), rxr->hn_rx_idx)); vmbus_chan_open(chan, NETVSC_DEVICE_RING_BUFFER_SIZE, NETVSC_DEVICE_RING_BUFFER_SIZE, NULL, 0, hv_nv_on_channel_callback, rxr); } /* * Net VSC on device add * * Callback when the device belonging to this driver is added */ int hv_nv_on_device_add(struct hn_softc *sc, struct hn_rx_ring *rxr) { struct vmbus_channel *chan = sc->hn_prichan; int ret = 0; /* * Open the channel */ KASSERT(rxr->hn_rx_idx == vmbus_chan_subidx(chan), ("chan%u subidx %u, rxr%d mismatch", vmbus_chan_id(chan), vmbus_chan_subidx(chan), rxr->hn_rx_idx)); ret = vmbus_chan_open(chan, NETVSC_DEVICE_RING_BUFFER_SIZE, NETVSC_DEVICE_RING_BUFFER_SIZE, NULL, 0, hv_nv_on_channel_callback, rxr); if (ret != 0) goto cleanup; /* * Connect with the NetVsp */ ret = hv_nv_connect_to_vsp(sc); if (ret != 0) goto close; return (0); close: /* Now, we can close the channel safely */ vmbus_chan_close(chan); cleanup: return (ret); } /* * Net VSC on device remove */ int -hv_nv_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel) +hv_nv_on_device_remove(struct hn_softc *sc) { hv_nv_disconnect_from_vsp(sc); /* Now, we can close the channel safely */ vmbus_chan_close(sc->hn_prichan); return (0); } -static void +void hn_nvs_sent_xact(struct hn_send_ctx *sndc, struct hn_softc *sc __unused, struct vmbus_channel *chan __unused, const void *data, int dlen) { vmbus_xact_wakeup(sndc->hn_cbarg, data, dlen); } static void hn_nvs_sent_none(struct hn_send_ctx *sndc __unused, struct hn_softc *sc __unused, struct vmbus_channel *chan __unused, const void *data __unused, int dlen __unused) { /* EMPTY */ } void hn_chim_free(struct hn_softc *sc, uint32_t chim_idx) { u_long mask; uint32_t idx; idx = chim_idx / LONG_BIT; KASSERT(idx < sc->hn_chim_bmap_cnt, ("invalid chimney index 0x%x", chim_idx)); mask = 1UL << (chim_idx % LONG_BIT); KASSERT(sc->hn_chim_bmap[idx] & mask, ("index bitmap 0x%lx, chimney index %u, " "bitmap idx %d, bitmask 0x%lx", sc->hn_chim_bmap[idx], chim_idx, idx, mask)); atomic_clear_long(&sc->hn_chim_bmap[idx], mask); } /* * Net VSC on send completion */ static void hv_nv_on_send_completion(struct hn_softc *sc, struct vmbus_channel *chan, const struct vmbus_chanpkt_hdr *pkt) { struct hn_send_ctx *sndc; sndc = (struct hn_send_ctx *)(uintptr_t)pkt->cph_xactid; sndc->hn_cb(sndc, sc, chan, VMBUS_CHANPKT_CONST_DATA(pkt), VMBUS_CHANPKT_DATALEN(pkt)); /* * NOTE: * 'sndc' CAN NOT be accessed anymore, since it can be freed by * its callback. */ } /* * Net VSC on send * Sends a packet on the specified Hyper-V device. * Returns 0 on success, non-zero on failure. */ int hv_nv_on_send(struct vmbus_channel *chan, uint32_t rndis_mtype, struct hn_send_ctx *sndc, struct vmbus_gpa *gpa, int gpa_cnt) { struct hn_nvs_rndis rndis; int ret; rndis.nvs_type = HN_NVS_TYPE_RNDIS; rndis.nvs_rndis_mtype = rndis_mtype; rndis.nvs_chim_idx = sndc->hn_chim_idx; rndis.nvs_chim_sz = sndc->hn_chim_sz; if (gpa_cnt) { ret = hn_nvs_send_sglist(chan, gpa, gpa_cnt, &rndis, sizeof(rndis), sndc); } else { ret = hn_nvs_send(chan, VMBUS_CHANPKT_FLAG_RC, &rndis, sizeof(rndis), sndc); } return (ret); } /* * Net VSC on receive * * In the FreeBSD Hyper-V virtual world, this function deals exclusively * with virtual addresses. */ static void hv_nv_on_receive(struct hn_softc *sc, struct hn_rx_ring *rxr, struct vmbus_channel *chan, const struct vmbus_chanpkt_hdr *pkthdr) { const struct vmbus_chanpkt_rxbuf *pkt; const struct hn_nvs_hdr *nvs_hdr; int count = 0; int i = 0; /* Make sure that this is a RNDIS message. */ nvs_hdr = VMBUS_CHANPKT_CONST_DATA(pkthdr); if (__predict_false(nvs_hdr->nvs_type != HN_NVS_TYPE_RNDIS)) { if_printf(rxr->hn_ifp, "nvs type %u, not RNDIS\n", nvs_hdr->nvs_type); return; } pkt = (const struct vmbus_chanpkt_rxbuf *)pkthdr; if (pkt->cp_rxbuf_id != NETVSC_RECEIVE_BUFFER_ID) { if_printf(rxr->hn_ifp, "rxbuf_id %d is invalid!\n", pkt->cp_rxbuf_id); return; } count = pkt->cp_rxbuf_cnt; /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */ for (i = 0; i < count; i++) { hv_rf_on_receive(sc, rxr, rxr->hn_rxbuf + pkt->cp_rxbuf[i].rb_ofs, pkt->cp_rxbuf[i].rb_len); } /* * Moved completion call back here so that all received * messages (not just data messages) will trigger a response * message back to the host. */ hv_nv_on_receive_completion(chan, pkt->cp_hdr.cph_xactid); } /* * Net VSC on receive completion * * Send a receive completion packet to RNDIS device (ie NetVsp) */ static void hv_nv_on_receive_completion(struct vmbus_channel *chan, uint64_t tid) { struct hn_nvs_rndis_ack ack; int retries = 0; int ret = 0; ack.nvs_type = HN_NVS_TYPE_RNDIS_ACK; ack.nvs_status = HN_NVS_STATUS_OK; retry_send_cmplt: /* Send the completion */ ret = vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP, VMBUS_CHANPKT_FLAG_NONE, &ack, sizeof(ack), tid); if (ret == 0) { /* success */ /* no-op */ } else if (ret == EAGAIN) { /* no more room... wait a bit and attempt to retry 3 times */ retries++; if (retries < 4) { DELAY(100); goto retry_send_cmplt; } } } static void hn_proc_notify(struct hn_softc *sc, const struct vmbus_chanpkt_hdr *pkt) { const struct hn_nvs_hdr *hdr; hdr = VMBUS_CHANPKT_CONST_DATA(pkt); if (hdr->nvs_type == HN_NVS_TYPE_TXTBL_NOTE) { /* Useless; ignore */ return; } if_printf(sc->hn_ifp, "got notify, nvs type %u\n", hdr->nvs_type); } /* * Net VSC on channel callback */ static void hv_nv_on_channel_callback(struct vmbus_channel *chan, void *xrxr) { struct hn_rx_ring *rxr = xrxr; struct hn_softc *sc = rxr->hn_ifp->if_softc; void *buffer; int bufferlen = NETVSC_PACKET_SIZE; buffer = rxr->hn_rdbuf; do { struct vmbus_chanpkt_hdr *pkt = buffer; uint32_t bytes_rxed; int ret; bytes_rxed = bufferlen; ret = vmbus_chan_recv_pkt(chan, pkt, &bytes_rxed); if (ret == 0) { if (bytes_rxed > 0) { switch (pkt->cph_type) { case VMBUS_CHANPKT_TYPE_COMP: hv_nv_on_send_completion(sc, chan, pkt); break; case VMBUS_CHANPKT_TYPE_RXBUF: hv_nv_on_receive(sc, rxr, chan, pkt); break; case VMBUS_CHANPKT_TYPE_INBAND: hn_proc_notify(sc, pkt); break; default: if_printf(rxr->hn_ifp, "unknown chan pkt %u\n", pkt->cph_type); break; } } } else if (ret == ENOBUFS) { /* Handle large packet */ if (bufferlen > NETVSC_PACKET_SIZE) { free(buffer, M_NETVSC); buffer = NULL; } /* alloc new buffer */ buffer = malloc(bytes_rxed, M_NETVSC, M_NOWAIT); if (buffer == NULL) { if_printf(rxr->hn_ifp, "hv_cb malloc buffer failed, len=%u\n", bytes_rxed); bufferlen = 0; break; } bufferlen = bytes_rxed; } else { /* No more packets */ break; } } while (1); if (bufferlen > NETVSC_PACKET_SIZE) free(buffer, M_NETVSC); hv_rf_channel_rollup(rxr, rxr->hn_txr); } Index: projects/clang390-import/sys/dev/hyperv/netvsc/hv_net_vsc.h =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/hv_net_vsc.h (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/hv_net_vsc.h (revision 305081) @@ -1,411 +1,409 @@ /*- * Copyright (c) 2009-2012,2016 Microsoft Corp. * Copyright (c) 2010-2012 Citrix Inc. * Copyright (c) 2012 NetApp Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /* * HyperV vmbus (virtual machine bus) network VSC (virtual services client) * header file * * (Updated from unencumbered NvspProtocol.h) */ #ifndef __HV_NET_VSC_H__ #define __HV_NET_VSC_H__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define HN_USE_TXDESC_BUFRING MALLOC_DECLARE(M_NETVSC); #define NVSP_INVALID_PROTOCOL_VERSION (0xFFFFFFFF) #define NVSP_PROTOCOL_VERSION_1 2 #define NVSP_PROTOCOL_VERSION_2 0x30002 #define NVSP_PROTOCOL_VERSION_4 0x40000 #define NVSP_PROTOCOL_VERSION_5 0x50000 #define NVSP_MIN_PROTOCOL_VERSION (NVSP_PROTOCOL_VERSION_1) #define NVSP_MAX_PROTOCOL_VERSION (NVSP_PROTOCOL_VERSION_2) #define NVSP_PROTOCOL_VERSION_CURRENT NVSP_PROTOCOL_VERSION_2 #define VERSION_4_OFFLOAD_SIZE 22 #define NVSP_OPERATIONAL_STATUS_OK (0x00000000) #define NVSP_OPERATIONAL_STATUS_DEGRADED (0x00000001) #define NVSP_OPERATIONAL_STATUS_NONRECOVERABLE (0x00000002) #define NVSP_OPERATIONAL_STATUS_NO_CONTACT (0x00000003) #define NVSP_OPERATIONAL_STATUS_LOST_COMMUNICATION (0x00000004) /* * Maximun number of transfer pages (packets) the VSP will use on a receive */ #define NVSP_MAX_PACKETS_PER_RECEIVE 375 /* vRSS stuff */ #define RNDIS_OBJECT_TYPE_RSS_CAPABILITIES 0x88 #define RNDIS_OBJECT_TYPE_RSS_PARAMETERS 0x89 #define RNDIS_RECEIVE_SCALE_CAPABILITIES_REVISION_2 2 #define RNDIS_RECEIVE_SCALE_PARAMETERS_REVISION_2 2 struct rndis_obj_header { uint8_t type; uint8_t rev; uint16_t size; } __packed; /* rndis_recv_scale_cap/cap_flag */ #define RNDIS_RSS_CAPS_MESSAGE_SIGNALED_INTERRUPTS 0x01000000 #define RNDIS_RSS_CAPS_CLASSIFICATION_AT_ISR 0x02000000 #define RNDIS_RSS_CAPS_CLASSIFICATION_AT_DPC 0x04000000 #define RNDIS_RSS_CAPS_USING_MSI_X 0x08000000 #define RNDIS_RSS_CAPS_RSS_AVAILABLE_ON_PORTS 0x10000000 #define RNDIS_RSS_CAPS_SUPPORTS_MSI_X 0x20000000 #define RNDIS_RSS_CAPS_HASH_TYPE_TCP_IPV4 0x00000100 #define RNDIS_RSS_CAPS_HASH_TYPE_TCP_IPV6 0x00000200 #define RNDIS_RSS_CAPS_HASH_TYPE_TCP_IPV6_EX 0x00000400 /* RNDIS_RECEIVE_SCALE_CAPABILITIES */ struct rndis_recv_scale_cap { struct rndis_obj_header hdr; uint32_t cap_flag; uint32_t num_int_msg; uint32_t num_recv_que; uint16_t num_indirect_tabent; } __packed; /* rndis_recv_scale_param flags */ #define RNDIS_RSS_PARAM_FLAG_BASE_CPU_UNCHANGED 0x0001 #define RNDIS_RSS_PARAM_FLAG_HASH_INFO_UNCHANGED 0x0002 #define RNDIS_RSS_PARAM_FLAG_ITABLE_UNCHANGED 0x0004 #define RNDIS_RSS_PARAM_FLAG_HASH_KEY_UNCHANGED 0x0008 #define RNDIS_RSS_PARAM_FLAG_DISABLE_RSS 0x0010 /* Hash info bits */ #define RNDIS_HASH_FUNC_TOEPLITZ 0x00000001 #define RNDIS_HASH_IPV4 0x00000100 #define RNDIS_HASH_TCP_IPV4 0x00000200 #define RNDIS_HASH_IPV6 0x00000400 #define RNDIS_HASH_IPV6_EX 0x00000800 #define RNDIS_HASH_TCP_IPV6 0x00001000 #define RNDIS_HASH_TCP_IPV6_EX 0x00002000 #define RNDIS_RSS_INDIRECTION_TABLE_MAX_SIZE_REVISION_2 (128 * 4) #define RNDIS_RSS_HASH_SECRET_KEY_MAX_SIZE_REVISION_2 40 #define ITAB_NUM 128 #define HASH_KEYLEN RNDIS_RSS_HASH_SECRET_KEY_MAX_SIZE_REVISION_2 /* RNDIS_RECEIVE_SCALE_PARAMETERS */ typedef struct rndis_recv_scale_param_ { struct rndis_obj_header hdr; /* Qualifies the rest of the information */ uint16_t flag; /* The base CPU number to do receive processing. not used */ uint16_t base_cpu_number; /* This describes the hash function and type being enabled */ uint32_t hashinfo; /* The size of indirection table array */ uint16_t indirect_tabsize; /* The offset of the indirection table from the beginning of this * structure */ uint32_t indirect_taboffset; /* The size of the hash secret key */ uint16_t hashkey_size; /* The offset of the secret key from the beginning of this structure */ uint32_t hashkey_offset; uint32_t processor_masks_offset; uint32_t num_processor_masks; uint32_t processor_masks_entry_size; } rndis_recv_scale_param; /* * The following arguably belongs in a separate header file */ /* * Defines */ #define NETVSC_SEND_BUFFER_SIZE (1024*1024*15) /* 15M */ #define NETVSC_SEND_BUFFER_ID 0xface #define NETVSC_RECEIVE_BUFFER_SIZE_LEGACY (1024*1024*15) /* 15MB */ #define NETVSC_RECEIVE_BUFFER_SIZE (1024*1024*16) /* 16MB */ #define NETVSC_RECEIVE_BUFFER_ID 0xcafe #define NETVSC_RECEIVE_SG_COUNT 1 /* Preallocated receive packets */ #define NETVSC_RECEIVE_PACKETLIST_COUNT 256 /* * Maximum MTU we permit to be configured for a netvsc interface. * When the code was developed, a max MTU of 12232 was tested and * proven to work. 9K is a reasonable maximum for an Ethernet. */ #define NETVSC_MAX_CONFIGURABLE_MTU (9 * 1024) #define NETVSC_PACKET_SIZE PAGE_SIZE #define VRSS_SEND_TABLE_SIZE 16 /* * Data types */ struct vmbus_channel; typedef void (*pfn_on_send_rx_completion)(struct vmbus_channel *, void *); #define NETVSC_DEVICE_RING_BUFFER_SIZE (128 * PAGE_SIZE) #define NETVSC_PACKET_MAXPAGE 32 #define NETVSC_VLAN_PRIO_MASK 0xe000 #define NETVSC_VLAN_PRIO_SHIFT 13 #define NETVSC_VLAN_VID_MASK 0x0fff #define TYPE_IPV4 2 #define TYPE_IPV6 4 #define TYPE_TCP 2 #define TYPE_UDP 4 #define TRANSPORT_TYPE_NOT_IP 0 #define TRANSPORT_TYPE_IPV4_TCP ((TYPE_IPV4 << 16) | TYPE_TCP) #define TRANSPORT_TYPE_IPV4_UDP ((TYPE_IPV4 << 16) | TYPE_UDP) #define TRANSPORT_TYPE_IPV6_TCP ((TYPE_IPV6 << 16) | TYPE_TCP) #define TRANSPORT_TYPE_IPV6_UDP ((TYPE_IPV6 << 16) | TYPE_UDP) typedef struct { - uint8_t mac_addr[6]; /* Assumption unsigned long */ - uint8_t link_state; + uint8_t mac_addr[ETHER_ADDR_LEN]; + uint32_t link_state; } netvsc_device_info; #define HN_XACT_REQ_PGCNT 2 #define HN_XACT_RESP_PGCNT 2 #define HN_XACT_REQ_SIZE (HN_XACT_REQ_PGCNT * PAGE_SIZE) #define HN_XACT_RESP_SIZE (HN_XACT_RESP_PGCNT * PAGE_SIZE) #ifndef HN_USE_TXDESC_BUFRING struct hn_txdesc; SLIST_HEAD(hn_txdesc_list, hn_txdesc); #else struct buf_ring; #endif struct hn_tx_ring; struct hn_rx_ring { struct ifnet *hn_ifp; struct hn_tx_ring *hn_txr; void *hn_rdbuf; uint8_t *hn_rxbuf; /* shadow sc->hn_rxbuf */ int hn_rx_idx; /* Trust csum verification on host side */ int hn_trust_hcsum; /* HN_TRUST_HCSUM_ */ struct lro_ctrl hn_lro; u_long hn_csum_ip; u_long hn_csum_tcp; u_long hn_csum_udp; u_long hn_csum_trusted; u_long hn_lro_tried; u_long hn_small_pkts; u_long hn_pkts; u_long hn_rss_pkts; /* Rarely used stuffs */ struct sysctl_oid *hn_rx_sysctl_tree; int hn_rx_flags; } __aligned(CACHE_LINE_SIZE); #define HN_TRUST_HCSUM_IP 0x0001 #define HN_TRUST_HCSUM_TCP 0x0002 #define HN_TRUST_HCSUM_UDP 0x0004 #define HN_RX_FLAG_ATTACHED 0x1 struct hn_tx_ring { #ifndef HN_USE_TXDESC_BUFRING struct mtx hn_txlist_spin; struct hn_txdesc_list hn_txlist; #else struct buf_ring *hn_txdesc_br; #endif int hn_txdesc_cnt; int hn_txdesc_avail; u_short hn_has_txeof; u_short hn_txdone_cnt; int hn_sched_tx; void (*hn_txeof)(struct hn_tx_ring *); struct taskqueue *hn_tx_taskq; struct task hn_tx_task; struct task hn_txeof_task; struct buf_ring *hn_mbuf_br; int hn_oactive; int hn_tx_idx; struct mtx hn_tx_lock; struct hn_softc *hn_sc; struct vmbus_channel *hn_chan; int hn_direct_tx_size; int hn_chim_size; bus_dma_tag_t hn_tx_data_dtag; uint64_t hn_csum_assist; int hn_gpa_cnt; struct vmbus_gpa hn_gpa[NETVSC_PACKET_MAXPAGE]; u_long hn_no_txdescs; u_long hn_send_failed; u_long hn_txdma_failed; u_long hn_tx_collapsed; u_long hn_tx_chimney_tried; u_long hn_tx_chimney; u_long hn_pkts; /* Rarely used stuffs */ struct hn_txdesc *hn_txdesc; bus_dma_tag_t hn_tx_rndis_dtag; struct sysctl_oid *hn_tx_sysctl_tree; int hn_tx_flags; } __aligned(CACHE_LINE_SIZE); #define HN_TX_FLAG_ATTACHED 0x1 /* * Device-specific softc structure */ typedef struct hn_softc { struct ifnet *hn_ifp; struct ifmedia hn_media; device_t hn_dev; uint8_t hn_unit; int hn_carrier; int hn_if_flags; struct mtx hn_lock; int hn_initdone; /* See hv_netvsc_drv_freebsd.c for rules on how to use */ int temp_unusable; - struct rndis_device_ *rndis_dev; struct vmbus_channel *hn_prichan; int hn_rx_ring_cnt; int hn_rx_ring_inuse; struct hn_rx_ring *hn_rx_ring; int hn_tx_ring_cnt; int hn_tx_ring_inuse; struct hn_tx_ring *hn_tx_ring; uint8_t *hn_chim; u_long *hn_chim_bmap; int hn_chim_bmap_cnt; int hn_chim_cnt; int hn_chim_szmax; int hn_cpu; struct taskqueue *hn_tx_taskq; struct sysctl_oid *hn_tx_sysctl_tree; struct sysctl_oid *hn_rx_sysctl_tree; struct vmbus_xact_ctx *hn_xact; uint32_t hn_nvs_ver; uint32_t hn_flags; void *hn_rxbuf; uint32_t hn_rxbuf_gpadl; struct hyperv_dma hn_rxbuf_dma; uint32_t hn_chim_gpadl; struct hyperv_dma hn_chim_dma; uint32_t hn_rndis_rid; uint32_t hn_ndis_ver; struct ndis_rssprm_toeplitz hn_rss; } hn_softc_t; #define HN_FLAG_RXBUF_CONNECTED 0x0001 #define HN_FLAG_CHIM_CONNECTED 0x0002 /* * Externs */ extern int hv_promisc_mode; struct hn_send_ctx; void netvsc_linkstatus_callback(struct hn_softc *sc, uint32_t status); int hv_nv_on_device_add(struct hn_softc *sc, struct hn_rx_ring *rxr); -int hv_nv_on_device_remove(struct hn_softc *sc, - boolean_t destroy_channel); +int hv_nv_on_device_remove(struct hn_softc *sc); int hv_nv_on_send(struct vmbus_channel *chan, uint32_t rndis_mtype, struct hn_send_ctx *sndc, struct vmbus_gpa *gpa, int gpa_cnt); void hv_nv_subchan_attach(struct vmbus_channel *chan, struct hn_rx_ring *rxr); #endif /* __HV_NET_VSC_H__ */ Index: projects/clang390-import/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c (revision 305081) @@ -1,3098 +1,3098 @@ /*- * Copyright (c) 2010-2012 Citrix Inc. * Copyright (c) 2009-2012,2016 Microsoft Corp. * Copyright (c) 2012 NetApp Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2004-2006 Kip Macy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet6.h" #include "opt_inet.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "vmbus_if.h" /* Short for Hyper-V network interface */ #define NETVSC_DEVNAME "hn" /* * It looks like offset 0 of buf is reserved to hold the softc pointer. * The sc pointer evidently not needed, and is not presently populated. * The packet offset is where the netvsc_packet starts in the buffer. */ #define HV_NV_SC_PTR_OFFSET_IN_BUF 0 #define HV_NV_PACKET_OFFSET_IN_BUF 16 /* YYY should get it from the underlying channel */ #define HN_TX_DESC_CNT 512 #define HN_LROENT_CNT_DEF 128 #define HN_RING_CNT_DEF_MAX 8 #define HN_RNDIS_MSG_LEN \ (sizeof(rndis_msg) + \ RNDIS_HASHVAL_PPI_SIZE + \ RNDIS_VLAN_PPI_SIZE + \ RNDIS_TSO_PPI_SIZE + \ RNDIS_CSUM_PPI_SIZE) #define HN_RNDIS_MSG_BOUNDARY PAGE_SIZE #define HN_RNDIS_MSG_ALIGN CACHE_LINE_SIZE #define HN_TX_DATA_BOUNDARY PAGE_SIZE #define HN_TX_DATA_MAXSIZE IP_MAXPACKET #define HN_TX_DATA_SEGSIZE PAGE_SIZE -#define HN_TX_DATA_SEGCNT_MAX \ - (NETVSC_PACKET_MAXPAGE - HV_RF_NUM_TX_RESERVED_PAGE_BUFS) +/* -1 for RNDIS packet message */ +#define HN_TX_DATA_SEGCNT_MAX (NETVSC_PACKET_MAXPAGE - 1) #define HN_DIRECT_TX_SIZE_DEF 128 #define HN_EARLY_TXEOF_THRESH 8 struct hn_txdesc { #ifndef HN_USE_TXDESC_BUFRING SLIST_ENTRY(hn_txdesc) link; #endif struct mbuf *m; struct hn_tx_ring *txr; int refs; uint32_t flags; /* HN_TXD_FLAG_ */ struct hn_send_ctx send_ctx; bus_dmamap_t data_dmap; bus_addr_t rndis_msg_paddr; rndis_msg *rndis_msg; bus_dmamap_t rndis_msg_dmap; }; #define HN_TXD_FLAG_ONLIST 0x1 #define HN_TXD_FLAG_DMAMAP 0x2 /* * Only enable UDP checksum offloading when it is on 2012R2 or * later. UDP checksum offloading doesn't work on earlier * Windows releases. */ #define HN_CSUM_ASSIST_WIN8 (CSUM_IP | CSUM_TCP) #define HN_CSUM_ASSIST (CSUM_IP | CSUM_UDP | CSUM_TCP) #define HN_LRO_LENLIM_MULTIRX_DEF (12 * ETHERMTU) #define HN_LRO_LENLIM_DEF (25 * ETHERMTU) /* YYY 2*MTU is a bit rough, but should be good enough. */ #define HN_LRO_LENLIM_MIN(ifp) (2 * (ifp)->if_mtu) #define HN_LRO_ACKCNT_DEF 1 /* * Be aware that this sleepable mutex will exhibit WITNESS errors when * certain TCP and ARP code paths are taken. This appears to be a * well-known condition, as all other drivers checked use a sleeping * mutex to protect their transmit paths. * Also Be aware that mutexes do not play well with semaphores, and there * is a conflicting semaphore in a certain channel code path. */ #define NV_LOCK_INIT(_sc, _name) \ mtx_init(&(_sc)->hn_lock, _name, MTX_NETWORK_LOCK, MTX_DEF) #define NV_LOCK(_sc) mtx_lock(&(_sc)->hn_lock) #define NV_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->hn_lock, MA_OWNED) #define NV_UNLOCK(_sc) mtx_unlock(&(_sc)->hn_lock) #define NV_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->hn_lock) /* * Globals */ int hv_promisc_mode = 0; /* normal mode by default */ SYSCTL_NODE(_hw, OID_AUTO, hn, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Hyper-V network interface"); /* Trust tcp segements verification on host side. */ static int hn_trust_hosttcp = 1; SYSCTL_INT(_hw_hn, OID_AUTO, trust_hosttcp, CTLFLAG_RDTUN, &hn_trust_hosttcp, 0, "Trust tcp segement verification on host side, " "when csum info is missing (global setting)"); /* Trust udp datagrams verification on host side. */ static int hn_trust_hostudp = 1; SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostudp, CTLFLAG_RDTUN, &hn_trust_hostudp, 0, "Trust udp datagram verification on host side, " "when csum info is missing (global setting)"); /* Trust ip packets verification on host side. */ static int hn_trust_hostip = 1; SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostip, CTLFLAG_RDTUN, &hn_trust_hostip, 0, "Trust ip packet verification on host side, " "when csum info is missing (global setting)"); #if __FreeBSD_version >= 1100045 /* Limit TSO burst size */ static int hn_tso_maxlen = 0; SYSCTL_INT(_hw_hn, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, &hn_tso_maxlen, 0, "TSO burst limit"); #endif /* Limit chimney send size */ static int hn_tx_chimney_size = 0; SYSCTL_INT(_hw_hn, OID_AUTO, tx_chimney_size, CTLFLAG_RDTUN, &hn_tx_chimney_size, 0, "Chimney send packet size limit"); /* Limit the size of packet for direct transmission */ static int hn_direct_tx_size = HN_DIRECT_TX_SIZE_DEF; SYSCTL_INT(_hw_hn, OID_AUTO, direct_tx_size, CTLFLAG_RDTUN, &hn_direct_tx_size, 0, "Size of the packet for direct transmission"); #if defined(INET) || defined(INET6) #if __FreeBSD_version >= 1100095 static int hn_lro_entry_count = HN_LROENT_CNT_DEF; SYSCTL_INT(_hw_hn, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN, &hn_lro_entry_count, 0, "LRO entry count"); #endif #endif static int hn_share_tx_taskq = 0; SYSCTL_INT(_hw_hn, OID_AUTO, share_tx_taskq, CTLFLAG_RDTUN, &hn_share_tx_taskq, 0, "Enable shared TX taskqueue"); static struct taskqueue *hn_tx_taskq; #ifndef HN_USE_TXDESC_BUFRING static int hn_use_txdesc_bufring = 0; #else static int hn_use_txdesc_bufring = 1; #endif SYSCTL_INT(_hw_hn, OID_AUTO, use_txdesc_bufring, CTLFLAG_RD, &hn_use_txdesc_bufring, 0, "Use buf_ring for TX descriptors"); static int hn_bind_tx_taskq = -1; SYSCTL_INT(_hw_hn, OID_AUTO, bind_tx_taskq, CTLFLAG_RDTUN, &hn_bind_tx_taskq, 0, "Bind TX taskqueue to the specified cpu"); static int hn_use_if_start = 0; SYSCTL_INT(_hw_hn, OID_AUTO, use_if_start, CTLFLAG_RDTUN, &hn_use_if_start, 0, "Use if_start TX method"); static int hn_chan_cnt = 0; SYSCTL_INT(_hw_hn, OID_AUTO, chan_cnt, CTLFLAG_RDTUN, &hn_chan_cnt, 0, "# of channels to use; each channel has one RX ring and one TX ring"); static int hn_tx_ring_cnt = 0; SYSCTL_INT(_hw_hn, OID_AUTO, tx_ring_cnt, CTLFLAG_RDTUN, &hn_tx_ring_cnt, 0, "# of TX rings to use"); static int hn_tx_swq_depth = 0; SYSCTL_INT(_hw_hn, OID_AUTO, tx_swq_depth, CTLFLAG_RDTUN, &hn_tx_swq_depth, 0, "Depth of IFQ or BUFRING"); #if __FreeBSD_version >= 1100095 static u_int hn_lro_mbufq_depth = 0; SYSCTL_UINT(_hw_hn, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN, &hn_lro_mbufq_depth, 0, "Depth of LRO mbuf queue"); #endif static u_int hn_cpu_index; /* * Forward declarations */ static void hn_stop(hn_softc_t *sc); static void hn_ifinit_locked(hn_softc_t *sc); static void hn_ifinit(void *xsc); static int hn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data); static int hn_start_locked(struct hn_tx_ring *txr, int len); static void hn_start(struct ifnet *ifp); static void hn_start_txeof(struct hn_tx_ring *); static int hn_ifmedia_upd(struct ifnet *ifp); static void hn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr); #if __FreeBSD_version >= 1100099 static int hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS); static int hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS); #endif static int hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS); static int hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS); static int hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); static int hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS); static int hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); static int hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS); static int hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS); static int hn_check_iplen(const struct mbuf *, int); static int hn_create_tx_ring(struct hn_softc *, int); static void hn_destroy_tx_ring(struct hn_tx_ring *); static int hn_create_tx_data(struct hn_softc *, int); static void hn_destroy_tx_data(struct hn_softc *); static void hn_start_taskfunc(void *, int); static void hn_start_txeof_taskfunc(void *, int); static void hn_stop_tx_tasks(struct hn_softc *); static int hn_encap(struct hn_tx_ring *, struct hn_txdesc *, struct mbuf **); static int hn_create_rx_data(struct hn_softc *sc, int); static void hn_destroy_rx_data(struct hn_softc *sc); static void hn_set_chim_size(struct hn_softc *, int); static void hn_channel_attach(struct hn_softc *, struct vmbus_channel *); static void hn_subchan_attach(struct hn_softc *, struct vmbus_channel *); static void hn_subchan_setup(struct hn_softc *); static int hn_transmit(struct ifnet *, struct mbuf *); static void hn_xmit_qflush(struct ifnet *); static int hn_xmit(struct hn_tx_ring *, int); static void hn_xmit_txeof(struct hn_tx_ring *); static void hn_xmit_taskfunc(void *, int); static void hn_xmit_txeof_taskfunc(void *, int); #if __FreeBSD_version >= 1100099 static void hn_set_lro_lenlim(struct hn_softc *sc, int lenlim) { int i; for (i = 0; i < sc->hn_rx_ring_inuse; ++i) sc->hn_rx_ring[i].hn_lro.lro_length_lim = lenlim; } #endif static int hn_get_txswq_depth(const struct hn_tx_ring *txr) { KASSERT(txr->hn_txdesc_cnt > 0, ("tx ring is not setup yet")); if (hn_tx_swq_depth < txr->hn_txdesc_cnt) return txr->hn_txdesc_cnt; return hn_tx_swq_depth; } static int hn_ifmedia_upd(struct ifnet *ifp __unused) { return EOPNOTSUPP; } static void hn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) { struct hn_softc *sc = ifp->if_softc; ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!sc->hn_carrier) { ifmr->ifm_active |= IFM_NONE; return; } ifmr->ifm_status |= IFM_ACTIVE; ifmr->ifm_active |= IFM_10G_T | IFM_FDX; } /* {F8615163-DF3E-46c5-913F-F2D2F965ED0E} */ static const struct hyperv_guid g_net_vsc_device_type = { .hv_guid = {0x63, 0x51, 0x61, 0xF8, 0x3E, 0xDF, 0xc5, 0x46, 0x91, 0x3F, 0xF2, 0xD2, 0xF9, 0x65, 0xED, 0x0E} }; /* * Standard probe entry point. * */ static int netvsc_probe(device_t dev) { if (VMBUS_PROBE_GUID(device_get_parent(dev), dev, &g_net_vsc_device_type) == 0) { device_set_desc(dev, "Hyper-V Network Interface"); return BUS_PROBE_DEFAULT; } return ENXIO; } /* * Standard attach entry point. * * Called when the driver is loaded. It allocates needed resources, * and initializes the "hardware" and software. */ static int netvsc_attach(device_t dev) { struct sysctl_oid_list *child; struct sysctl_ctx_list *ctx; netvsc_device_info device_info; hn_softc_t *sc; int unit = device_get_unit(dev); struct ifnet *ifp = NULL; int error, ring_cnt, tx_ring_cnt; #if __FreeBSD_version >= 1100045 int tso_maxlen; #endif sc = device_get_softc(dev); sc->hn_unit = unit; sc->hn_dev = dev; sc->hn_prichan = vmbus_get_channel(dev); if (hn_tx_taskq == NULL) { sc->hn_tx_taskq = taskqueue_create("hn_tx", M_WAITOK, taskqueue_thread_enqueue, &sc->hn_tx_taskq); if (hn_bind_tx_taskq >= 0) { int cpu = hn_bind_tx_taskq; cpuset_t cpu_set; if (cpu > mp_ncpus - 1) cpu = mp_ncpus - 1; CPU_SETOF(cpu, &cpu_set); taskqueue_start_threads_cpuset(&sc->hn_tx_taskq, 1, PI_NET, &cpu_set, "%s tx", device_get_nameunit(dev)); } else { taskqueue_start_threads(&sc->hn_tx_taskq, 1, PI_NET, "%s tx", device_get_nameunit(dev)); } } else { sc->hn_tx_taskq = hn_tx_taskq; } NV_LOCK_INIT(sc, "NetVSCLock"); ifp = sc->hn_ifp = if_alloc(IFT_ETHER); ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); /* * Figure out the # of RX rings (ring_cnt) and the # of TX rings * to use (tx_ring_cnt). * * NOTE: * The # of RX rings to use is same as the # of channels to use. */ ring_cnt = hn_chan_cnt; if (ring_cnt <= 0) { /* Default */ ring_cnt = mp_ncpus; if (ring_cnt > HN_RING_CNT_DEF_MAX) ring_cnt = HN_RING_CNT_DEF_MAX; } else if (ring_cnt > mp_ncpus) { ring_cnt = mp_ncpus; } tx_ring_cnt = hn_tx_ring_cnt; if (tx_ring_cnt <= 0 || tx_ring_cnt > ring_cnt) tx_ring_cnt = ring_cnt; if (hn_use_if_start) { /* ifnet.if_start only needs one TX ring. */ tx_ring_cnt = 1; } /* * Set the leader CPU for channels. */ sc->hn_cpu = atomic_fetchadd_int(&hn_cpu_index, ring_cnt) % mp_ncpus; error = hn_create_tx_data(sc, tx_ring_cnt); if (error) goto failed; error = hn_create_rx_data(sc, ring_cnt); if (error) goto failed; /* * Associate the first TX/RX ring w/ the primary channel. */ hn_channel_attach(sc, sc->hn_prichan); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = hn_ioctl; ifp->if_init = hn_ifinit; /* needed by hv_rf_on_device_add() code */ ifp->if_mtu = ETHERMTU; if (hn_use_if_start) { int qdepth = hn_get_txswq_depth(&sc->hn_tx_ring[0]); ifp->if_start = hn_start; IFQ_SET_MAXLEN(&ifp->if_snd, qdepth); ifp->if_snd.ifq_drv_maxlen = qdepth - 1; IFQ_SET_READY(&ifp->if_snd); } else { ifp->if_transmit = hn_transmit; ifp->if_qflush = hn_xmit_qflush; } ifmedia_init(&sc->hn_media, 0, hn_ifmedia_upd, hn_ifmedia_sts); ifmedia_add(&sc->hn_media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&sc->hn_media, IFM_ETHER | IFM_AUTO); /* XXX ifmedia_set really should do this for us */ sc->hn_media.ifm_media = sc->hn_media.ifm_cur->ifm_media; /* * Tell upper layers that we support full VLAN capability. */ ifp->if_hdrlen = sizeof(struct ether_vlan_header); ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_TSO | IFCAP_LRO; ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_TSO | IFCAP_LRO; ifp->if_hwassist = sc->hn_tx_ring[0].hn_csum_assist | CSUM_TSO; sc->hn_xact = vmbus_xact_ctx_create(bus_get_dma_tag(dev), HN_XACT_REQ_SIZE, HN_XACT_RESP_SIZE, 0); if (sc->hn_xact == NULL) goto failed; error = hv_rf_on_device_add(sc, &device_info, &ring_cnt, &sc->hn_rx_ring[0]); if (error) goto failed; KASSERT(ring_cnt > 0 && ring_cnt <= sc->hn_rx_ring_inuse, ("invalid channel count %d, should be less than %d", ring_cnt, sc->hn_rx_ring_inuse)); /* * Set the # of TX/RX rings that could be used according to * the # of channels that host offered. */ if (sc->hn_tx_ring_inuse > ring_cnt) sc->hn_tx_ring_inuse = ring_cnt; sc->hn_rx_ring_inuse = ring_cnt; device_printf(dev, "%d TX ring, %d RX ring\n", sc->hn_tx_ring_inuse, sc->hn_rx_ring_inuse); if (sc->hn_rx_ring_inuse > 1) hn_subchan_setup(sc); #if __FreeBSD_version >= 1100099 if (sc->hn_rx_ring_inuse > 1) { /* * Reduce TCP segment aggregation limit for multiple * RX rings to increase ACK timeliness. */ hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MULTIRX_DEF); } #endif if (device_info.link_state == NDIS_MEDIA_STATE_CONNECTED) { sc->hn_carrier = 1; } #if __FreeBSD_version >= 1100045 tso_maxlen = hn_tso_maxlen; if (tso_maxlen <= 0 || tso_maxlen > IP_MAXPACKET) tso_maxlen = IP_MAXPACKET; ifp->if_hw_tsomaxsegcount = HN_TX_DATA_SEGCNT_MAX; ifp->if_hw_tsomaxsegsize = PAGE_SIZE; ifp->if_hw_tsomax = tso_maxlen - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); #endif ether_ifattach(ifp, device_info.mac_addr); #if __FreeBSD_version >= 1100045 if_printf(ifp, "TSO: %u/%u/%u\n", ifp->if_hw_tsomax, ifp->if_hw_tsomaxsegcount, ifp->if_hw_tsomaxsegsize); #endif hn_set_chim_size(sc, sc->hn_chim_szmax); if (hn_tx_chimney_size > 0 && hn_tx_chimney_size < sc->hn_chim_szmax) hn_set_chim_size(sc, hn_tx_chimney_size); ctx = device_get_sysctl_ctx(dev); child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "nvs_version", CTLFLAG_RD, &sc->hn_nvs_ver, 0, "NVS version"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "ndis_version", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, hn_ndis_version_sysctl, "A", "NDIS version"); return (0); failed: hn_destroy_tx_data(sc); if (ifp != NULL) if_free(ifp); return (error); } /* * Standard detach entry point */ static int netvsc_detach(device_t dev) { struct hn_softc *sc = device_get_softc(dev); if (bootverbose) printf("netvsc_detach\n"); /* * XXXKYS: Need to clean up all our * driver state; this is the driver * unloading. */ /* * XXXKYS: Need to stop outgoing traffic and unregister * the netdevice. */ - hv_rf_on_device_remove(sc, HV_RF_NV_DESTROY_CHANNEL); + hv_rf_on_device_remove(sc); hn_stop_tx_tasks(sc); ifmedia_removeall(&sc->hn_media); hn_destroy_rx_data(sc); hn_destroy_tx_data(sc); if (sc->hn_tx_taskq != hn_tx_taskq) taskqueue_free(sc->hn_tx_taskq); vmbus_xact_ctx_destroy(sc->hn_xact); return (0); } /* * Standard shutdown entry point */ static int netvsc_shutdown(device_t dev) { return (0); } static __inline int hn_txdesc_dmamap_load(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head, bus_dma_segment_t *segs, int *nsegs) { struct mbuf *m = *m_head; int error; error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, txd->data_dmap, m, segs, nsegs, BUS_DMA_NOWAIT); if (error == EFBIG) { struct mbuf *m_new; m_new = m_collapse(m, M_NOWAIT, HN_TX_DATA_SEGCNT_MAX); if (m_new == NULL) return ENOBUFS; else *m_head = m = m_new; txr->hn_tx_collapsed++; error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, txd->data_dmap, m, segs, nsegs, BUS_DMA_NOWAIT); } if (!error) { bus_dmamap_sync(txr->hn_tx_data_dtag, txd->data_dmap, BUS_DMASYNC_PREWRITE); txd->flags |= HN_TXD_FLAG_DMAMAP; } return error; } static __inline void hn_txdesc_dmamap_unload(struct hn_tx_ring *txr, struct hn_txdesc *txd) { if (txd->flags & HN_TXD_FLAG_DMAMAP) { bus_dmamap_sync(txr->hn_tx_data_dtag, txd->data_dmap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->hn_tx_data_dtag, txd->data_dmap); txd->flags &= ~HN_TXD_FLAG_DMAMAP; } } static __inline int hn_txdesc_put(struct hn_tx_ring *txr, struct hn_txdesc *txd) { KASSERT((txd->flags & HN_TXD_FLAG_ONLIST) == 0, ("put an onlist txd %#x", txd->flags)); KASSERT(txd->refs > 0, ("invalid txd refs %d", txd->refs)); if (atomic_fetchadd_int(&txd->refs, -1) != 1) return 0; hn_txdesc_dmamap_unload(txr, txd); if (txd->m != NULL) { m_freem(txd->m); txd->m = NULL; } txd->flags |= HN_TXD_FLAG_ONLIST; #ifndef HN_USE_TXDESC_BUFRING mtx_lock_spin(&txr->hn_txlist_spin); KASSERT(txr->hn_txdesc_avail >= 0 && txr->hn_txdesc_avail < txr->hn_txdesc_cnt, ("txdesc_put: invalid txd avail %d", txr->hn_txdesc_avail)); txr->hn_txdesc_avail++; SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); mtx_unlock_spin(&txr->hn_txlist_spin); #else atomic_add_int(&txr->hn_txdesc_avail, 1); buf_ring_enqueue(txr->hn_txdesc_br, txd); #endif return 1; } static __inline struct hn_txdesc * hn_txdesc_get(struct hn_tx_ring *txr) { struct hn_txdesc *txd; #ifndef HN_USE_TXDESC_BUFRING mtx_lock_spin(&txr->hn_txlist_spin); txd = SLIST_FIRST(&txr->hn_txlist); if (txd != NULL) { KASSERT(txr->hn_txdesc_avail > 0, ("txdesc_get: invalid txd avail %d", txr->hn_txdesc_avail)); txr->hn_txdesc_avail--; SLIST_REMOVE_HEAD(&txr->hn_txlist, link); } mtx_unlock_spin(&txr->hn_txlist_spin); #else txd = buf_ring_dequeue_sc(txr->hn_txdesc_br); #endif if (txd != NULL) { #ifdef HN_USE_TXDESC_BUFRING atomic_subtract_int(&txr->hn_txdesc_avail, 1); #endif KASSERT(txd->m == NULL && txd->refs == 0 && (txd->flags & HN_TXD_FLAG_ONLIST), ("invalid txd")); txd->flags &= ~HN_TXD_FLAG_ONLIST; txd->refs = 1; } return txd; } static __inline void hn_txdesc_hold(struct hn_txdesc *txd) { /* 0->1 transition will never work */ KASSERT(txd->refs > 0, ("invalid refs %d", txd->refs)); atomic_add_int(&txd->refs, 1); } static __inline void hn_txeof(struct hn_tx_ring *txr) { txr->hn_has_txeof = 0; txr->hn_txeof(txr); } static void hn_tx_done(struct hn_send_ctx *sndc, struct hn_softc *sc, struct vmbus_channel *chan, const void *data __unused, int dlen __unused) { struct hn_txdesc *txd = sndc->hn_cbarg; struct hn_tx_ring *txr; if (sndc->hn_chim_idx != HN_NVS_CHIM_IDX_INVALID) hn_chim_free(sc, sndc->hn_chim_idx); txr = txd->txr; KASSERT(txr->hn_chan == chan, ("channel mismatch, on chan%u, should be chan%u", vmbus_chan_subidx(chan), vmbus_chan_subidx(txr->hn_chan))); txr->hn_has_txeof = 1; hn_txdesc_put(txr, txd); ++txr->hn_txdone_cnt; if (txr->hn_txdone_cnt >= HN_EARLY_TXEOF_THRESH) { txr->hn_txdone_cnt = 0; if (txr->hn_oactive) hn_txeof(txr); } } void netvsc_channel_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr) { #if defined(INET) || defined(INET6) tcp_lro_flush_all(&rxr->hn_lro); #endif /* * NOTE: * 'txr' could be NULL, if multiple channels and * ifnet.if_start method are enabled. */ if (txr == NULL || !txr->hn_has_txeof) return; txr->hn_txdone_cnt = 0; hn_txeof(txr); } /* * NOTE: * If this function fails, then both txd and m_head0 will be freed. */ static int hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) { bus_dma_segment_t segs[HN_TX_DATA_SEGCNT_MAX]; int error, nsegs, i; struct mbuf *m_head = *m_head0; rndis_msg *rndis_mesg; rndis_packet *rndis_pkt; rndis_per_packet_info *rppi; struct rndis_hash_value *hash_value; uint32_t rndis_msg_size, tot_data_buf_len, send_buf_section_idx; int send_buf_section_size; tot_data_buf_len = m_head->m_pkthdr.len; /* * extension points to the area reserved for the * rndis_filter_packet, which is placed just after * the netvsc_packet (and rppi struct, if present; * length is updated later). */ rndis_mesg = txd->rndis_msg; /* XXX not necessary */ memset(rndis_mesg, 0, HN_RNDIS_MSG_LEN); rndis_mesg->ndis_msg_type = REMOTE_NDIS_PACKET_MSG; rndis_pkt = &rndis_mesg->msg.packet; rndis_pkt->data_offset = sizeof(rndis_packet); rndis_pkt->data_length = tot_data_buf_len; rndis_pkt->per_pkt_info_offset = sizeof(rndis_packet); rndis_msg_size = RNDIS_MESSAGE_SIZE(rndis_packet); /* * Set the hash value for this packet, so that the host could * dispatch the TX done event for this packet back to this TX * ring's channel. */ rndis_msg_size += RNDIS_HASHVAL_PPI_SIZE; rppi = hv_set_rppi_data(rndis_mesg, RNDIS_HASHVAL_PPI_SIZE, nbl_hash_value); hash_value = (struct rndis_hash_value *)((uint8_t *)rppi + rppi->per_packet_info_offset); hash_value->hash_value = txr->hn_tx_idx; if (m_head->m_flags & M_VLANTAG) { ndis_8021q_info *rppi_vlan_info; rndis_msg_size += RNDIS_VLAN_PPI_SIZE; rppi = hv_set_rppi_data(rndis_mesg, RNDIS_VLAN_PPI_SIZE, ieee_8021q_info); rppi_vlan_info = (ndis_8021q_info *)((uint8_t *)rppi + rppi->per_packet_info_offset); rppi_vlan_info->u1.s1.vlan_id = m_head->m_pkthdr.ether_vtag & 0xfff; } if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { rndis_tcp_tso_info *tso_info; struct ether_vlan_header *eh; int ether_len; /* * XXX need m_pullup and use mtodo */ eh = mtod(m_head, struct ether_vlan_header*); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) ether_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; else ether_len = ETHER_HDR_LEN; rndis_msg_size += RNDIS_TSO_PPI_SIZE; rppi = hv_set_rppi_data(rndis_mesg, RNDIS_TSO_PPI_SIZE, tcp_large_send_info); tso_info = (rndis_tcp_tso_info *)((uint8_t *)rppi + rppi->per_packet_info_offset); tso_info->lso_v2_xmit.type = RNDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE; #ifdef INET if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { struct ip *ip = (struct ip *)(m_head->m_data + ether_len); unsigned long iph_len = ip->ip_hl << 2; struct tcphdr *th = (struct tcphdr *)((caddr_t)ip + iph_len); tso_info->lso_v2_xmit.ip_version = RNDIS_TCP_LARGE_SEND_OFFLOAD_IPV4; ip->ip_len = 0; ip->ip_sum = 0; th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(IPPROTO_TCP)); } #endif #if defined(INET6) && defined(INET) else #endif #ifdef INET6 { struct ip6_hdr *ip6 = (struct ip6_hdr *) (m_head->m_data + ether_len); struct tcphdr *th = (struct tcphdr *)(ip6 + 1); tso_info->lso_v2_xmit.ip_version = RNDIS_TCP_LARGE_SEND_OFFLOAD_IPV6; ip6->ip6_plen = 0; th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); } #endif tso_info->lso_v2_xmit.tcp_header_offset = 0; tso_info->lso_v2_xmit.mss = m_head->m_pkthdr.tso_segsz; } else if (m_head->m_pkthdr.csum_flags & txr->hn_csum_assist) { rndis_tcp_ip_csum_info *csum_info; rndis_msg_size += RNDIS_CSUM_PPI_SIZE; rppi = hv_set_rppi_data(rndis_mesg, RNDIS_CSUM_PPI_SIZE, tcpip_chksum_info); csum_info = (rndis_tcp_ip_csum_info *)((uint8_t *)rppi + rppi->per_packet_info_offset); csum_info->xmit.is_ipv4 = 1; if (m_head->m_pkthdr.csum_flags & CSUM_IP) csum_info->xmit.ip_header_csum = 1; if (m_head->m_pkthdr.csum_flags & CSUM_TCP) { csum_info->xmit.tcp_csum = 1; csum_info->xmit.tcp_header_offset = 0; } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) { csum_info->xmit.udp_csum = 1; } } rndis_mesg->msg_len = tot_data_buf_len + rndis_msg_size; tot_data_buf_len = rndis_mesg->msg_len; /* * Chimney send, if the packet could fit into one chimney buffer. */ if (tot_data_buf_len < txr->hn_chim_size) { txr->hn_tx_chimney_tried++; send_buf_section_idx = hn_chim_alloc(txr->hn_sc); if (send_buf_section_idx != HN_NVS_CHIM_IDX_INVALID) { uint8_t *dest = txr->hn_sc->hn_chim + (send_buf_section_idx * txr->hn_sc->hn_chim_szmax); memcpy(dest, rndis_mesg, rndis_msg_size); dest += rndis_msg_size; m_copydata(m_head, 0, m_head->m_pkthdr.len, dest); send_buf_section_size = tot_data_buf_len; txr->hn_gpa_cnt = 0; txr->hn_tx_chimney++; goto done; } } error = hn_txdesc_dmamap_load(txr, txd, &m_head, segs, &nsegs); if (error) { int freed; /* * This mbuf is not linked w/ the txd yet, so free it now. */ m_freem(m_head); *m_head0 = NULL; freed = hn_txdesc_put(txr, txd); KASSERT(freed != 0, ("fail to free txd upon txdma error")); txr->hn_txdma_failed++; if_inc_counter(txr->hn_sc->hn_ifp, IFCOUNTER_OERRORS, 1); return error; } *m_head0 = m_head; - txr->hn_gpa_cnt = nsegs + HV_RF_NUM_TX_RESERVED_PAGE_BUFS; + /* +1 RNDIS packet message */ + txr->hn_gpa_cnt = nsegs + 1; /* send packet with page buffer */ txr->hn_gpa[0].gpa_page = atop(txd->rndis_msg_paddr); txr->hn_gpa[0].gpa_ofs = txd->rndis_msg_paddr & PAGE_MASK; txr->hn_gpa[0].gpa_len = rndis_msg_size; /* - * Fill the page buffers with mbuf info starting at index - * HV_RF_NUM_TX_RESERVED_PAGE_BUFS. + * Fill the page buffers with mbuf info after the page + * buffer for RNDIS packet message. */ for (i = 0; i < nsegs; ++i) { - struct vmbus_gpa *gpa = &txr->hn_gpa[ - i + HV_RF_NUM_TX_RESERVED_PAGE_BUFS]; + struct vmbus_gpa *gpa = &txr->hn_gpa[i + 1]; gpa->gpa_page = atop(segs[i].ds_addr); gpa->gpa_ofs = segs[i].ds_addr & PAGE_MASK; gpa->gpa_len = segs[i].ds_len; } send_buf_section_idx = HN_NVS_CHIM_IDX_INVALID; send_buf_section_size = 0; done: txd->m = m_head; /* Set the completion routine */ hn_send_ctx_init(&txd->send_ctx, hn_tx_done, txd, send_buf_section_idx, send_buf_section_size); return 0; } /* * NOTE: * If this function fails, then txd will be freed, but the mbuf * associated w/ the txd will _not_ be freed. */ static int hn_send_pkt(struct ifnet *ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd) { int error, send_failed = 0; again: /* * Make sure that txd is not freed before ETHER_BPF_MTAP. */ hn_txdesc_hold(txd); error = hv_nv_on_send(txr->hn_chan, HN_NVS_RNDIS_MTYPE_DATA, &txd->send_ctx, txr->hn_gpa, txr->hn_gpa_cnt); if (!error) { ETHER_BPF_MTAP(ifp, txd->m); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); if (!hn_use_if_start) { if_inc_counter(ifp, IFCOUNTER_OBYTES, txd->m->m_pkthdr.len); if (txd->m->m_flags & M_MCAST) if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); } txr->hn_pkts++; } hn_txdesc_put(txr, txd); if (__predict_false(error)) { int freed; /* * This should "really rarely" happen. * * XXX Too many RX to be acked or too many sideband * commands to run? Ask netvsc_channel_rollup() * to kick start later. */ txr->hn_has_txeof = 1; if (!send_failed) { txr->hn_send_failed++; send_failed = 1; /* * Try sending again after set hn_has_txeof; * in case that we missed the last * netvsc_channel_rollup(). */ goto again; } if_printf(ifp, "send failed\n"); /* * Caller will perform further processing on the * associated mbuf, so don't free it in hn_txdesc_put(); * only unload it from the DMA map in hn_txdesc_put(), * if it was loaded. */ txd->m = NULL; freed = hn_txdesc_put(txr, txd); KASSERT(freed != 0, ("fail to free txd upon send error")); txr->hn_send_failed++; } return error; } /* * Start a transmit of one or more packets */ static int hn_start_locked(struct hn_tx_ring *txr, int len) { struct hn_softc *sc = txr->hn_sc; struct ifnet *ifp = sc->hn_ifp; KASSERT(hn_use_if_start, ("hn_start_locked is called, when if_start is disabled")); KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); mtx_assert(&txr->hn_tx_lock, MA_OWNED); if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return 0; while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { struct hn_txdesc *txd; struct mbuf *m_head; int error; IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; if (len > 0 && m_head->m_pkthdr.len > len) { /* * This sending could be time consuming; let callers * dispatch this packet sending (and sending of any * following up packets) to tx taskqueue. */ IFQ_DRV_PREPEND(&ifp->if_snd, m_head); return 1; } txd = hn_txdesc_get(txr); if (txd == NULL) { txr->hn_no_txdescs++; IFQ_DRV_PREPEND(&ifp->if_snd, m_head); atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); break; } error = hn_encap(txr, txd, &m_head); if (error) { /* Both txd and m_head are freed */ continue; } error = hn_send_pkt(ifp, txr, txd); if (__predict_false(error)) { /* txd is freed, but m_head is not */ IFQ_DRV_PREPEND(&ifp->if_snd, m_head); atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); break; } } return 0; } /* * Link up/down notification */ void netvsc_linkstatus_callback(struct hn_softc *sc, uint32_t status) { if (status == 1) { sc->hn_carrier = 1; } else { sc->hn_carrier = 0; } } /* * Append the specified data to the indicated mbuf chain, * Extend the mbuf chain if the new data does not fit in * existing space. * * This is a minor rewrite of m_append() from sys/kern/uipc_mbuf.c. * There should be an equivalent in the kernel mbuf code, * but there does not appear to be one yet. * * Differs from m_append() in that additional mbufs are * allocated with cluster size MJUMPAGESIZE, and filled * accordingly. * * Return 1 if able to complete the job; otherwise 0. */ static int hv_m_append(struct mbuf *m0, int len, c_caddr_t cp) { struct mbuf *m, *n; int remainder, space; for (m = m0; m->m_next != NULL; m = m->m_next) ; remainder = len; space = M_TRAILINGSPACE(m); if (space > 0) { /* * Copy into available space. */ if (space > remainder) space = remainder; bcopy(cp, mtod(m, caddr_t) + m->m_len, space); m->m_len += space; cp += space; remainder -= space; } while (remainder > 0) { /* * Allocate a new mbuf; could check space * and allocate a cluster instead. */ n = m_getjcl(M_NOWAIT, m->m_type, 0, MJUMPAGESIZE); if (n == NULL) break; n->m_len = min(MJUMPAGESIZE, remainder); bcopy(cp, mtod(n, caddr_t), n->m_len); cp += n->m_len; remainder -= n->m_len; m->m_next = n; m = n; } if (m0->m_flags & M_PKTHDR) m0->m_pkthdr.len += len - remainder; return (remainder == 0); } #if defined(INET) || defined(INET6) static __inline int hn_lro_rx(struct lro_ctrl *lc, struct mbuf *m) { #if __FreeBSD_version >= 1100095 if (hn_lro_mbufq_depth) { tcp_lro_queue_mbuf(lc, m); return 0; } #endif return tcp_lro_rx(lc, m, 0); } #endif /* * Called when we receive a data packet from the "wire" on the * specified device * * Note: This is no longer used as a callback */ int netvsc_recv(struct hn_rx_ring *rxr, const void *data, int dlen, const struct hn_recvinfo *info) { struct ifnet *ifp = rxr->hn_ifp; struct mbuf *m_new; int size, do_lro = 0, do_csum = 1; int hash_type = M_HASHTYPE_OPAQUE_HASH; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) return (0); /* * Bail out if packet contains more data than configured MTU. */ if (dlen > (ifp->if_mtu + ETHER_HDR_LEN)) { return (0); } else if (dlen <= MHLEN) { m_new = m_gethdr(M_NOWAIT, MT_DATA); if (m_new == NULL) { if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); return (0); } memcpy(mtod(m_new, void *), data, dlen); m_new->m_pkthdr.len = m_new->m_len = dlen; rxr->hn_small_pkts++; } else { /* * Get an mbuf with a cluster. For packets 2K or less, * get a standard 2K cluster. For anything larger, get a * 4K cluster. Any buffers larger than 4K can cause problems * if looped around to the Hyper-V TX channel, so avoid them. */ size = MCLBYTES; if (dlen > MCLBYTES) { /* 4096 */ size = MJUMPAGESIZE; } m_new = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size); if (m_new == NULL) { if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); return (0); } hv_m_append(m_new, dlen, data); } m_new->m_pkthdr.rcvif = ifp; if (__predict_false((ifp->if_capenable & IFCAP_RXCSUM) == 0)) do_csum = 0; /* receive side checksum offload */ if (info->csum_info != NULL) { /* IP csum offload */ if (info->csum_info->receive.ip_csum_succeeded && do_csum) { m_new->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID); rxr->hn_csum_ip++; } /* TCP/UDP csum offload */ if ((info->csum_info->receive.tcp_csum_succeeded || info->csum_info->receive.udp_csum_succeeded) && do_csum) { m_new->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); m_new->m_pkthdr.csum_data = 0xffff; if (info->csum_info->receive.tcp_csum_succeeded) rxr->hn_csum_tcp++; else rxr->hn_csum_udp++; } if (info->csum_info->receive.ip_csum_succeeded && info->csum_info->receive.tcp_csum_succeeded) do_lro = 1; } else { const struct ether_header *eh; uint16_t etype; int hoff; hoff = sizeof(*eh); if (m_new->m_len < hoff) goto skip; eh = mtod(m_new, struct ether_header *); etype = ntohs(eh->ether_type); if (etype == ETHERTYPE_VLAN) { const struct ether_vlan_header *evl; hoff = sizeof(*evl); if (m_new->m_len < hoff) goto skip; evl = mtod(m_new, struct ether_vlan_header *); etype = ntohs(evl->evl_proto); } if (etype == ETHERTYPE_IP) { int pr; pr = hn_check_iplen(m_new, hoff); if (pr == IPPROTO_TCP) { if (do_csum && (rxr->hn_trust_hcsum & HN_TRUST_HCSUM_TCP)) { rxr->hn_csum_trusted++; m_new->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR); m_new->m_pkthdr.csum_data = 0xffff; } do_lro = 1; } else if (pr == IPPROTO_UDP) { if (do_csum && (rxr->hn_trust_hcsum & HN_TRUST_HCSUM_UDP)) { rxr->hn_csum_trusted++; m_new->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR); m_new->m_pkthdr.csum_data = 0xffff; } } else if (pr != IPPROTO_DONE && do_csum && (rxr->hn_trust_hcsum & HN_TRUST_HCSUM_IP)) { rxr->hn_csum_trusted++; m_new->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID); } } } skip: if (info->vlan_info != NULL) { m_new->m_pkthdr.ether_vtag = info->vlan_info->u1.s1.vlan_id; m_new->m_flags |= M_VLANTAG; } if (info->hash_info != NULL && info->hash_value != NULL) { rxr->hn_rss_pkts++; m_new->m_pkthdr.flowid = info->hash_value->hash_value; if ((info->hash_info->hash_info & NDIS_HASH_FUNCTION_MASK) == NDIS_HASH_FUNCTION_TOEPLITZ) { uint32_t type = (info->hash_info->hash_info & NDIS_HASH_TYPE_MASK); switch (type) { case NDIS_HASH_IPV4: hash_type = M_HASHTYPE_RSS_IPV4; break; case NDIS_HASH_TCP_IPV4: hash_type = M_HASHTYPE_RSS_TCP_IPV4; break; case NDIS_HASH_IPV6: hash_type = M_HASHTYPE_RSS_IPV6; break; case NDIS_HASH_IPV6_EX: hash_type = M_HASHTYPE_RSS_IPV6_EX; break; case NDIS_HASH_TCP_IPV6: hash_type = M_HASHTYPE_RSS_TCP_IPV6; break; case NDIS_HASH_TCP_IPV6_EX: hash_type = M_HASHTYPE_RSS_TCP_IPV6_EX; break; } } } else { if (info->hash_value != NULL) { m_new->m_pkthdr.flowid = info->hash_value->hash_value; } else { m_new->m_pkthdr.flowid = rxr->hn_rx_idx; hash_type = M_HASHTYPE_OPAQUE; } } M_HASHTYPE_SET(m_new, hash_type); /* * Note: Moved RX completion back to hv_nv_on_receive() so all * messages (not just data messages) will trigger a response. */ if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); rxr->hn_pkts++; if ((ifp->if_capenable & IFCAP_LRO) && do_lro) { #if defined(INET) || defined(INET6) struct lro_ctrl *lro = &rxr->hn_lro; if (lro->lro_cnt) { rxr->hn_lro_tried++; if (hn_lro_rx(lro, m_new) == 0) { /* DONE! */ return 0; } } #endif } /* We're not holding the lock here, so don't release it */ (*ifp->if_input)(ifp, m_new); return (0); } /* * Rules for using sc->temp_unusable: * 1. sc->temp_unusable can only be read or written while holding NV_LOCK() * 2. code reading sc->temp_unusable under NV_LOCK(), and finding * sc->temp_unusable set, must release NV_LOCK() and exit * 3. to retain exclusive control of the interface, * sc->temp_unusable must be set by code before releasing NV_LOCK() * 4. only code setting sc->temp_unusable can clear sc->temp_unusable * 5. code setting sc->temp_unusable must eventually clear sc->temp_unusable */ /* * Standard ioctl entry point. Called when the user wants to configure * the interface. */ static int hn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { hn_softc_t *sc = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; #ifdef INET struct ifaddr *ifa = (struct ifaddr *)data; #endif netvsc_device_info device_info; int mask, error = 0, ring_cnt; int retry_cnt = 500; switch(cmd) { case SIOCSIFADDR: #ifdef INET if (ifa->ifa_addr->sa_family == AF_INET) { ifp->if_flags |= IFF_UP; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) hn_ifinit(sc); arp_ifinit(ifp, ifa); } else #endif error = ether_ioctl(ifp, cmd, data); break; case SIOCSIFMTU: /* Check MTU value change */ if (ifp->if_mtu == ifr->ifr_mtu) break; if (ifr->ifr_mtu > NETVSC_MAX_CONFIGURABLE_MTU) { error = EINVAL; break; } /* Obtain and record requested MTU */ ifp->if_mtu = ifr->ifr_mtu; #if __FreeBSD_version >= 1100099 /* * Make sure that LRO aggregation length limit is still * valid, after the MTU change. */ NV_LOCK(sc); if (sc->hn_rx_ring[0].hn_lro.lro_length_lim < HN_LRO_LENLIM_MIN(ifp)) hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MIN(ifp)); NV_UNLOCK(sc); #endif do { NV_LOCK(sc); if (!sc->temp_unusable) { sc->temp_unusable = TRUE; retry_cnt = -1; } NV_UNLOCK(sc); if (retry_cnt > 0) { retry_cnt--; DELAY(5 * 1000); } } while (retry_cnt > 0); if (retry_cnt == 0) { error = EINVAL; break; } /* We must remove and add back the device to cause the new * MTU to take effect. This includes tearing down, but not * deleting the channel, then bringing it back up. */ - error = hv_rf_on_device_remove(sc, HV_RF_NV_RETAIN_CHANNEL); + error = hv_rf_on_device_remove(sc); if (error) { NV_LOCK(sc); sc->temp_unusable = FALSE; NV_UNLOCK(sc); break; } /* Wait for subchannels to be destroyed */ vmbus_subchan_drain(sc->hn_prichan); ring_cnt = sc->hn_rx_ring_inuse; error = hv_rf_on_device_add(sc, &device_info, &ring_cnt, &sc->hn_rx_ring[0]); if (error) { NV_LOCK(sc); sc->temp_unusable = FALSE; NV_UNLOCK(sc); break; } /* # of channels can _not_ be changed */ KASSERT(sc->hn_rx_ring_inuse == ring_cnt, ("RX ring count %d and channel count %u mismatch", sc->hn_rx_ring_cnt, ring_cnt)); if (sc->hn_rx_ring_inuse > 1) { int r; /* * Skip the rings on primary channel; they are * handled by the hv_rf_on_device_add() above. */ for (r = 1; r < sc->hn_rx_ring_cnt; ++r) { sc->hn_rx_ring[r].hn_rx_flags &= ~HN_RX_FLAG_ATTACHED; } for (r = 1; r < sc->hn_tx_ring_cnt; ++r) { sc->hn_tx_ring[r].hn_tx_flags &= ~HN_TX_FLAG_ATTACHED; } hn_subchan_setup(sc); } if (sc->hn_tx_ring[0].hn_chim_size > sc->hn_chim_szmax) hn_set_chim_size(sc, sc->hn_chim_szmax); hn_ifinit_locked(sc); NV_LOCK(sc); sc->temp_unusable = FALSE; NV_UNLOCK(sc); break; case SIOCSIFFLAGS: do { NV_LOCK(sc); if (!sc->temp_unusable) { sc->temp_unusable = TRUE; retry_cnt = -1; } NV_UNLOCK(sc); if (retry_cnt > 0) { retry_cnt--; DELAY(5 * 1000); } } while (retry_cnt > 0); if (retry_cnt == 0) { error = EINVAL; break; } if (ifp->if_flags & IFF_UP) { /* * If only the state of the PROMISC flag changed, * then just use the 'set promisc mode' command * instead of reinitializing the entire NIC. Doing * a full re-init means reloading the firmware and * waiting for it to start up, which may take a * second or two. */ #ifdef notyet /* Fixme: Promiscuous mode? */ if (ifp->if_drv_flags & IFF_DRV_RUNNING && ifp->if_flags & IFF_PROMISC && !(sc->hn_if_flags & IFF_PROMISC)) { /* do something here for Hyper-V */ } else if (ifp->if_drv_flags & IFF_DRV_RUNNING && !(ifp->if_flags & IFF_PROMISC) && sc->hn_if_flags & IFF_PROMISC) { /* do something here for Hyper-V */ } else #endif hn_ifinit_locked(sc); } else { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { hn_stop(sc); } } NV_LOCK(sc); sc->temp_unusable = FALSE; NV_UNLOCK(sc); sc->hn_if_flags = ifp->if_flags; error = 0; break; case SIOCSIFCAP: NV_LOCK(sc); mask = ifr->ifr_reqcap ^ ifp->if_capenable; if (mask & IFCAP_TXCSUM) { ifp->if_capenable ^= IFCAP_TXCSUM; if (ifp->if_capenable & IFCAP_TXCSUM) { ifp->if_hwassist |= sc->hn_tx_ring[0].hn_csum_assist; } else { ifp->if_hwassist &= ~sc->hn_tx_ring[0].hn_csum_assist; } } if (mask & IFCAP_RXCSUM) ifp->if_capenable ^= IFCAP_RXCSUM; if (mask & IFCAP_LRO) ifp->if_capenable ^= IFCAP_LRO; if (mask & IFCAP_TSO4) { ifp->if_capenable ^= IFCAP_TSO4; if (ifp->if_capenable & IFCAP_TSO4) ifp->if_hwassist |= CSUM_IP_TSO; else ifp->if_hwassist &= ~CSUM_IP_TSO; } if (mask & IFCAP_TSO6) { ifp->if_capenable ^= IFCAP_TSO6; if (ifp->if_capenable & IFCAP_TSO6) ifp->if_hwassist |= CSUM_IP6_TSO; else ifp->if_hwassist &= ~CSUM_IP6_TSO; } NV_UNLOCK(sc); error = 0; break; case SIOCADDMULTI: case SIOCDELMULTI: #ifdef notyet /* Fixme: Multicast mode? */ if (ifp->if_drv_flags & IFF_DRV_RUNNING) { NV_LOCK(sc); netvsc_setmulti(sc); NV_UNLOCK(sc); error = 0; } #endif error = EINVAL; break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &sc->hn_media, cmd); break; default: error = ether_ioctl(ifp, cmd, data); break; } return (error); } /* * */ static void hn_stop(hn_softc_t *sc) { struct ifnet *ifp; int ret, i; ifp = sc->hn_ifp; if (bootverbose) printf(" Closing Device ...\n"); atomic_clear_int(&ifp->if_drv_flags, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)); for (i = 0; i < sc->hn_tx_ring_inuse; ++i) sc->hn_tx_ring[i].hn_oactive = 0; if_link_state_change(ifp, LINK_STATE_DOWN); sc->hn_initdone = 0; ret = hv_rf_on_close(sc); } /* * FreeBSD transmit entry point */ static void hn_start(struct ifnet *ifp) { struct hn_softc *sc = ifp->if_softc; struct hn_tx_ring *txr = &sc->hn_tx_ring[0]; if (txr->hn_sched_tx) goto do_sched; if (mtx_trylock(&txr->hn_tx_lock)) { int sched; sched = hn_start_locked(txr, txr->hn_direct_tx_size); mtx_unlock(&txr->hn_tx_lock); if (!sched) return; } do_sched: taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); } static void hn_start_txeof(struct hn_tx_ring *txr) { struct hn_softc *sc = txr->hn_sc; struct ifnet *ifp = sc->hn_ifp; KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); if (txr->hn_sched_tx) goto do_sched; if (mtx_trylock(&txr->hn_tx_lock)) { int sched; atomic_clear_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); sched = hn_start_locked(txr, txr->hn_direct_tx_size); mtx_unlock(&txr->hn_tx_lock); if (sched) { taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); } } else { do_sched: /* * Release the OACTIVE earlier, with the hope, that * others could catch up. The task will clear the * flag again with the hn_tx_lock to avoid possible * races. */ atomic_clear_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); } } /* * */ static void hn_ifinit_locked(hn_softc_t *sc) { struct ifnet *ifp; int ret, i; ifp = sc->hn_ifp; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { return; } hv_promisc_mode = 1; ret = hv_rf_on_open(sc); if (ret != 0) { return; } else { sc->hn_initdone = 1; } atomic_clear_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); for (i = 0; i < sc->hn_tx_ring_inuse; ++i) sc->hn_tx_ring[i].hn_oactive = 0; atomic_set_int(&ifp->if_drv_flags, IFF_DRV_RUNNING); if_link_state_change(ifp, LINK_STATE_UP); } /* * */ static void hn_ifinit(void *xsc) { hn_softc_t *sc = xsc; NV_LOCK(sc); if (sc->temp_unusable) { NV_UNLOCK(sc); return; } sc->temp_unusable = TRUE; NV_UNLOCK(sc); hn_ifinit_locked(sc); NV_LOCK(sc); sc->temp_unusable = FALSE; NV_UNLOCK(sc); } #ifdef LATER /* * */ static void hn_watchdog(struct ifnet *ifp) { hn_softc_t *sc; sc = ifp->if_softc; printf("hn%d: watchdog timeout -- resetting\n", sc->hn_unit); hn_ifinit(sc); /*???*/ if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); } #endif #if __FreeBSD_version >= 1100099 static int hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; unsigned int lenlim; int error; lenlim = sc->hn_rx_ring[0].hn_lro.lro_length_lim; error = sysctl_handle_int(oidp, &lenlim, 0, req); if (error || req->newptr == NULL) return error; if (lenlim < HN_LRO_LENLIM_MIN(sc->hn_ifp) || lenlim > TCP_LRO_LENGTH_MAX) return EINVAL; NV_LOCK(sc); hn_set_lro_lenlim(sc, lenlim); NV_UNLOCK(sc); return 0; } static int hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; int ackcnt, error, i; /* * lro_ackcnt_lim is append count limit, * +1 to turn it into aggregation limit. */ ackcnt = sc->hn_rx_ring[0].hn_lro.lro_ackcnt_lim + 1; error = sysctl_handle_int(oidp, &ackcnt, 0, req); if (error || req->newptr == NULL) return error; if (ackcnt < 2 || ackcnt > (TCP_LRO_ACKCNT_MAX + 1)) return EINVAL; /* * Convert aggregation limit back to append * count limit. */ --ackcnt; NV_LOCK(sc); for (i = 0; i < sc->hn_rx_ring_inuse; ++i) sc->hn_rx_ring[i].hn_lro.lro_ackcnt_lim = ackcnt; NV_UNLOCK(sc); return 0; } #endif static int hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; int hcsum = arg2; int on, error, i; on = 0; if (sc->hn_rx_ring[0].hn_trust_hcsum & hcsum) on = 1; error = sysctl_handle_int(oidp, &on, 0, req); if (error || req->newptr == NULL) return error; NV_LOCK(sc); for (i = 0; i < sc->hn_rx_ring_inuse; ++i) { struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; if (on) rxr->hn_trust_hcsum |= hcsum; else rxr->hn_trust_hcsum &= ~hcsum; } NV_UNLOCK(sc); return 0; } static int hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; int chim_size, error; chim_size = sc->hn_tx_ring[0].hn_chim_size; error = sysctl_handle_int(oidp, &chim_size, 0, req); if (error || req->newptr == NULL) return error; if (chim_size > sc->hn_chim_szmax || chim_size <= 0) return EINVAL; hn_set_chim_size(sc, chim_size); return 0; } static int hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; int ofs = arg2, i, error; struct hn_rx_ring *rxr; u_long stat; stat = 0; for (i = 0; i < sc->hn_rx_ring_inuse; ++i) { rxr = &sc->hn_rx_ring[i]; stat += *((u_long *)((uint8_t *)rxr + ofs)); } error = sysctl_handle_long(oidp, &stat, 0, req); if (error || req->newptr == NULL) return error; /* Zero out this stat. */ for (i = 0; i < sc->hn_rx_ring_inuse; ++i) { rxr = &sc->hn_rx_ring[i]; *((u_long *)((uint8_t *)rxr + ofs)) = 0; } return 0; } static int hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; int ofs = arg2, i, error; struct hn_rx_ring *rxr; uint64_t stat; stat = 0; for (i = 0; i < sc->hn_rx_ring_inuse; ++i) { rxr = &sc->hn_rx_ring[i]; stat += *((uint64_t *)((uint8_t *)rxr + ofs)); } error = sysctl_handle_64(oidp, &stat, 0, req); if (error || req->newptr == NULL) return error; /* Zero out this stat. */ for (i = 0; i < sc->hn_rx_ring_inuse; ++i) { rxr = &sc->hn_rx_ring[i]; *((uint64_t *)((uint8_t *)rxr + ofs)) = 0; } return 0; } static int hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; int ofs = arg2, i, error; struct hn_tx_ring *txr; u_long stat; stat = 0; for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { txr = &sc->hn_tx_ring[i]; stat += *((u_long *)((uint8_t *)txr + ofs)); } error = sysctl_handle_long(oidp, &stat, 0, req); if (error || req->newptr == NULL) return error; /* Zero out this stat. */ for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { txr = &sc->hn_tx_ring[i]; *((u_long *)((uint8_t *)txr + ofs)) = 0; } return 0; } static int hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; int ofs = arg2, i, error, conf; struct hn_tx_ring *txr; txr = &sc->hn_tx_ring[0]; conf = *((int *)((uint8_t *)txr + ofs)); error = sysctl_handle_int(oidp, &conf, 0, req); if (error || req->newptr == NULL) return error; NV_LOCK(sc); for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { txr = &sc->hn_tx_ring[i]; *((int *)((uint8_t *)txr + ofs)) = conf; } NV_UNLOCK(sc); return 0; } static int hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS) { struct hn_softc *sc = arg1; char verstr[16]; snprintf(verstr, sizeof(verstr), "%u.%u", NDIS_VERSION_MAJOR(sc->hn_ndis_ver), NDIS_VERSION_MINOR(sc->hn_ndis_ver)); return sysctl_handle_string(oidp, verstr, sizeof(verstr), req); } static int hn_check_iplen(const struct mbuf *m, int hoff) { const struct ip *ip; int len, iphlen, iplen; const struct tcphdr *th; int thoff; /* TCP data offset */ len = hoff + sizeof(struct ip); /* The packet must be at least the size of an IP header. */ if (m->m_pkthdr.len < len) return IPPROTO_DONE; /* The fixed IP header must reside completely in the first mbuf. */ if (m->m_len < len) return IPPROTO_DONE; ip = mtodo(m, hoff); /* Bound check the packet's stated IP header length. */ iphlen = ip->ip_hl << 2; if (iphlen < sizeof(struct ip)) /* minimum header length */ return IPPROTO_DONE; /* The full IP header must reside completely in the one mbuf. */ if (m->m_len < hoff + iphlen) return IPPROTO_DONE; iplen = ntohs(ip->ip_len); /* * Check that the amount of data in the buffers is as * at least much as the IP header would have us expect. */ if (m->m_pkthdr.len < hoff + iplen) return IPPROTO_DONE; /* * Ignore IP fragments. */ if (ntohs(ip->ip_off) & (IP_OFFMASK | IP_MF)) return IPPROTO_DONE; /* * The TCP/IP or UDP/IP header must be entirely contained within * the first fragment of a packet. */ switch (ip->ip_p) { case IPPROTO_TCP: if (iplen < iphlen + sizeof(struct tcphdr)) return IPPROTO_DONE; if (m->m_len < hoff + iphlen + sizeof(struct tcphdr)) return IPPROTO_DONE; th = (const struct tcphdr *)((const uint8_t *)ip + iphlen); thoff = th->th_off << 2; if (thoff < sizeof(struct tcphdr) || thoff + iphlen > iplen) return IPPROTO_DONE; if (m->m_len < hoff + iphlen + thoff) return IPPROTO_DONE; break; case IPPROTO_UDP: if (iplen < iphlen + sizeof(struct udphdr)) return IPPROTO_DONE; if (m->m_len < hoff + iphlen + sizeof(struct udphdr)) return IPPROTO_DONE; break; default: if (iplen < iphlen) return IPPROTO_DONE; break; } return ip->ip_p; } static int hn_create_rx_data(struct hn_softc *sc, int ring_cnt) { struct sysctl_oid_list *child; struct sysctl_ctx_list *ctx; device_t dev = sc->hn_dev; #if defined(INET) || defined(INET6) #if __FreeBSD_version >= 1100095 int lroent_cnt; #endif #endif int i; /* * Create RXBUF for reception. * * NOTE: * - It is shared by all channels. * - A large enough buffer is allocated, certain version of NVSes * may further limit the usable space. */ sc->hn_rxbuf = hyperv_dmamem_alloc(bus_get_dma_tag(dev), PAGE_SIZE, 0, NETVSC_RECEIVE_BUFFER_SIZE, &sc->hn_rxbuf_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); if (sc->hn_rxbuf == NULL) { device_printf(sc->hn_dev, "allocate rxbuf failed\n"); return (ENOMEM); } sc->hn_rx_ring_cnt = ring_cnt; sc->hn_rx_ring_inuse = sc->hn_rx_ring_cnt; sc->hn_rx_ring = malloc(sizeof(struct hn_rx_ring) * sc->hn_rx_ring_cnt, M_NETVSC, M_WAITOK | M_ZERO); #if defined(INET) || defined(INET6) #if __FreeBSD_version >= 1100095 lroent_cnt = hn_lro_entry_count; if (lroent_cnt < TCP_LRO_ENTRIES) lroent_cnt = TCP_LRO_ENTRIES; device_printf(dev, "LRO: entry count %d\n", lroent_cnt); #endif #endif /* INET || INET6 */ ctx = device_get_sysctl_ctx(dev); child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); /* Create dev.hn.UNIT.rx sysctl tree */ sc->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "rx", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; if (hn_trust_hosttcp) rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_TCP; if (hn_trust_hostudp) rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_UDP; if (hn_trust_hostip) rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_IP; rxr->hn_ifp = sc->hn_ifp; if (i < sc->hn_tx_ring_cnt) rxr->hn_txr = &sc->hn_tx_ring[i]; rxr->hn_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK); rxr->hn_rx_idx = i; rxr->hn_rxbuf = sc->hn_rxbuf; /* * Initialize LRO. */ #if defined(INET) || defined(INET6) #if __FreeBSD_version >= 1100095 tcp_lro_init_args(&rxr->hn_lro, sc->hn_ifp, lroent_cnt, hn_lro_mbufq_depth); #else tcp_lro_init(&rxr->hn_lro); rxr->hn_lro.ifp = sc->hn_ifp; #endif #if __FreeBSD_version >= 1100099 rxr->hn_lro.lro_length_lim = HN_LRO_LENLIM_DEF; rxr->hn_lro.lro_ackcnt_lim = HN_LRO_ACKCNT_DEF; #endif #endif /* INET || INET6 */ if (sc->hn_rx_sysctl_tree != NULL) { char name[16]; /* * Create per RX ring sysctl tree: * dev.hn.UNIT.rx.RINGID */ snprintf(name, sizeof(name), "%d", i); rxr->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(sc->hn_rx_sysctl_tree), OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); if (rxr->hn_rx_sysctl_tree != NULL) { SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), OID_AUTO, "packets", CTLFLAG_RW, &rxr->hn_pkts, "# of packets received"); SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), OID_AUTO, "rss_pkts", CTLFLAG_RW, &rxr->hn_rss_pkts, "# of packets w/ RSS info received"); } } } SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_queued", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_lro.lro_queued), hn_rx_stat_u64_sysctl, "LU", "LRO queued"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_flushed", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_lro.lro_flushed), hn_rx_stat_u64_sysctl, "LU", "LRO flushed"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_tried", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_lro_tried), hn_rx_stat_ulong_sysctl, "LU", "# of LRO tries"); #if __FreeBSD_version >= 1100099 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_length_lim", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, hn_lro_lenlim_sysctl, "IU", "Max # of data bytes to be aggregated by LRO"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_ackcnt_lim", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, hn_lro_ackcnt_sysctl, "I", "Max # of ACKs to be aggregated by LRO"); #endif SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hosttcp", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_TCP, hn_trust_hcsum_sysctl, "I", "Trust tcp segement verification on host side, " "when csum info is missing"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostudp", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_UDP, hn_trust_hcsum_sysctl, "I", "Trust udp datagram verification on host side, " "when csum info is missing"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostip", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_IP, hn_trust_hcsum_sysctl, "I", "Trust ip packet verification on host side, " "when csum info is missing"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_ip", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_csum_ip), hn_rx_stat_ulong_sysctl, "LU", "RXCSUM IP"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_tcp", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_csum_tcp), hn_rx_stat_ulong_sysctl, "LU", "RXCSUM TCP"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_udp", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_csum_udp), hn_rx_stat_ulong_sysctl, "LU", "RXCSUM UDP"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_trusted", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_csum_trusted), hn_rx_stat_ulong_sysctl, "LU", "# of packets that we trust host's csum verification"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "small_pkts", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_rx_ring, hn_small_pkts), hn_rx_stat_ulong_sysctl, "LU", "# of small packets received"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_cnt", CTLFLAG_RD, &sc->hn_rx_ring_cnt, 0, "# created RX rings"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_inuse", CTLFLAG_RD, &sc->hn_rx_ring_inuse, 0, "# used RX rings"); return (0); } static void hn_destroy_rx_data(struct hn_softc *sc) { int i; if (sc->hn_rxbuf != NULL) { hyperv_dmamem_free(&sc->hn_rxbuf_dma, sc->hn_rxbuf); sc->hn_rxbuf = NULL; } if (sc->hn_rx_ring_cnt == 0) return; for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; #if defined(INET) || defined(INET6) tcp_lro_free(&rxr->hn_lro); #endif free(rxr->hn_rdbuf, M_NETVSC); } free(sc->hn_rx_ring, M_NETVSC); sc->hn_rx_ring = NULL; sc->hn_rx_ring_cnt = 0; sc->hn_rx_ring_inuse = 0; } static int hn_create_tx_ring(struct hn_softc *sc, int id) { struct hn_tx_ring *txr = &sc->hn_tx_ring[id]; device_t dev = sc->hn_dev; bus_dma_tag_t parent_dtag; int error, i; uint32_t version; txr->hn_sc = sc; txr->hn_tx_idx = id; #ifndef HN_USE_TXDESC_BUFRING mtx_init(&txr->hn_txlist_spin, "hn txlist", NULL, MTX_SPIN); #endif mtx_init(&txr->hn_tx_lock, "hn tx", NULL, MTX_DEF); txr->hn_txdesc_cnt = HN_TX_DESC_CNT; txr->hn_txdesc = malloc(sizeof(struct hn_txdesc) * txr->hn_txdesc_cnt, M_NETVSC, M_WAITOK | M_ZERO); #ifndef HN_USE_TXDESC_BUFRING SLIST_INIT(&txr->hn_txlist); #else txr->hn_txdesc_br = buf_ring_alloc(txr->hn_txdesc_cnt, M_NETVSC, M_WAITOK, &txr->hn_tx_lock); #endif txr->hn_tx_taskq = sc->hn_tx_taskq; if (hn_use_if_start) { txr->hn_txeof = hn_start_txeof; TASK_INIT(&txr->hn_tx_task, 0, hn_start_taskfunc, txr); TASK_INIT(&txr->hn_txeof_task, 0, hn_start_txeof_taskfunc, txr); } else { int br_depth; txr->hn_txeof = hn_xmit_txeof; TASK_INIT(&txr->hn_tx_task, 0, hn_xmit_taskfunc, txr); TASK_INIT(&txr->hn_txeof_task, 0, hn_xmit_txeof_taskfunc, txr); br_depth = hn_get_txswq_depth(txr); txr->hn_mbuf_br = buf_ring_alloc(br_depth, M_NETVSC, M_WAITOK, &txr->hn_tx_lock); } txr->hn_direct_tx_size = hn_direct_tx_size; version = VMBUS_GET_VERSION(device_get_parent(dev), dev); if (version >= VMBUS_VERSION_WIN8_1) { txr->hn_csum_assist = HN_CSUM_ASSIST; } else { txr->hn_csum_assist = HN_CSUM_ASSIST_WIN8; if (id == 0) { device_printf(dev, "bus version %u.%u, " "no UDP checksum offloading\n", VMBUS_VERSION_MAJOR(version), VMBUS_VERSION_MINOR(version)); } } /* * Always schedule transmission instead of trying to do direct * transmission. This one gives the best performance so far. */ txr->hn_sched_tx = 1; parent_dtag = bus_get_dma_tag(dev); /* DMA tag for RNDIS messages. */ error = bus_dma_tag_create(parent_dtag, /* parent */ HN_RNDIS_MSG_ALIGN, /* alignment */ HN_RNDIS_MSG_BOUNDARY, /* boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ HN_RNDIS_MSG_LEN, /* maxsize */ 1, /* nsegments */ HN_RNDIS_MSG_LEN, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &txr->hn_tx_rndis_dtag); if (error) { device_printf(dev, "failed to create rndis dmatag\n"); return error; } /* DMA tag for data. */ error = bus_dma_tag_create(parent_dtag, /* parent */ 1, /* alignment */ HN_TX_DATA_BOUNDARY, /* boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ HN_TX_DATA_MAXSIZE, /* maxsize */ HN_TX_DATA_SEGCNT_MAX, /* nsegments */ HN_TX_DATA_SEGSIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &txr->hn_tx_data_dtag); if (error) { device_printf(dev, "failed to create data dmatag\n"); return error; } for (i = 0; i < txr->hn_txdesc_cnt; ++i) { struct hn_txdesc *txd = &txr->hn_txdesc[i]; txd->txr = txr; /* * Allocate and load RNDIS messages. */ error = bus_dmamem_alloc(txr->hn_tx_rndis_dtag, (void **)&txd->rndis_msg, BUS_DMA_WAITOK | BUS_DMA_COHERENT, &txd->rndis_msg_dmap); if (error) { device_printf(dev, "failed to allocate rndis_msg, %d\n", i); return error; } error = bus_dmamap_load(txr->hn_tx_rndis_dtag, txd->rndis_msg_dmap, txd->rndis_msg, HN_RNDIS_MSG_LEN, hyperv_dma_map_paddr, &txd->rndis_msg_paddr, BUS_DMA_NOWAIT); if (error) { device_printf(dev, "failed to load rndis_msg, %d\n", i); bus_dmamem_free(txr->hn_tx_rndis_dtag, txd->rndis_msg, txd->rndis_msg_dmap); return error; } /* DMA map for TX data. */ error = bus_dmamap_create(txr->hn_tx_data_dtag, 0, &txd->data_dmap); if (error) { device_printf(dev, "failed to allocate tx data dmamap\n"); bus_dmamap_unload(txr->hn_tx_rndis_dtag, txd->rndis_msg_dmap); bus_dmamem_free(txr->hn_tx_rndis_dtag, txd->rndis_msg, txd->rndis_msg_dmap); return error; } /* All set, put it to list */ txd->flags |= HN_TXD_FLAG_ONLIST; #ifndef HN_USE_TXDESC_BUFRING SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); #else buf_ring_enqueue(txr->hn_txdesc_br, txd); #endif } txr->hn_txdesc_avail = txr->hn_txdesc_cnt; if (sc->hn_tx_sysctl_tree != NULL) { struct sysctl_oid_list *child; struct sysctl_ctx_list *ctx; char name[16]; /* * Create per TX ring sysctl tree: * dev.hn.UNIT.tx.RINGID */ ctx = device_get_sysctl_ctx(dev); child = SYSCTL_CHILDREN(sc->hn_tx_sysctl_tree); snprintf(name, sizeof(name), "%d", id); txr->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); if (txr->hn_tx_sysctl_tree != NULL) { child = SYSCTL_CHILDREN(txr->hn_tx_sysctl_tree); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_avail", CTLFLAG_RD, &txr->hn_txdesc_avail, 0, "# of available TX descs"); if (!hn_use_if_start) { SYSCTL_ADD_INT(ctx, child, OID_AUTO, "oactive", CTLFLAG_RD, &txr->hn_oactive, 0, "over active"); } SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "packets", CTLFLAG_RW, &txr->hn_pkts, "# of packets transmitted"); } } return 0; } static void hn_txdesc_dmamap_destroy(struct hn_txdesc *txd) { struct hn_tx_ring *txr = txd->txr; KASSERT(txd->m == NULL, ("still has mbuf installed")); KASSERT((txd->flags & HN_TXD_FLAG_DMAMAP) == 0, ("still dma mapped")); bus_dmamap_unload(txr->hn_tx_rndis_dtag, txd->rndis_msg_dmap); bus_dmamem_free(txr->hn_tx_rndis_dtag, txd->rndis_msg, txd->rndis_msg_dmap); bus_dmamap_destroy(txr->hn_tx_data_dtag, txd->data_dmap); } static void hn_destroy_tx_ring(struct hn_tx_ring *txr) { struct hn_txdesc *txd; if (txr->hn_txdesc == NULL) return; #ifndef HN_USE_TXDESC_BUFRING while ((txd = SLIST_FIRST(&txr->hn_txlist)) != NULL) { SLIST_REMOVE_HEAD(&txr->hn_txlist, link); hn_txdesc_dmamap_destroy(txd); } #else mtx_lock(&txr->hn_tx_lock); while ((txd = buf_ring_dequeue_sc(txr->hn_txdesc_br)) != NULL) hn_txdesc_dmamap_destroy(txd); mtx_unlock(&txr->hn_tx_lock); #endif if (txr->hn_tx_data_dtag != NULL) bus_dma_tag_destroy(txr->hn_tx_data_dtag); if (txr->hn_tx_rndis_dtag != NULL) bus_dma_tag_destroy(txr->hn_tx_rndis_dtag); #ifdef HN_USE_TXDESC_BUFRING buf_ring_free(txr->hn_txdesc_br, M_NETVSC); #endif free(txr->hn_txdesc, M_NETVSC); txr->hn_txdesc = NULL; if (txr->hn_mbuf_br != NULL) buf_ring_free(txr->hn_mbuf_br, M_NETVSC); #ifndef HN_USE_TXDESC_BUFRING mtx_destroy(&txr->hn_txlist_spin); #endif mtx_destroy(&txr->hn_tx_lock); } static int hn_create_tx_data(struct hn_softc *sc, int ring_cnt) { struct sysctl_oid_list *child; struct sysctl_ctx_list *ctx; int i; /* * Create TXBUF for chimney sending. * * NOTE: It is shared by all channels. */ sc->hn_chim = hyperv_dmamem_alloc(bus_get_dma_tag(sc->hn_dev), PAGE_SIZE, 0, NETVSC_SEND_BUFFER_SIZE, &sc->hn_chim_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); if (sc->hn_chim == NULL) { device_printf(sc->hn_dev, "allocate txbuf failed\n"); return (ENOMEM); } sc->hn_tx_ring_cnt = ring_cnt; sc->hn_tx_ring_inuse = sc->hn_tx_ring_cnt; sc->hn_tx_ring = malloc(sizeof(struct hn_tx_ring) * sc->hn_tx_ring_cnt, M_NETVSC, M_WAITOK | M_ZERO); ctx = device_get_sysctl_ctx(sc->hn_dev); child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->hn_dev)); /* Create dev.hn.UNIT.tx sysctl tree */ sc->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "tx", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { int error; error = hn_create_tx_ring(sc, i); if (error) return error; } SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "no_txdescs", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_no_txdescs), hn_tx_stat_ulong_sysctl, "LU", "# of times short of TX descs"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "send_failed", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_send_failed), hn_tx_stat_ulong_sysctl, "LU", "# of hyper-v sending failure"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txdma_failed", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_txdma_failed), hn_tx_stat_ulong_sysctl, "LU", "# of TX DMA failure"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_collapsed", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_tx_collapsed), hn_tx_stat_ulong_sysctl, "LU", "# of TX mbuf collapsed"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_tx_chimney), hn_tx_stat_ulong_sysctl, "LU", "# of chimney send"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_tried", CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_tx_chimney_tried), hn_tx_stat_ulong_sysctl, "LU", "# of chimney send tries"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_cnt", CTLFLAG_RD, &sc->hn_tx_ring[0].hn_txdesc_cnt, 0, "# of total TX descs"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_chimney_max", CTLFLAG_RD, &sc->hn_chim_szmax, 0, "Chimney send packet size upper boundary"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_size", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, hn_chim_size_sysctl, "I", "Chimney send packet size limit"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "direct_tx_size", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_direct_tx_size), hn_tx_conf_int_sysctl, "I", "Size of the packet for direct transmission"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "sched_tx", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, __offsetof(struct hn_tx_ring, hn_sched_tx), hn_tx_conf_int_sysctl, "I", "Always schedule transmission " "instead of doing direct transmission"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_cnt", CTLFLAG_RD, &sc->hn_tx_ring_cnt, 0, "# created TX rings"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_inuse", CTLFLAG_RD, &sc->hn_tx_ring_inuse, 0, "# used TX rings"); return 0; } static void hn_set_chim_size(struct hn_softc *sc, int chim_size) { int i; NV_LOCK(sc); for (i = 0; i < sc->hn_tx_ring_inuse; ++i) sc->hn_tx_ring[i].hn_chim_size = chim_size; NV_UNLOCK(sc); } static void hn_destroy_tx_data(struct hn_softc *sc) { int i; if (sc->hn_chim != NULL) { hyperv_dmamem_free(&sc->hn_chim_dma, sc->hn_chim); sc->hn_chim = NULL; } if (sc->hn_tx_ring_cnt == 0) return; for (i = 0; i < sc->hn_tx_ring_cnt; ++i) hn_destroy_tx_ring(&sc->hn_tx_ring[i]); free(sc->hn_tx_ring, M_NETVSC); sc->hn_tx_ring = NULL; sc->hn_tx_ring_cnt = 0; sc->hn_tx_ring_inuse = 0; } static void hn_start_taskfunc(void *xtxr, int pending __unused) { struct hn_tx_ring *txr = xtxr; mtx_lock(&txr->hn_tx_lock); hn_start_locked(txr, 0); mtx_unlock(&txr->hn_tx_lock); } static void hn_start_txeof_taskfunc(void *xtxr, int pending __unused) { struct hn_tx_ring *txr = xtxr; mtx_lock(&txr->hn_tx_lock); atomic_clear_int(&txr->hn_sc->hn_ifp->if_drv_flags, IFF_DRV_OACTIVE); hn_start_locked(txr, 0); mtx_unlock(&txr->hn_tx_lock); } static void hn_stop_tx_tasks(struct hn_softc *sc) { int i; for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; taskqueue_drain(txr->hn_tx_taskq, &txr->hn_tx_task); taskqueue_drain(txr->hn_tx_taskq, &txr->hn_txeof_task); } } static int hn_xmit(struct hn_tx_ring *txr, int len) { struct hn_softc *sc = txr->hn_sc; struct ifnet *ifp = sc->hn_ifp; struct mbuf *m_head; mtx_assert(&txr->hn_tx_lock, MA_OWNED); KASSERT(hn_use_if_start == 0, ("hn_xmit is called, when if_start is enabled")); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || txr->hn_oactive) return 0; while ((m_head = drbr_peek(ifp, txr->hn_mbuf_br)) != NULL) { struct hn_txdesc *txd; int error; if (len > 0 && m_head->m_pkthdr.len > len) { /* * This sending could be time consuming; let callers * dispatch this packet sending (and sending of any * following up packets) to tx taskqueue. */ drbr_putback(ifp, txr->hn_mbuf_br, m_head); return 1; } txd = hn_txdesc_get(txr); if (txd == NULL) { txr->hn_no_txdescs++; drbr_putback(ifp, txr->hn_mbuf_br, m_head); txr->hn_oactive = 1; break; } error = hn_encap(txr, txd, &m_head); if (error) { /* Both txd and m_head are freed; discard */ drbr_advance(ifp, txr->hn_mbuf_br); continue; } error = hn_send_pkt(ifp, txr, txd); if (__predict_false(error)) { /* txd is freed, but m_head is not */ drbr_putback(ifp, txr->hn_mbuf_br, m_head); txr->hn_oactive = 1; break; } /* Sent */ drbr_advance(ifp, txr->hn_mbuf_br); } return 0; } static int hn_transmit(struct ifnet *ifp, struct mbuf *m) { struct hn_softc *sc = ifp->if_softc; struct hn_tx_ring *txr; int error, idx = 0; /* * Select the TX ring based on flowid */ if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) idx = m->m_pkthdr.flowid % sc->hn_tx_ring_inuse; txr = &sc->hn_tx_ring[idx]; error = drbr_enqueue(ifp, txr->hn_mbuf_br, m); if (error) { if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); return error; } if (txr->hn_oactive) return 0; if (txr->hn_sched_tx) goto do_sched; if (mtx_trylock(&txr->hn_tx_lock)) { int sched; sched = hn_xmit(txr, txr->hn_direct_tx_size); mtx_unlock(&txr->hn_tx_lock); if (!sched) return 0; } do_sched: taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); return 0; } static void hn_xmit_qflush(struct ifnet *ifp) { struct hn_softc *sc = ifp->if_softc; int i; for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; struct mbuf *m; mtx_lock(&txr->hn_tx_lock); while ((m = buf_ring_dequeue_sc(txr->hn_mbuf_br)) != NULL) m_freem(m); mtx_unlock(&txr->hn_tx_lock); } if_qflush(ifp); } static void hn_xmit_txeof(struct hn_tx_ring *txr) { if (txr->hn_sched_tx) goto do_sched; if (mtx_trylock(&txr->hn_tx_lock)) { int sched; txr->hn_oactive = 0; sched = hn_xmit(txr, txr->hn_direct_tx_size); mtx_unlock(&txr->hn_tx_lock); if (sched) { taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); } } else { do_sched: /* * Release the oactive earlier, with the hope, that * others could catch up. The task will clear the * oactive again with the hn_tx_lock to avoid possible * races. */ txr->hn_oactive = 0; taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); } } static void hn_xmit_taskfunc(void *xtxr, int pending __unused) { struct hn_tx_ring *txr = xtxr; mtx_lock(&txr->hn_tx_lock); hn_xmit(txr, 0); mtx_unlock(&txr->hn_tx_lock); } static void hn_xmit_txeof_taskfunc(void *xtxr, int pending __unused) { struct hn_tx_ring *txr = xtxr; mtx_lock(&txr->hn_tx_lock); txr->hn_oactive = 0; hn_xmit(txr, 0); mtx_unlock(&txr->hn_tx_lock); } static void hn_channel_attach(struct hn_softc *sc, struct vmbus_channel *chan) { struct hn_rx_ring *rxr; int idx; idx = vmbus_chan_subidx(chan); KASSERT(idx >= 0 && idx < sc->hn_rx_ring_inuse, ("invalid channel index %d, should > 0 && < %d", idx, sc->hn_rx_ring_inuse)); rxr = &sc->hn_rx_ring[idx]; KASSERT((rxr->hn_rx_flags & HN_RX_FLAG_ATTACHED) == 0, ("RX ring %d already attached", idx)); rxr->hn_rx_flags |= HN_RX_FLAG_ATTACHED; if (bootverbose) { if_printf(sc->hn_ifp, "link RX ring %d to channel%u\n", idx, vmbus_chan_id(chan)); } if (idx < sc->hn_tx_ring_inuse) { struct hn_tx_ring *txr = &sc->hn_tx_ring[idx]; KASSERT((txr->hn_tx_flags & HN_TX_FLAG_ATTACHED) == 0, ("TX ring %d already attached", idx)); txr->hn_tx_flags |= HN_TX_FLAG_ATTACHED; txr->hn_chan = chan; if (bootverbose) { if_printf(sc->hn_ifp, "link TX ring %d to channel%u\n", idx, vmbus_chan_id(chan)); } } /* Bind channel to a proper CPU */ vmbus_chan_cpu_set(chan, (sc->hn_cpu + idx) % mp_ncpus); } static void hn_subchan_attach(struct hn_softc *sc, struct vmbus_channel *chan) { KASSERT(!vmbus_chan_is_primary(chan), ("subchannel callback on primary channel")); hn_channel_attach(sc, chan); } static void hn_subchan_setup(struct hn_softc *sc) { struct vmbus_channel **subchans; int subchan_cnt = sc->hn_rx_ring_inuse - 1; int i; /* Wait for sub-channels setup to complete. */ subchans = vmbus_subchan_get(sc->hn_prichan, subchan_cnt); /* Attach the sub-channels. */ for (i = 0; i < subchan_cnt; ++i) { struct vmbus_channel *subchan = subchans[i]; /* NOTE: Calling order is critical. */ hn_subchan_attach(sc, subchan); hv_nv_subchan_attach(subchan, &sc->hn_rx_ring[vmbus_chan_subidx(subchan)]); } /* Release the sub-channels */ vmbus_subchan_rel(subchans, subchan_cnt); if_printf(sc->hn_ifp, "%d sub-channels setup done\n", subchan_cnt); } static void hn_tx_taskq_create(void *arg __unused) { if (!hn_share_tx_taskq) return; hn_tx_taskq = taskqueue_create("hn_tx", M_WAITOK, taskqueue_thread_enqueue, &hn_tx_taskq); if (hn_bind_tx_taskq >= 0) { int cpu = hn_bind_tx_taskq; cpuset_t cpu_set; if (cpu > mp_ncpus - 1) cpu = mp_ncpus - 1; CPU_SETOF(cpu, &cpu_set); taskqueue_start_threads_cpuset(&hn_tx_taskq, 1, PI_NET, &cpu_set, "hn tx"); } else { taskqueue_start_threads(&hn_tx_taskq, 1, PI_NET, "hn tx"); } } SYSINIT(hn_txtq_create, SI_SUB_DRIVERS, SI_ORDER_FIRST, hn_tx_taskq_create, NULL); static void hn_tx_taskq_destroy(void *arg __unused) { if (hn_tx_taskq != NULL) taskqueue_free(hn_tx_taskq); } SYSUNINIT(hn_txtq_destroy, SI_SUB_DRIVERS, SI_ORDER_FIRST, hn_tx_taskq_destroy, NULL); static device_method_t netvsc_methods[] = { /* Device interface */ DEVMETHOD(device_probe, netvsc_probe), DEVMETHOD(device_attach, netvsc_attach), DEVMETHOD(device_detach, netvsc_detach), DEVMETHOD(device_shutdown, netvsc_shutdown), { 0, 0 } }; static driver_t netvsc_driver = { NETVSC_DEVNAME, netvsc_methods, sizeof(hn_softc_t) }; static devclass_t netvsc_devclass; DRIVER_MODULE(hn, vmbus, netvsc_driver, netvsc_devclass, 0, 0); MODULE_VERSION(hn, 1); MODULE_DEPEND(hn, vmbus, 1, 1, 1); Index: projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis.h =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis.h (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis.h (revision 305081) @@ -1,923 +1,910 @@ /*- * Copyright (c) 2009-2012,2016 Microsoft Corp. * Copyright (c) 2010-2012 Citrix Inc. * Copyright (c) 2012 NetApp Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __HV_RNDIS_H__ #define __HV_RNDIS_H__ #include /* * NDIS protocol version numbers */ #define NDIS_VERSION_5_0 0x00050000 #define NDIS_VERSION_5_1 0x00050001 #define NDIS_VERSION_6_0 0x00060000 #define NDIS_VERSION_6_1 0x00060001 #define NDIS_VERSION_6_30 0x0006001e #define NDIS_VERSION_MAJOR(ver) (((ver) & 0xffff0000) >> 16) #define NDIS_VERSION_MINOR(ver) ((ver) & 0xffff) /* * Object Identifiers used by NdisRequest Query/Set Information */ /* * General Objects */ #define RNDIS_OID_GEN_SUPPORTED_LIST 0x00010101 #define RNDIS_OID_GEN_HARDWARE_STATUS 0x00010102 #define RNDIS_OID_GEN_MEDIA_SUPPORTED 0x00010103 #define RNDIS_OID_GEN_MEDIA_IN_USE 0x00010104 #define RNDIS_OID_GEN_MAXIMUM_LOOKAHEAD 0x00010105 #define RNDIS_OID_GEN_MAXIMUM_FRAME_SIZE 0x00010106 #define RNDIS_OID_GEN_LINK_SPEED 0x00010107 #define RNDIS_OID_GEN_TRANSMIT_BUFFER_SPACE 0x00010108 #define RNDIS_OID_GEN_RECEIVE_BUFFER_SPACE 0x00010109 #define RNDIS_OID_GEN_TRANSMIT_BLOCK_SIZE 0x0001010A #define RNDIS_OID_GEN_RECEIVE_BLOCK_SIZE 0x0001010B #define RNDIS_OID_GEN_VENDOR_ID 0x0001010C #define RNDIS_OID_GEN_VENDOR_DESCRIPTION 0x0001010D #define RNDIS_OID_GEN_CURRENT_PACKET_FILTER 0x0001010E #define RNDIS_OID_GEN_CURRENT_LOOKAHEAD 0x0001010F #define RNDIS_OID_GEN_DRIVER_VERSION 0x00010110 #define RNDIS_OID_GEN_MAXIMUM_TOTAL_SIZE 0x00010111 #define RNDIS_OID_GEN_PROTOCOL_OPTIONS 0x00010112 #define RNDIS_OID_GEN_MAC_OPTIONS 0x00010113 #define RNDIS_OID_GEN_MEDIA_CONNECT_STATUS 0x00010114 #define RNDIS_OID_GEN_MAXIMUM_SEND_PACKETS 0x00010115 #define RNDIS_OID_GEN_VENDOR_DRIVER_VERSION 0x00010116 #define RNDIS_OID_GEN_NETWORK_LAYER_ADDRESSES 0x00010118 #define RNDIS_OID_GEN_TRANSPORT_HEADER_OFFSET 0x00010119 #define RNDIS_OID_GEN_MACHINE_NAME 0x0001021A #define RNDIS_OID_GEN_RNDIS_CONFIG_PARAMETER 0x0001021B /* * For receive side scale */ /* Query only */ #define RNDIS_OID_GEN_RSS_CAPABILITIES 0x00010203 /* Query and set */ #define RNDIS_OID_GEN_RSS_PARAMETERS 0x00010204 #define RNDIS_OID_GEN_XMIT_OK 0x00020101 #define RNDIS_OID_GEN_RCV_OK 0x00020102 #define RNDIS_OID_GEN_XMIT_ERROR 0x00020103 #define RNDIS_OID_GEN_RCV_ERROR 0x00020104 #define RNDIS_OID_GEN_RCV_NO_BUFFER 0x00020105 #define RNDIS_OID_GEN_DIRECTED_BYTES_XMIT 0x00020201 #define RNDIS_OID_GEN_DIRECTED_FRAMES_XMIT 0x00020202 #define RNDIS_OID_GEN_MULTICAST_BYTES_XMIT 0x00020203 #define RNDIS_OID_GEN_MULTICAST_FRAMES_XMIT 0x00020204 #define RNDIS_OID_GEN_BROADCAST_BYTES_XMIT 0x00020205 #define RNDIS_OID_GEN_BROADCAST_FRAMES_XMIT 0x00020206 #define RNDIS_OID_GEN_DIRECTED_BYTES_RCV 0x00020207 #define RNDIS_OID_GEN_DIRECTED_FRAMES_RCV 0x00020208 #define RNDIS_OID_GEN_MULTICAST_BYTES_RCV 0x00020209 #define RNDIS_OID_GEN_MULTICAST_FRAMES_RCV 0x0002020A #define RNDIS_OID_GEN_BROADCAST_BYTES_RCV 0x0002020B #define RNDIS_OID_GEN_BROADCAST_FRAMES_RCV 0x0002020C #define RNDIS_OID_GEN_RCV_CRC_ERROR 0x0002020D #define RNDIS_OID_GEN_TRANSMIT_QUEUE_LENGTH 0x0002020E #define RNDIS_OID_GEN_GET_TIME_CAPS 0x0002020F #define RNDIS_OID_GEN_GET_NETCARD_TIME 0x00020210 /* * These are connection-oriented general OIDs. * These replace the above OIDs for connection-oriented media. */ #define RNDIS_OID_GEN_CO_SUPPORTED_LIST 0x00010101 #define RNDIS_OID_GEN_CO_HARDWARE_STATUS 0x00010102 #define RNDIS_OID_GEN_CO_MEDIA_SUPPORTED 0x00010103 #define RNDIS_OID_GEN_CO_MEDIA_IN_USE 0x00010104 #define RNDIS_OID_GEN_CO_LINK_SPEED 0x00010105 #define RNDIS_OID_GEN_CO_VENDOR_ID 0x00010106 #define RNDIS_OID_GEN_CO_VENDOR_DESCRIPTION 0x00010107 #define RNDIS_OID_GEN_CO_DRIVER_VERSION 0x00010108 #define RNDIS_OID_GEN_CO_PROTOCOL_OPTIONS 0x00010109 #define RNDIS_OID_GEN_CO_MAC_OPTIONS 0x0001010A #define RNDIS_OID_GEN_CO_MEDIA_CONNECT_STATUS 0x0001010B #define RNDIS_OID_GEN_CO_VENDOR_DRIVER_VERSION 0x0001010C #define RNDIS_OID_GEN_CO_MINIMUM_LINK_SPEED 0x0001010D #define RNDIS_OID_GEN_CO_GET_TIME_CAPS 0x00010201 #define RNDIS_OID_GEN_CO_GET_NETCARD_TIME 0x00010202 /* * These are connection-oriented statistics OIDs. */ #define RNDIS_OID_GEN_CO_XMIT_PDUS_OK 0x00020101 #define RNDIS_OID_GEN_CO_RCV_PDUS_OK 0x00020102 #define RNDIS_OID_GEN_CO_XMIT_PDUS_ERROR 0x00020103 #define RNDIS_OID_GEN_CO_RCV_PDUS_ERROR 0x00020104 #define RNDIS_OID_GEN_CO_RCV_PDUS_NO_BUFFER 0x00020105 #define RNDIS_OID_GEN_CO_RCV_CRC_ERROR 0x00020201 #define RNDIS_OID_GEN_CO_TRANSMIT_QUEUE_LENGTH 0x00020202 #define RNDIS_OID_GEN_CO_BYTES_XMIT 0x00020203 #define RNDIS_OID_GEN_CO_BYTES_RCV 0x00020204 #define RNDIS_OID_GEN_CO_BYTES_XMIT_OUTSTANDING 0x00020205 #define RNDIS_OID_GEN_CO_NETCARD_LOAD 0x00020206 /* * These are objects for Connection-oriented media call-managers. */ #define RNDIS_OID_CO_ADD_PVC 0xFF000001 #define RNDIS_OID_CO_DELETE_PVC 0xFF000002 #define RNDIS_OID_CO_GET_CALL_INFORMATION 0xFF000003 #define RNDIS_OID_CO_ADD_ADDRESS 0xFF000004 #define RNDIS_OID_CO_DELETE_ADDRESS 0xFF000005 #define RNDIS_OID_CO_GET_ADDRESSES 0xFF000006 #define RNDIS_OID_CO_ADDRESS_CHANGE 0xFF000007 #define RNDIS_OID_CO_SIGNALING_ENABLED 0xFF000008 #define RNDIS_OID_CO_SIGNALING_DISABLED 0xFF000009 /* * 802.3 Objects (Ethernet) */ #define RNDIS_OID_802_3_PERMANENT_ADDRESS 0x01010101 #define RNDIS_OID_802_3_CURRENT_ADDRESS 0x01010102 #define RNDIS_OID_802_3_MULTICAST_LIST 0x01010103 #define RNDIS_OID_802_3_MAXIMUM_LIST_SIZE 0x01010104 #define RNDIS_OID_802_3_MAC_OPTIONS 0x01010105 /* * */ #define NDIS_802_3_MAC_OPTION_PRIORITY 0x00000001 #define RNDIS_OID_802_3_RCV_ERROR_ALIGNMENT 0x01020101 #define RNDIS_OID_802_3_XMIT_ONE_COLLISION 0x01020102 #define RNDIS_OID_802_3_XMIT_MORE_COLLISIONS 0x01020103 #define RNDIS_OID_802_3_XMIT_DEFERRED 0x01020201 #define RNDIS_OID_802_3_XMIT_MAX_COLLISIONS 0x01020202 #define RNDIS_OID_802_3_RCV_OVERRUN 0x01020203 #define RNDIS_OID_802_3_XMIT_UNDERRUN 0x01020204 #define RNDIS_OID_802_3_XMIT_HEARTBEAT_FAILURE 0x01020205 #define RNDIS_OID_802_3_XMIT_TIMES_CRS_LOST 0x01020206 #define RNDIS_OID_802_3_XMIT_LATE_COLLISIONS 0x01020207 /* * RNDIS MP custom OID for test */ #define OID_RNDISMP_GET_RECEIVE_BUFFERS 0xFFA0C90D // Query only /* * Remote NDIS offload parameters */ #define RNDIS_OBJECT_TYPE_DEFAULT 0x80 #define RNDIS_OFFLOAD_PARAMETERS_REVISION_3 3 #define RNDIS_OFFLOAD_PARAMETERS_NO_CHANGE 0 #define RNDIS_OFFLOAD_PARAMETERS_LSOV2_DISABLED 1 #define RNDIS_OFFLOAD_PARAMETERS_LSOV2_ENABLED 2 #define RNDIS_OFFLOAD_PARAMETERS_LSOV1_ENABLED 2 #define RNDIS_OFFLOAD_PARAMETERS_RSC_DISABLED 1 #define RNDIS_OFFLOAD_PARAMETERS_RSC_ENABLED 2 #define RNDIS_OFFLOAD_PARAMETERS_TX_RX_DISABLED 1 #define RNDIS_OFFLOAD_PARAMETERS_TX_ENABLED_RX_DISABLED 2 #define RNDIS_OFFLOAD_PARAMETERS_RX_ENABLED_TX_DISABLED 3 #define RNDIS_OFFLOAD_PARAMETERS_TX_RX_ENABLED 4 #define RNDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE 1 #define RNDIS_TCP_LARGE_SEND_OFFLOAD_IPV4 0 #define RNDIS_TCP_LARGE_SEND_OFFLOAD_IPV6 1 #define RNDIS_OID_TCP_OFFLOAD_CURRENT_CONFIG 0xFC01020B /* query only */ #define RNDIS_OID_TCP_OFFLOAD_PARAMETERS 0xFC01020C /* set only */ #define RNDIS_OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES 0xFC01020D/* query only */ #define RNDIS_OID_TCP_CONNECTION_OFFLOAD_CURRENT_CONFIG 0xFC01020E /* query only */ #define RNDIS_OID_TCP_CONNECTION_OFFLOAD_HARDWARE_CAPABILITIES 0xFC01020F /* query */ #define RNDIS_OID_OFFLOAD_ENCAPSULATION 0x0101010A /* set/query */ /* * NdisInitialize message */ typedef struct rndis_initialize_request_ { /* RNDIS request ID */ uint32_t request_id; uint32_t major_version; uint32_t minor_version; uint32_t max_xfer_size; } rndis_initialize_request; /* * Response to NdisInitialize */ typedef struct rndis_initialize_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS status */ uint32_t status; uint32_t major_version; uint32_t minor_version; uint32_t device_flags; /* RNDIS medium */ uint32_t medium; uint32_t max_pkts_per_msg; uint32_t max_xfer_size; uint32_t pkt_align_factor; uint32_t af_list_offset; uint32_t af_list_size; } rndis_initialize_complete; /* * Call manager devices only: Information about an address family * supported by the device is appended to the response to NdisInitialize. */ typedef struct rndis_co_address_family_ { /* RNDIS AF */ uint32_t address_family; uint32_t major_version; uint32_t minor_version; } rndis_co_address_family; /* * NdisHalt message */ typedef struct rndis_halt_request_ { /* RNDIS request ID */ uint32_t request_id; } rndis_halt_request; /* * NdisQueryRequest message */ typedef struct rndis_query_request_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS OID */ uint32_t oid; uint32_t info_buffer_length; uint32_t info_buffer_offset; /* RNDIS handle */ uint32_t device_vc_handle; } rndis_query_request; /* * Response to NdisQueryRequest */ typedef struct rndis_query_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS status */ uint32_t status; uint32_t info_buffer_length; uint32_t info_buffer_offset; } rndis_query_complete; /* * NdisSetRequest message */ typedef struct rndis_set_request_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS OID */ uint32_t oid; uint32_t info_buffer_length; uint32_t info_buffer_offset; /* RNDIS handle */ uint32_t device_vc_handle; } rndis_set_request; /* * Response to NdisSetRequest */ typedef struct rndis_set_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS status */ uint32_t status; } rndis_set_complete; /* * NdisReset message */ typedef struct rndis_reset_request_ { uint32_t reserved; } rndis_reset_request; /* * Response to NdisReset */ typedef struct rndis_reset_complete_ { /* RNDIS status */ uint32_t status; uint32_t addressing_reset; } rndis_reset_complete; /* * NdisMIndicateStatus message */ typedef struct rndis_indicate_status_ { /* RNDIS status */ uint32_t status; uint32_t status_buf_length; uint32_t status_buf_offset; } rndis_indicate_status; /* * Diagnostic information passed as the status buffer in * rndis_indicate_status messages signifying error conditions. */ typedef struct rndis_diagnostic_info_ { /* RNDIS status */ uint32_t diag_status; uint32_t error_offset; } rndis_diagnostic_info; /* * NdisKeepAlive message */ typedef struct rndis_keepalive_request_ { /* RNDIS request ID */ uint32_t request_id; } rndis_keepalive_request; /* * Response to NdisKeepAlive */ typedef struct rndis_keepalive_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS status */ uint32_t status; } rndis_keepalive_complete; /* * Data message. All offset fields contain byte offsets from the beginning * of the rndis_packet structure. All length fields are in bytes. * VcHandle is set to 0 for connectionless data, otherwise it * contains the VC handle. */ typedef struct rndis_packet_ { uint32_t data_offset; uint32_t data_length; uint32_t oob_data_offset; uint32_t oob_data_length; uint32_t num_oob_data_elements; uint32_t per_pkt_info_offset; uint32_t per_pkt_info_length; /* RNDIS handle */ uint32_t vc_handle; uint32_t reserved; } rndis_packet; typedef struct rndis_packet_ex_ { uint32_t data_offset; uint32_t data_length; uint32_t oob_data_offset; uint32_t oob_data_length; uint32_t num_oob_data_elements; uint32_t per_pkt_info_offset; uint32_t per_pkt_info_length; /* RNDIS handle */ uint32_t vc_handle; uint32_t reserved; uint64_t data_buf_id; uint32_t data_buf_offset; uint64_t next_header_buf_id; uint32_t next_header_byte_offset; uint32_t next_header_byte_count; } rndis_packet_ex; /* * Optional Out of Band data associated with a Data message. */ typedef struct rndis_oobd_ { uint32_t size; /* RNDIS class ID */ uint32_t type; uint32_t class_info_offset; } rndis_oobd; /* * Packet extension field contents associated with a Data message. */ typedef struct rndis_per_packet_info_ { uint32_t size; uint32_t type; uint32_t per_packet_info_offset; } rndis_per_packet_info; typedef enum ndis_per_pkt_infotype_ { tcpip_chksum_info, ipsec_info, tcp_large_send_info, classification_handle_info, ndis_reserved, sgl_info, ieee_8021q_info, original_pkt_info, pkt_cancel_id, original_netbuf_list, cached_netbuf_list, short_pkt_padding_info, max_perpkt_info } ndis_per_pkt_infotype; #define nbl_hash_value pkt_cancel_id #define nbl_hash_info original_netbuf_list typedef struct ndis_8021q_info_ { union { struct { uint32_t user_pri : 3; /* User Priority */ uint32_t cfi : 1; /* Canonical Format ID */ uint32_t vlan_id : 12; uint32_t reserved : 16; } s1; uint32_t value; } u1; } ndis_8021q_info; struct rndis_object_header { uint8_t type; uint8_t revision; uint16_t size; }; typedef struct rndis_offload_params_ { struct rndis_object_header header; uint8_t ipv4_csum; uint8_t tcp_ipv4_csum; uint8_t udp_ipv4_csum; uint8_t tcp_ipv6_csum; uint8_t udp_ipv6_csum; uint8_t lso_v1; uint8_t ip_sec_v1; uint8_t lso_v2_ipv4; uint8_t lso_v2_ipv6; uint8_t tcp_connection_ipv4; uint8_t tcp_connection_ipv6; uint32_t flags; uint8_t ip_sec_v2; uint8_t ip_sec_v2_ipv4; struct { uint8_t rsc_ipv4; uint8_t rsc_ipv6; }; struct { uint8_t encapsulated_packet_task_offload; uint8_t encapsulation_types; }; } rndis_offload_params; typedef struct rndis_tcp_ip_csum_info_ { union { struct { uint32_t is_ipv4:1; uint32_t is_ipv6:1; uint32_t tcp_csum:1; uint32_t udp_csum:1; uint32_t ip_header_csum:1; uint32_t reserved:11; uint32_t tcp_header_offset:10; } xmit; struct { uint32_t tcp_csum_failed:1; uint32_t udp_csum_failed:1; uint32_t ip_csum_failed:1; uint32_t tcp_csum_succeeded:1; uint32_t udp_csum_succeeded:1; uint32_t ip_csum_succeeded:1; uint32_t loopback:1; uint32_t tcp_csum_value_invalid:1; uint32_t ip_csum_value_invalid:1; } receive; uint32_t value; }; } rndis_tcp_ip_csum_info; struct rndis_hash_value { uint32_t hash_value; } __packed; struct rndis_hash_info { uint32_t hash_info; } __packed; typedef struct rndis_tcp_tso_info_ { union { struct { uint32_t unused:30; uint32_t type:1; uint32_t reserved2:1; } xmit; struct { uint32_t mss:20; uint32_t tcp_header_offset:10; uint32_t type:1; uint32_t reserved2:1; } lso_v1_xmit; struct { uint32_t tcp_payload:30; uint32_t type:1; uint32_t reserved2:1; } lso_v1_xmit_complete; struct { uint32_t mss:20; uint32_t tcp_header_offset:10; uint32_t type:1; uint32_t ip_version:1; } lso_v2_xmit; struct { uint32_t reserved:30; uint32_t type:1; uint32_t reserved2:1; } lso_v2_xmit_complete; uint32_t value; }; } rndis_tcp_tso_info; #define RNDIS_HASHVAL_PPI_SIZE (sizeof(rndis_per_packet_info) + \ sizeof(struct rndis_hash_value)) #define RNDIS_VLAN_PPI_SIZE (sizeof(rndis_per_packet_info) + \ sizeof(ndis_8021q_info)) #define RNDIS_CSUM_PPI_SIZE (sizeof(rndis_per_packet_info) + \ sizeof(rndis_tcp_ip_csum_info)) #define RNDIS_TSO_PPI_SIZE (sizeof(rndis_per_packet_info) + \ sizeof(rndis_tcp_tso_info)) /* * Format of Information buffer passed in a SetRequest for the OID * OID_GEN_RNDIS_CONFIG_PARAMETER. */ typedef struct rndis_config_parameter_info_ { uint32_t parameter_name_offset; uint32_t parameter_name_length; uint32_t parameter_type; uint32_t parameter_value_offset; uint32_t parameter_value_length; } rndis_config_parameter_info; /* * Values for ParameterType in rndis_config_parameter_info */ #define RNDIS_CONFIG_PARAM_TYPE_INTEGER 0 #define RNDIS_CONFIG_PARAM_TYPE_STRING 2 /* * CONDIS Miniport messages for connection oriented devices * that do not implement a call manager. */ /* * CoNdisMiniportCreateVc message */ typedef struct rcondis_mp_create_vc_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS handle */ uint32_t ndis_vc_handle; } rcondis_mp_create_vc; /* * Response to CoNdisMiniportCreateVc */ typedef struct rcondis_mp_create_vc_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS handle */ uint32_t device_vc_handle; /* RNDIS status */ uint32_t status; } rcondis_mp_create_vc_complete; /* * CoNdisMiniportDeleteVc message */ typedef struct rcondis_mp_delete_vc_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS handle */ uint32_t device_vc_handle; } rcondis_mp_delete_vc; /* * Response to CoNdisMiniportDeleteVc */ typedef struct rcondis_mp_delete_vc_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS status */ uint32_t status; } rcondis_mp_delete_vc_complete; /* * CoNdisMiniportQueryRequest message */ typedef struct rcondis_mp_query_request_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS request type */ uint32_t request_type; /* RNDIS OID */ uint32_t oid; /* RNDIS handle */ uint32_t device_vc_handle; uint32_t info_buf_length; uint32_t info_buf_offset; } rcondis_mp_query_request; /* * CoNdisMiniportSetRequest message */ typedef struct rcondis_mp_set_request_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS request type */ uint32_t request_type; /* RNDIS OID */ uint32_t oid; /* RNDIS handle */ uint32_t device_vc_handle; uint32_t info_buf_length; uint32_t info_buf_offset; } rcondis_mp_set_request; /* * CoNdisIndicateStatus message */ typedef struct rcondis_indicate_status_ { /* RNDIS handle */ uint32_t ndis_vc_handle; /* RNDIS status */ uint32_t status; uint32_t status_buf_length; uint32_t status_buf_offset; } rcondis_indicate_status; /* * CONDIS Call/VC parameters */ typedef struct rcondis_specific_parameters_ { uint32_t parameter_type; uint32_t parameter_length; uint32_t parameter_offset; } rcondis_specific_parameters; typedef struct rcondis_media_parameters_ { uint32_t flags; uint32_t reserved1; uint32_t reserved2; rcondis_specific_parameters media_specific; } rcondis_media_parameters; typedef struct rndis_flowspec_ { uint32_t token_rate; uint32_t token_bucket_size; uint32_t peak_bandwidth; uint32_t latency; uint32_t delay_variation; uint32_t service_type; uint32_t max_sdu_size; uint32_t minimum_policed_size; } rndis_flowspec; typedef struct rcondis_call_manager_parameters_ { rndis_flowspec transmit; rndis_flowspec receive; rcondis_specific_parameters call_mgr_specific; } rcondis_call_manager_parameters; /* * CoNdisMiniportActivateVc message */ typedef struct rcondis_mp_activate_vc_request_ { /* RNDIS request ID */ uint32_t request_id; uint32_t flags; /* RNDIS handle */ uint32_t device_vc_handle; uint32_t media_params_offset; uint32_t media_params_length; uint32_t call_mgr_params_offset; uint32_t call_mgr_params_length; } rcondis_mp_activate_vc_request; /* * Response to CoNdisMiniportActivateVc */ typedef struct rcondis_mp_activate_vc_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS status */ uint32_t status; } rcondis_mp_activate_vc_complete; /* * CoNdisMiniportDeactivateVc message */ typedef struct rcondis_mp_deactivate_vc_request_ { /* RNDIS request ID */ uint32_t request_id; uint32_t flags; /* RNDIS handle */ uint32_t device_vc_handle; } rcondis_mp_deactivate_vc_request; /* * Response to CoNdisMiniportDeactivateVc */ typedef struct rcondis_mp_deactivate_vc_complete_ { /* RNDIS request ID */ uint32_t request_id; /* RNDIS status */ uint32_t status; } rcondis_mp_deactivate_vc_complete; /* * union with all of the RNDIS messages */ typedef union rndis_msg_container_ { rndis_packet packet; rndis_initialize_request init_request; rndis_halt_request halt_request; rndis_query_request query_request; rndis_set_request set_request; rndis_reset_request reset_request; rndis_keepalive_request keepalive_request; rndis_indicate_status indicate_status; rndis_initialize_complete init_complete; rndis_query_complete query_complete; rndis_set_complete set_complete; rndis_reset_complete reset_complete; rndis_keepalive_complete keepalive_complete; rcondis_mp_create_vc co_miniport_create_vc; rcondis_mp_delete_vc co_miniport_delete_vc; rcondis_indicate_status co_miniport_status; rcondis_mp_activate_vc_request co_miniport_activate_vc; rcondis_mp_deactivate_vc_request co_miniport_deactivate_vc; rcondis_mp_create_vc_complete co_miniport_create_vc_complete; rcondis_mp_delete_vc_complete co_miniport_delete_vc_complete; rcondis_mp_activate_vc_complete co_miniport_activate_vc_complete; rcondis_mp_deactivate_vc_complete co_miniport_deactivate_vc_complete; rndis_packet_ex packet_ex; } rndis_msg_container; /* * Remote NDIS message format */ typedef struct rndis_msg_ { uint32_t ndis_msg_type; /* * Total length of this message, from the beginning * of the rndis_msg struct, in bytes. */ uint32_t msg_len; /* Actual message */ rndis_msg_container msg; } rndis_msg; /* * Handy macros */ /* * get the size of an RNDIS message. Pass in the message type, * rndis_set_request, rndis_packet for example */ #define RNDIS_MESSAGE_SIZE(message) \ (sizeof(message) + (sizeof(rndis_msg) - sizeof(rndis_msg_container))) /* * get pointer to info buffer with message pointer */ #define MESSAGE_TO_INFO_BUFFER(message) \ (((PUCHAR)(message)) + message->InformationBufferOffset) /* * get pointer to status buffer with message pointer */ #define MESSAGE_TO_STATUS_BUFFER(message) \ (((PUCHAR)(message)) + message->StatusBufferOffset) /* * get pointer to OOBD buffer with message pointer */ #define MESSAGE_TO_OOBD_BUFFER(message) \ (((PUCHAR)(message)) + message->OOBDataOffset) /* * get pointer to data buffer with message pointer */ #define MESSAGE_TO_DATA_BUFFER(message) \ (((PUCHAR)(message)) + message->PerPacketInfoOffset) /* * get pointer to contained message from NDIS_MESSAGE pointer */ #define RNDIS_MESSAGE_PTR_TO_MESSAGE_PTR(rndis_message) \ ((void *) &rndis_message->Message) /* * get pointer to contained message from NDIS_MESSAGE pointer */ #define RNDIS_MESSAGE_RAW_PTR_TO_MESSAGE_PTR(rndis_message) \ ((void *) rndis_message) /* * Structures used in OID_RNDISMP_GET_RECEIVE_BUFFERS */ #define RNDISMP_RECEIVE_BUFFER_ELEM_FLAG_VMQ_RECEIVE_BUFFER 0x00000001 typedef struct rndismp_rx_buf_elem_ { uint32_t flags; uint32_t length; uint64_t rx_buf_id; uint32_t gpadl_handle; void *rx_buf; } rndismp_rx_buf_elem; typedef struct rndismp_rx_bufs_info_ { uint32_t num_rx_bufs; rndismp_rx_buf_elem rx_buf_elems[1]; } rndismp_rx_bufs_info; #define RNDIS_HEADER_SIZE (sizeof(rndis_msg) - sizeof(rndis_msg_container)) -#define NDIS_PACKET_TYPE_DIRECTED 0x00000001 -#define NDIS_PACKET_TYPE_MULTICAST 0x00000002 -#define NDIS_PACKET_TYPE_ALL_MULTICAST 0x00000004 -#define NDIS_PACKET_TYPE_BROADCAST 0x00000008 -#define NDIS_PACKET_TYPE_SOURCE_ROUTING 0x00000010 -#define NDIS_PACKET_TYPE_PROMISCUOUS 0x00000020 -#define NDIS_PACKET_TYPE_SMT 0x00000040 -#define NDIS_PACKET_TYPE_ALL_LOCAL 0x00000080 -#define NDIS_PACKET_TYPE_GROUP 0x00000100 -#define NDIS_PACKET_TYPE_ALL_FUNCTIONAL 0x00000200 -#define NDIS_PACKET_TYPE_FUNCTIONAL 0x00000400 -#define NDIS_PACKET_TYPE_MAC_FRAME 0x00000800 - /* * Externs */ struct hn_rx_ring; struct hn_tx_ring; struct hn_recvinfo; int netvsc_recv(struct hn_rx_ring *rxr, const void *data, int dlen, const struct hn_recvinfo *info); void netvsc_channel_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr); void* hv_set_rppi_data(rndis_msg *rndis_mesg, uint32_t rppi_size, int pkt_type); void* hv_get_ppi_data(rndis_packet *rpkt, uint32_t type); #endif /* __HV_RNDIS_H__ */ Index: projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis_filter.c =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis_filter.c (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis_filter.c (revision 305081) @@ -1,1490 +1,1071 @@ /*- * Copyright (c) 2009-2012,2016 Microsoft Corp. * Copyright (c) 2010-2012 Citrix Inc. * Copyright (c) 2012 NetApp Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define HV_RF_RECVINFO_VLAN 0x1 #define HV_RF_RECVINFO_CSUM 0x2 #define HV_RF_RECVINFO_HASHINF 0x4 #define HV_RF_RECVINFO_HASHVAL 0x8 #define HV_RF_RECVINFO_ALL \ (HV_RF_RECVINFO_VLAN | \ HV_RF_RECVINFO_CSUM | \ HV_RF_RECVINFO_HASHINF | \ HV_RF_RECVINFO_HASHVAL) #define HN_RNDIS_RID_COMPAT_MASK 0xffff #define HN_RNDIS_RID_COMPAT_MAX HN_RNDIS_RID_COMPAT_MASK #define HN_RNDIS_XFER_SIZE 2048 /* * Forward declarations */ -static int hv_rf_send_request(rndis_device *device, rndis_request *request, - uint32_t message_type); -static void hv_rf_receive_response(rndis_device *device, +static void hv_rf_receive_indicate_status(struct hn_softc *sc, const rndis_msg *response); -static void hv_rf_receive_indicate_status(rndis_device *device, - const rndis_msg *response); static void hv_rf_receive_data(struct hn_rx_ring *rxr, const void *data, int dlen); -static inline int hv_rf_query_device_mac(rndis_device *device); -static inline int hv_rf_query_device_link_status(rndis_device *device); -static int hv_rf_set_packet_filter(rndis_device *device, uint32_t new_filter); -static int hv_rf_init_device(rndis_device *device); -static int hv_rf_open_device(rndis_device *device); -static int hv_rf_close_device(rndis_device *device); -int -hv_rf_send_offload_request(struct hn_softc *sc, - rndis_offload_params *offloads); +static int hv_rf_query_device_mac(struct hn_softc *sc, uint8_t *eaddr); +static int hv_rf_query_device_link_status(struct hn_softc *sc, + uint32_t *link_status); +static int hv_rf_init_device(struct hn_softc *sc); -static void hn_rndis_sent_halt(struct hn_send_ctx *sndc, - struct hn_softc *sc, struct vmbus_channel *chan, - const void *data, int dlen); -static void hn_rndis_sent_cb(struct hn_send_ctx *sndc, - struct hn_softc *sc, struct vmbus_channel *chan, - const void *data, int dlen); static int hn_rndis_query(struct hn_softc *sc, uint32_t oid, const void *idata, size_t idlen, void *odata, size_t *odlen0); static int hn_rndis_set(struct hn_softc *sc, uint32_t oid, const void *data, size_t dlen); static int hn_rndis_conf_offload(struct hn_softc *sc); static int hn_rndis_get_rsscaps(struct hn_softc *sc, int *rxr_cnt); static int hn_rndis_conf_rss(struct hn_softc *sc, int nchan); static __inline uint32_t hn_rndis_rid(struct hn_softc *sc) { uint32_t rid; again: rid = atomic_fetchadd_int(&sc->hn_rndis_rid, 1); if (rid == 0) goto again; /* Use upper 16 bits for non-compat RNDIS messages. */ return ((rid & 0xffff) << 16); } /* * Set the Per-Packet-Info with the specified type */ void * hv_set_rppi_data(rndis_msg *rndis_mesg, uint32_t rppi_size, int pkt_type) { rndis_packet *rndis_pkt; rndis_per_packet_info *rppi; rndis_pkt = &rndis_mesg->msg.packet; rndis_pkt->data_offset += rppi_size; rppi = (rndis_per_packet_info *)((char *)rndis_pkt + rndis_pkt->per_pkt_info_offset + rndis_pkt->per_pkt_info_length); rppi->size = rppi_size; rppi->type = pkt_type; rppi->per_packet_info_offset = sizeof(rndis_per_packet_info); rndis_pkt->per_pkt_info_length += rppi_size; return (rppi); } /* * Get the Per-Packet-Info with the specified type * return NULL if not found. */ void * hv_get_ppi_data(rndis_packet *rpkt, uint32_t type) { rndis_per_packet_info *ppi; int len; if (rpkt->per_pkt_info_offset == 0) return (NULL); ppi = (rndis_per_packet_info *)((unsigned long)rpkt + rpkt->per_pkt_info_offset); len = rpkt->per_pkt_info_length; while (len > 0) { if (ppi->type == type) return (void *)((unsigned long)ppi + ppi->per_packet_info_offset); len -= ppi->size; ppi = (rndis_per_packet_info *)((unsigned long)ppi + ppi->size); } return (NULL); } - /* - * Allow module_param to work and override to switch to promiscuous mode. - */ -static inline rndis_device * -hv_get_rndis_device(void) -{ - rndis_device *device; - - device = malloc(sizeof(rndis_device), M_NETVSC, M_WAITOK | M_ZERO); - - mtx_init(&device->req_lock, "HV-FRL", NULL, MTX_DEF); - - /* Same effect as STAILQ_HEAD_INITIALIZER() static initializer */ - STAILQ_INIT(&device->myrequest_list); - - device->state = RNDIS_DEV_UNINITIALIZED; - - return (device); -} - -/* - * - */ -static inline void -hv_put_rndis_device(rndis_device *device) -{ - mtx_destroy(&device->req_lock); - free(device, M_NETVSC); -} - -/* - * - */ -static inline rndis_request * -hv_rndis_request(rndis_device *device, uint32_t message_type, - uint32_t message_length) -{ - rndis_request *request; - rndis_msg *rndis_mesg; - rndis_set_request *set; - - request = malloc(sizeof(rndis_request), M_NETVSC, M_WAITOK | M_ZERO); - - sema_init(&request->wait_sema, 0, "rndis sema"); - - rndis_mesg = &request->request_msg; - rndis_mesg->ndis_msg_type = message_type; - rndis_mesg->msg_len = message_length; - - /* - * Set the request id. This field is always after the rndis header - * for request/response packet types so we just use the set_request - * as a template. - */ - set = &rndis_mesg->msg.set_request; - set->request_id = atomic_fetchadd_int(&device->new_request_id, 1) & - HN_RNDIS_RID_COMPAT_MASK; - - /* Add to the request list */ - mtx_lock(&device->req_lock); - STAILQ_INSERT_TAIL(&device->myrequest_list, request, mylist_entry); - mtx_unlock(&device->req_lock); - - return (request); -} - -/* - * - */ -static inline void -hv_put_rndis_request(rndis_device *device, rndis_request *request) -{ - mtx_lock(&device->req_lock); - /* Fixme: Has O(n) performance */ - /* - * XXXKYS: Use Doubly linked lists. - */ - STAILQ_REMOVE(&device->myrequest_list, request, rndis_request_, - mylist_entry); - mtx_unlock(&device->req_lock); - - sema_destroy(&request->wait_sema); - free(request, M_NETVSC); -} - -/* - * - */ -static int -hv_rf_send_request(rndis_device *device, rndis_request *request, - uint32_t message_type) -{ - struct hn_softc *sc = device->sc; - uint32_t send_buf_section_idx, tot_data_buf_len; - struct vmbus_gpa gpa[2]; - int gpa_cnt, send_buf_section_size; - hn_sent_callback_t cb; - - /* Set up the packet to send it */ - tot_data_buf_len = request->request_msg.msg_len; - - gpa_cnt = 1; - gpa[0].gpa_page = hv_get_phys_addr(&request->request_msg) >> PAGE_SHIFT; - gpa[0].gpa_len = request->request_msg.msg_len; - gpa[0].gpa_ofs = (unsigned long)&request->request_msg & (PAGE_SIZE - 1); - - if (gpa[0].gpa_ofs + gpa[0].gpa_len > PAGE_SIZE) { - gpa_cnt = 2; - gpa[0].gpa_len = PAGE_SIZE - gpa[0].gpa_ofs; - gpa[1].gpa_page = - hv_get_phys_addr((char*)&request->request_msg + - gpa[0].gpa_len) >> PAGE_SHIFT; - gpa[1].gpa_ofs = 0; - gpa[1].gpa_len = request->request_msg.msg_len - gpa[0].gpa_len; - } - - if (message_type != REMOTE_NDIS_HALT_MSG) - cb = hn_rndis_sent_cb; - else - cb = hn_rndis_sent_halt; - - if (tot_data_buf_len < sc->hn_chim_szmax) { - send_buf_section_idx = hn_chim_alloc(sc); - if (send_buf_section_idx != HN_NVS_CHIM_IDX_INVALID) { - uint8_t *dest = sc->hn_chim + - (send_buf_section_idx * sc->hn_chim_szmax); - - memcpy(dest, &request->request_msg, request->request_msg.msg_len); - send_buf_section_size = tot_data_buf_len; - gpa_cnt = 0; - goto sendit; - } - /* Failed to allocate chimney send buffer; move on */ - } - send_buf_section_idx = HN_NVS_CHIM_IDX_INVALID; - send_buf_section_size = 0; - -sendit: - hn_send_ctx_init(&request->send_ctx, cb, request, - send_buf_section_idx, send_buf_section_size); - return hv_nv_on_send(sc->hn_prichan, HN_NVS_RNDIS_MTYPE_CTRL, - &request->send_ctx, gpa, gpa_cnt); -} - -/* - * RNDIS filter receive response - */ -static void -hv_rf_receive_response(rndis_device *device, const rndis_msg *response) -{ - rndis_request *request = NULL; - rndis_request *next_request; - boolean_t found = FALSE; - - mtx_lock(&device->req_lock); - request = STAILQ_FIRST(&device->myrequest_list); - while (request != NULL) { - /* - * All request/response message contains request_id as the - * first field - */ - if (request->request_msg.msg.init_request.request_id == - response->msg.init_complete.request_id) { - found = TRUE; - break; - } - next_request = STAILQ_NEXT(request, mylist_entry); - request = next_request; - } - mtx_unlock(&device->req_lock); - - if (found) { - if (response->msg_len <= sizeof(rndis_msg)) { - memcpy(&request->response_msg, response, - response->msg_len); - } else { - request->response_msg.msg.init_complete.status = - RNDIS_STATUS_BUFFER_OVERFLOW; - } - sema_post(&request->wait_sema); - } -} - -int -hv_rf_send_offload_request(struct hn_softc *sc, - rndis_offload_params *offloads) -{ - rndis_request *request; - rndis_set_request *set; - rndis_offload_params *offload_req; - rndis_set_complete *set_complete; - rndis_device *rndis_dev = sc->rndis_dev; - device_t dev = sc->hn_dev; - uint32_t extlen = sizeof(rndis_offload_params); - int ret; - - if (sc->hn_nvs_ver <= NVSP_PROTOCOL_VERSION_4) { - extlen = VERSION_4_OFFLOAD_SIZE; - /* On NVSP_PROTOCOL_VERSION_4 and below, we do not support - * UDP checksum offload. - */ - offloads->udp_ipv4_csum = 0; - offloads->udp_ipv6_csum = 0; - } - - request = hv_rndis_request(rndis_dev, REMOTE_NDIS_SET_MSG, - RNDIS_MESSAGE_SIZE(rndis_set_request) + extlen); - if (!request) - return (ENOMEM); - - set = &request->request_msg.msg.set_request; - set->oid = RNDIS_OID_TCP_OFFLOAD_PARAMETERS; - set->info_buffer_length = extlen; - set->info_buffer_offset = sizeof(rndis_set_request); - set->device_vc_handle = 0; - - offload_req = (rndis_offload_params *)((unsigned long)set + - set->info_buffer_offset); - *offload_req = *offloads; - offload_req->header.type = RNDIS_OBJECT_TYPE_DEFAULT; - offload_req->header.revision = RNDIS_OFFLOAD_PARAMETERS_REVISION_3; - offload_req->header.size = extlen; - - ret = hv_rf_send_request(rndis_dev, request, REMOTE_NDIS_SET_MSG); - if (ret != 0) { - device_printf(dev, "hv send offload request failed, ret=%d!\n", - ret); - goto cleanup; - } - - ret = sema_timedwait(&request->wait_sema, 5 * hz); - if (ret != 0) { - device_printf(dev, "hv send offload request timeout\n"); - goto cleanup; - } - - set_complete = &request->response_msg.msg.set_complete; - if (set_complete->status == RNDIS_STATUS_SUCCESS) { - device_printf(dev, "hv send offload request succeeded\n"); - ret = 0; - } else { - if (set_complete->status == RNDIS_STATUS_NOT_SUPPORTED) { - device_printf(dev, "HV Not support offload\n"); - ret = 0; - } else { - ret = set_complete->status; - } - } - -cleanup: - hv_put_rndis_request(rndis_dev, request); - - return (ret); -} - -/* * RNDIS filter receive indicate status */ static void -hv_rf_receive_indicate_status(rndis_device *device, const rndis_msg *response) +hv_rf_receive_indicate_status(struct hn_softc *sc, const rndis_msg *response) { const rndis_indicate_status *indicate = &response->msg.indicate_status; switch(indicate->status) { case RNDIS_STATUS_MEDIA_CONNECT: - netvsc_linkstatus_callback(device->sc, 1); + netvsc_linkstatus_callback(sc, 1); break; case RNDIS_STATUS_MEDIA_DISCONNECT: - netvsc_linkstatus_callback(device->sc, 0); + netvsc_linkstatus_callback(sc, 0); break; default: /* TODO: */ - device_printf(device->sc->hn_dev, + if_printf(sc->hn_ifp, "unknown status %d received\n", indicate->status); break; } } static int hv_rf_find_recvinfo(const rndis_packet *rpkt, struct hn_recvinfo *info) { const rndis_per_packet_info *ppi; uint32_t mask, len; info->vlan_info = NULL; info->csum_info = NULL; info->hash_info = NULL; info->hash_value = NULL; if (rpkt->per_pkt_info_offset == 0) return 0; ppi = (const rndis_per_packet_info *) ((const uint8_t *)rpkt + rpkt->per_pkt_info_offset); len = rpkt->per_pkt_info_length; mask = 0; while (len != 0) { const void *ppi_dptr; uint32_t ppi_dlen; if (__predict_false(ppi->size < ppi->per_packet_info_offset)) return EINVAL; ppi_dlen = ppi->size - ppi->per_packet_info_offset; ppi_dptr = (const uint8_t *)ppi + ppi->per_packet_info_offset; switch (ppi->type) { case ieee_8021q_info: if (__predict_false(ppi_dlen < sizeof(ndis_8021q_info))) return EINVAL; info->vlan_info = ppi_dptr; mask |= HV_RF_RECVINFO_VLAN; break; case tcpip_chksum_info: if (__predict_false(ppi_dlen < sizeof(rndis_tcp_ip_csum_info))) return EINVAL; info->csum_info = ppi_dptr; mask |= HV_RF_RECVINFO_CSUM; break; case nbl_hash_value: if (__predict_false(ppi_dlen < sizeof(struct rndis_hash_value))) return EINVAL; info->hash_value = ppi_dptr; mask |= HV_RF_RECVINFO_HASHVAL; break; case nbl_hash_info: if (__predict_false(ppi_dlen < sizeof(struct rndis_hash_info))) return EINVAL; info->hash_info = ppi_dptr; mask |= HV_RF_RECVINFO_HASHINF; break; default: goto skip; } if (mask == HV_RF_RECVINFO_ALL) { /* All found; done */ break; } skip: if (__predict_false(len < ppi->size)) return EINVAL; len -= ppi->size; ppi = (const rndis_per_packet_info *) ((const uint8_t *)ppi + ppi->size); } return 0; } /* * RNDIS filter receive data */ static void hv_rf_receive_data(struct hn_rx_ring *rxr, const void *data, int dlen) { const rndis_msg *message = data; const rndis_packet *rndis_pkt; uint32_t data_offset; struct hn_recvinfo info; rndis_pkt = &message->msg.packet; /* * Fixme: Handle multiple rndis pkt msgs that may be enclosed in this * netvsc packet (ie tot_data_buf_len != message_length) */ /* Remove rndis header, then pass data packet up the stack */ data_offset = RNDIS_HEADER_SIZE + rndis_pkt->data_offset; dlen -= data_offset; if (dlen < rndis_pkt->data_length) { if_printf(rxr->hn_ifp, "total length %u is less than data length %u\n", dlen, rndis_pkt->data_length); return; } dlen = rndis_pkt->data_length; data = (const uint8_t *)data + data_offset; if (hv_rf_find_recvinfo(rndis_pkt, &info)) { if_printf(rxr->hn_ifp, "recvinfo parsing failed\n"); return; } netvsc_recv(rxr, data, dlen, &info); } /* * RNDIS filter on receive */ int hv_rf_on_receive(struct hn_softc *sc, struct hn_rx_ring *rxr, const void *data, int dlen) { - rndis_device *rndis_dev; const rndis_msg *rndis_hdr; const struct rndis_comp_hdr *comp; - rndis_dev = sc->rndis_dev; - if (rndis_dev->state == RNDIS_DEV_UNINITIALIZED) - return (EINVAL); - rndis_hdr = data; switch (rndis_hdr->ndis_msg_type) { /* data message */ case REMOTE_NDIS_PACKET_MSG: hv_rf_receive_data(rxr, data, dlen); break; /* completion messages */ case REMOTE_NDIS_INITIALIZE_CMPLT: case REMOTE_NDIS_QUERY_CMPLT: case REMOTE_NDIS_SET_CMPLT: case REMOTE_NDIS_KEEPALIVE_CMPLT: comp = data; - if (comp->rm_rid <= HN_RNDIS_RID_COMPAT_MAX) { - /* Transition time compat code */ - hv_rf_receive_response(rndis_dev, rndis_hdr); - } else { - vmbus_xact_ctx_wakeup(sc->hn_xact, data, dlen); - } + KASSERT(comp->rm_rid > HN_RNDIS_RID_COMPAT_MAX, + ("invalid rid 0x%08x\n", comp->rm_rid)); + vmbus_xact_ctx_wakeup(sc->hn_xact, comp, dlen); break; /* notification message */ case REMOTE_NDIS_INDICATE_STATUS_MSG: - hv_rf_receive_indicate_status(rndis_dev, rndis_hdr); + hv_rf_receive_indicate_status(sc, rndis_hdr); break; case REMOTE_NDIS_RESET_CMPLT: /* * Reset completed, no rid. * * NOTE: * RESET is not issued by hn(4), so this message should * _not_ be observed. */ if_printf(sc->hn_ifp, "RESET CMPLT received\n"); break; default: if_printf(sc->hn_ifp, "unknown RNDIS message 0x%x\n", rndis_hdr->ndis_msg_type); break; } return (0); } /* * RNDIS filter query device MAC address */ static int -hv_rf_query_device_mac(rndis_device *device) +hv_rf_query_device_mac(struct hn_softc *sc, uint8_t *eaddr) { - struct hn_softc *sc = device->sc; - size_t hwaddr_len; + size_t eaddr_len; int error; - hwaddr_len = ETHER_ADDR_LEN; + eaddr_len = ETHER_ADDR_LEN; error = hn_rndis_query(sc, OID_802_3_PERMANENT_ADDRESS, NULL, 0, - device->hw_mac_addr, &hwaddr_len); + eaddr, &eaddr_len); if (error) return (error); - if (hwaddr_len != ETHER_ADDR_LEN) { - if_printf(sc->hn_ifp, "invalid hwaddr len %zu\n", hwaddr_len); + if (eaddr_len != ETHER_ADDR_LEN) { + if_printf(sc->hn_ifp, "invalid eaddr len %zu\n", eaddr_len); return (EINVAL); } return (0); } /* * RNDIS filter query device link status */ -static inline int -hv_rf_query_device_link_status(rndis_device *device) +static int +hv_rf_query_device_link_status(struct hn_softc *sc, uint32_t *link_status) { - struct hn_softc *sc = device->sc; size_t size; int error; - size = sizeof(uint32_t); + size = sizeof(*link_status); error = hn_rndis_query(sc, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0, - &device->link_status, &size); + link_status, &size); if (error) return (error); if (size != sizeof(uint32_t)) { if_printf(sc->hn_ifp, "invalid link status len %zu\n", size); return (EINVAL); } return (0); } static uint8_t netvsc_hash_key[NDIS_HASH_KEYSIZE_TOEPLITZ] = { 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa }; -/* - * RNDIS filter set packet filter - * Sends an rndis request with the new filter, then waits for a response - * from the host. - * Returns zero on success, non-zero on failure. - */ -static int -hv_rf_set_packet_filter(rndis_device *device, uint32_t new_filter) -{ - rndis_request *request; - rndis_set_request *set; - rndis_set_complete *set_complete; - uint32_t status; - int ret; - - request = hv_rndis_request(device, REMOTE_NDIS_SET_MSG, - RNDIS_MESSAGE_SIZE(rndis_set_request) + sizeof(uint32_t)); - if (request == NULL) { - ret = -1; - goto cleanup; - } - - /* Set up the rndis set */ - set = &request->request_msg.msg.set_request; - set->oid = RNDIS_OID_GEN_CURRENT_PACKET_FILTER; - set->info_buffer_length = sizeof(uint32_t); - set->info_buffer_offset = sizeof(rndis_set_request); - - memcpy((void *)((unsigned long)set + sizeof(rndis_set_request)), - &new_filter, sizeof(uint32_t)); - - ret = hv_rf_send_request(device, request, REMOTE_NDIS_SET_MSG); - if (ret != 0) { - goto cleanup; - } - - /* - * Wait for the response from the host. Another thread will signal - * us when the response has arrived. In the failure case, - * sema_timedwait() returns a non-zero status after waiting 5 seconds. - */ - ret = sema_timedwait(&request->wait_sema, 5 * hz); - if (ret == 0) { - /* Response received, check status */ - set_complete = &request->response_msg.msg.set_complete; - status = set_complete->status; - if (status != RNDIS_STATUS_SUCCESS) { - /* Bad response status, return error */ - ret = -2; - } - } else { - /* - * We cannot deallocate the request since we may still - * receive a send completion for it. - */ - goto exit; - } - -cleanup: - if (request != NULL) { - hv_put_rndis_request(device, request); - } -exit: - return (ret); -} - static const void * -hn_rndis_xact_execute(struct hn_softc *sc, struct vmbus_xact *xact, uint32_t rid, - size_t reqlen, size_t *comp_len0, uint32_t comp_type) +hn_rndis_xact_exec1(struct hn_softc *sc, struct vmbus_xact *xact, size_t reqlen, + struct hn_send_ctx *sndc, size_t *comp_len) { struct vmbus_gpa gpa[HN_XACT_REQ_PGCNT]; - const struct rndis_comp_hdr *comp; - bus_addr_t paddr; - size_t comp_len, min_complen = *comp_len0; int gpa_cnt, error; + bus_addr_t paddr; - KASSERT(rid > HN_RNDIS_RID_COMPAT_MAX, ("invalid rid %u\n", rid)); KASSERT(reqlen <= HN_XACT_REQ_SIZE && reqlen > 0, ("invalid request length %zu", reqlen)); - KASSERT(min_complen >= sizeof(*comp), - ("invalid minimum complete len %zu", min_complen)); /* * Setup the SG list. */ paddr = vmbus_xact_req_paddr(xact); KASSERT((paddr & PAGE_MASK) == 0, ("vmbus xact request is not page aligned 0x%jx", (uintmax_t)paddr)); for (gpa_cnt = 0; gpa_cnt < HN_XACT_REQ_PGCNT; ++gpa_cnt) { int len = PAGE_SIZE; if (reqlen == 0) break; if (reqlen < len) len = reqlen; gpa[gpa_cnt].gpa_page = atop(paddr) + gpa_cnt; gpa[gpa_cnt].gpa_len = len; gpa[gpa_cnt].gpa_ofs = 0; reqlen -= len; } KASSERT(reqlen == 0, ("still have %zu request data left", reqlen)); /* * Send this RNDIS control message and wait for its completion * message. */ vmbus_xact_activate(xact); - error = hv_nv_on_send(sc->hn_prichan, HN_NVS_RNDIS_MTYPE_CTRL, - &hn_send_ctx_none, gpa, gpa_cnt); + error = hv_nv_on_send(sc->hn_prichan, HN_NVS_RNDIS_MTYPE_CTRL, sndc, + gpa, gpa_cnt); if (error) { vmbus_xact_deactivate(xact); if_printf(sc->hn_ifp, "RNDIS ctrl send failed: %d\n", error); return (NULL); } - comp = vmbus_xact_wait(xact, &comp_len); + return (vmbus_xact_wait(xact, comp_len)); +} +static const void * +hn_rndis_xact_execute(struct hn_softc *sc, struct vmbus_xact *xact, uint32_t rid, + size_t reqlen, size_t *comp_len0, uint32_t comp_type) +{ + const struct rndis_comp_hdr *comp; + size_t comp_len, min_complen = *comp_len0; + + KASSERT(rid > HN_RNDIS_RID_COMPAT_MAX, ("invalid rid %u\n", rid)); + KASSERT(min_complen >= sizeof(*comp), + ("invalid minimum complete len %zu", min_complen)); + /* + * Execute the xact setup by the caller. + */ + comp = hn_rndis_xact_exec1(sc, xact, reqlen, &hn_send_ctx_none, + &comp_len); + if (comp == NULL) + return (NULL); + + /* * Check this RNDIS complete message. */ if (comp_len < min_complen) { if (comp_len >= sizeof(*comp)) { /* rm_status field is valid */ if_printf(sc->hn_ifp, "invalid RNDIS comp len %zu, " "status 0x%08x\n", comp_len, comp->rm_status); } else { if_printf(sc->hn_ifp, "invalid RNDIS comp len %zu\n", comp_len); } return (NULL); } if (comp->rm_len < min_complen) { if_printf(sc->hn_ifp, "invalid RNDIS comp msglen %u\n", comp->rm_len); return (NULL); } if (comp->rm_type != comp_type) { if_printf(sc->hn_ifp, "unexpected RNDIS comp 0x%08x, " "expect 0x%08x\n", comp->rm_type, comp_type); return (NULL); } if (comp->rm_rid != rid) { if_printf(sc->hn_ifp, "RNDIS comp rid mismatch %u, " "expect %u\n", comp->rm_rid, rid); return (NULL); } /* All pass! */ *comp_len0 = comp_len; return (comp); } static int hn_rndis_query(struct hn_softc *sc, uint32_t oid, const void *idata, size_t idlen, void *odata, size_t *odlen0) { struct rndis_query_req *req; const struct rndis_query_comp *comp; struct vmbus_xact *xact; size_t reqlen, odlen = *odlen0, comp_len; int error, ofs; uint32_t rid; reqlen = sizeof(*req) + idlen; xact = vmbus_xact_get(sc->hn_xact, reqlen); if (xact == NULL) { if_printf(sc->hn_ifp, "no xact for RNDIS query 0x%08x\n", oid); return (ENXIO); } rid = hn_rndis_rid(sc); req = vmbus_xact_req_data(xact); req->rm_type = REMOTE_NDIS_QUERY_MSG; req->rm_len = reqlen; req->rm_rid = rid; req->rm_oid = oid; /* * XXX * This is _not_ RNDIS Spec conforming: * "This MUST be set to 0 when there is no input data * associated with the OID." * * If this field was set to 0 according to the RNDIS Spec, * Hyper-V would set non-SUCCESS status in the query * completion. */ req->rm_infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET; if (idlen > 0) { req->rm_infobuflen = idlen; /* Input data immediately follows RNDIS query. */ memcpy(req + 1, idata, idlen); } comp_len = sizeof(*comp) + odlen; comp = hn_rndis_xact_execute(sc, xact, rid, reqlen, &comp_len, REMOTE_NDIS_QUERY_CMPLT); if (comp == NULL) { if_printf(sc->hn_ifp, "exec RNDIS query 0x%08x failed\n", oid); error = EIO; goto done; } if (comp->rm_status != RNDIS_STATUS_SUCCESS) { if_printf(sc->hn_ifp, "RNDIS query 0x%08x failed: " "status 0x%08x\n", oid, comp->rm_status); error = EIO; goto done; } if (comp->rm_infobuflen == 0 || comp->rm_infobufoffset == 0) { /* No output data! */ if_printf(sc->hn_ifp, "RNDIS query 0x%08x, no data\n", oid); *odlen0 = 0; error = 0; goto done; } /* * Check output data length and offset. */ /* ofs is the offset from the beginning of comp. */ ofs = RNDIS_QUERY_COMP_INFOBUFABS(comp->rm_infobufoffset); if (ofs < sizeof(*comp) || ofs + comp->rm_infobuflen > comp_len) { if_printf(sc->hn_ifp, "RNDIS query invalid comp ib off/len, " "%u/%u\n", comp->rm_infobufoffset, comp->rm_infobuflen); error = EINVAL; goto done; } /* * Save output data. */ if (comp->rm_infobuflen < odlen) odlen = comp->rm_infobuflen; memcpy(odata, ((const uint8_t *)comp) + ofs, odlen); *odlen0 = odlen; error = 0; done: vmbus_xact_put(xact); return (error); } static int hn_rndis_get_rsscaps(struct hn_softc *sc, int *rxr_cnt) { struct ndis_rss_caps in, caps; size_t caps_len; int error; /* * Only NDIS 6.30+ is supported. */ KASSERT(sc->hn_ndis_ver >= NDIS_VERSION_6_30, ("NDIS 6.30+ is required, NDIS version 0x%08x", sc->hn_ndis_ver)); *rxr_cnt = 0; memset(&in, 0, sizeof(in)); in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; caps_len = NDIS_RSS_CAPS_SIZE; - error = hn_rndis_query(sc, OID_GEN_RSS_CAPABILITIES, + error = hn_rndis_query(sc, OID_GEN_RECEIVE_SCALE_CAPABILITIES, &in, NDIS_RSS_CAPS_SIZE, &caps, &caps_len); if (error) return (error); if (caps_len < NDIS_RSS_CAPS_SIZE_6_0) { if_printf(sc->hn_ifp, "invalid NDIS RSS caps len %zu", caps_len); return (EINVAL); } if (caps.ndis_nrxr == 0) { if_printf(sc->hn_ifp, "0 RX rings!?\n"); return (EINVAL); } *rxr_cnt = caps.ndis_nrxr; if (caps_len == NDIS_RSS_CAPS_SIZE) { if (bootverbose) { if_printf(sc->hn_ifp, "RSS indirect table size %u\n", caps.ndis_nind); } } return (0); } static int hn_rndis_set(struct hn_softc *sc, uint32_t oid, const void *data, size_t dlen) { struct rndis_set_req *req; const struct rndis_set_comp *comp; struct vmbus_xact *xact; size_t reqlen, comp_len; uint32_t rid; int error; KASSERT(dlen > 0, ("invalid dlen %zu", dlen)); reqlen = sizeof(*req) + dlen; xact = vmbus_xact_get(sc->hn_xact, reqlen); if (xact == NULL) { if_printf(sc->hn_ifp, "no xact for RNDIS set 0x%08x\n", oid); return (ENXIO); } rid = hn_rndis_rid(sc); req = vmbus_xact_req_data(xact); req->rm_type = REMOTE_NDIS_SET_MSG; req->rm_len = reqlen; req->rm_rid = rid; req->rm_oid = oid; req->rm_infobuflen = dlen; req->rm_infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET; /* Data immediately follows RNDIS set. */ memcpy(req + 1, data, dlen); comp_len = sizeof(*comp); comp = hn_rndis_xact_execute(sc, xact, rid, reqlen, &comp_len, REMOTE_NDIS_SET_CMPLT); if (comp == NULL) { if_printf(sc->hn_ifp, "exec RNDIS set 0x%08x failed\n", oid); error = EIO; goto done; } if (comp->rm_status != RNDIS_STATUS_SUCCESS) { if_printf(sc->hn_ifp, "RNDIS set 0x%08x failed: " "status 0x%08x\n", oid, comp->rm_status); error = EIO; goto done; } error = 0; done: vmbus_xact_put(xact); return (error); } static int hn_rndis_conf_offload(struct hn_softc *sc) { struct ndis_offload_params params; size_t paramsz; int error; /* NOTE: 0 means "no change" */ memset(¶ms, 0, sizeof(params)); params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; if (sc->hn_ndis_ver < NDIS_VERSION_6_30) { params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2; paramsz = NDIS_OFFLOAD_PARAMS_SIZE_6_1; } else { params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3; paramsz = NDIS_OFFLOAD_PARAMS_SIZE; } params.ndis_hdr.ndis_size = paramsz; params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TXRX; params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TXRX; params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TXRX; if (sc->hn_ndis_ver >= NDIS_VERSION_6_30) { params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TXRX; params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TXRX; } params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; /* XXX ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON */ error = hn_rndis_set(sc, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, paramsz); if (error) { if_printf(sc->hn_ifp, "offload config failed: %d\n", error); } else { if (bootverbose) if_printf(sc->hn_ifp, "offload config done\n"); } return (error); } static int hn_rndis_conf_rss(struct hn_softc *sc, int nchan) { struct ndis_rssprm_toeplitz *rss = &sc->hn_rss; struct ndis_rss_params *prm = &rss->rss_params; int i, error; /* * Only NDIS 6.30+ is supported. */ KASSERT(sc->hn_ndis_ver >= NDIS_VERSION_6_30, ("NDIS 6.30+ is required, NDIS version 0x%08x", sc->hn_ndis_ver)); memset(rss, 0, sizeof(*rss)); prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; prm->ndis_hdr.ndis_size = sizeof(*rss); prm->ndis_hash = NDIS_HASH_FUNCTION_TOEPLITZ | NDIS_HASH_IPV4 | NDIS_HASH_TCP_IPV4 | NDIS_HASH_IPV6 | NDIS_HASH_TCP_IPV6; /* TODO: Take ndis_rss_caps.ndis_nind into account */ prm->ndis_indsize = sizeof(rss->rss_ind); prm->ndis_indoffset = __offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); prm->ndis_keysize = sizeof(rss->rss_key); prm->ndis_keyoffset = __offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); /* Setup RSS key */ memcpy(rss->rss_key, netvsc_hash_key, sizeof(rss->rss_key)); /* Setup RSS indirect table */ /* TODO: Take ndis_rss_caps.ndis_nind into account */ for (i = 0; i < NDIS_HASH_INDCNT; ++i) rss->rss_ind[i] = i % nchan; - error = hn_rndis_set(sc, OID_GEN_RSS_PARAMETERS, rss, sizeof(*rss)); + error = hn_rndis_set(sc, OID_GEN_RECEIVE_SCALE_PARAMETERS, + rss, sizeof(*rss)); if (error) { if_printf(sc->hn_ifp, "RSS config failed: %d\n", error); } else { if (bootverbose) if_printf(sc->hn_ifp, "RSS config done\n"); } return (error); } +static int +hn_rndis_set_rxfilter(struct hn_softc *sc, uint32_t filter) +{ + int error; + + error = hn_rndis_set(sc, OID_GEN_CURRENT_PACKET_FILTER, + &filter, sizeof(filter)); + if (error) { + if_printf(sc->hn_ifp, "set RX filter 0x%08x failed: %d\n", + filter, error); + } else { + if (bootverbose) { + if_printf(sc->hn_ifp, "set RX filter 0x%08x done\n", + filter); + } + } + return (error); +} + /* * RNDIS filter init device */ static int -hv_rf_init_device(rndis_device *device) +hv_rf_init_device(struct hn_softc *sc) { - struct hn_softc *sc = device->sc; struct rndis_init_req *req; const struct rndis_init_comp *comp; struct vmbus_xact *xact; size_t comp_len; uint32_t rid; int error; - /* XXX */ - device->state = RNDIS_DEV_INITIALIZED; - xact = vmbus_xact_get(sc->hn_xact, sizeof(*req)); if (xact == NULL) { if_printf(sc->hn_ifp, "no xact for RNDIS init\n"); return (ENXIO); } rid = hn_rndis_rid(sc); req = vmbus_xact_req_data(xact); req->rm_type = REMOTE_NDIS_INITIALIZE_MSG; req->rm_len = sizeof(*req); req->rm_rid = rid; req->rm_ver_major = RNDIS_VERSION_MAJOR; req->rm_ver_minor = RNDIS_VERSION_MINOR; req->rm_max_xfersz = HN_RNDIS_XFER_SIZE; comp_len = RNDIS_INIT_COMP_SIZE_MIN; comp = hn_rndis_xact_execute(sc, xact, rid, sizeof(*req), &comp_len, REMOTE_NDIS_INITIALIZE_CMPLT); if (comp == NULL) { if_printf(sc->hn_ifp, "exec RNDIS init failed\n"); error = EIO; goto done; } if (comp->rm_status != RNDIS_STATUS_SUCCESS) { if_printf(sc->hn_ifp, "RNDIS init failed: status 0x%08x\n", comp->rm_status); error = EIO; goto done; } if (bootverbose) { - if_printf(sc->hn_ifp, "RNDIS ver %u.%u, pktsz %u, pktcnt %u\n", - comp->rm_ver_major, comp->rm_ver_minor, - comp->rm_pktmaxsz, comp->rm_pktmaxcnt); + if_printf(sc->hn_ifp, "RNDIS ver %u.%u, pktsz %u, pktcnt %u, " + "align %u\n", comp->rm_ver_major, comp->rm_ver_minor, + comp->rm_pktmaxsz, comp->rm_pktmaxcnt, + 1U << comp->rm_align); } error = 0; - done: - if (xact != NULL) - vmbus_xact_put(xact); + vmbus_xact_put(xact); return (error); } -#define HALT_COMPLETION_WAIT_COUNT 25 - /* * RNDIS filter halt device */ static int -hv_rf_halt_device(rndis_device *device) +hv_rf_halt_device(struct hn_softc *sc) { - rndis_request *request; - int i, ret; + struct vmbus_xact *xact; + struct rndis_halt_req *halt; + struct hn_send_ctx sndc; + size_t comp_len; - /* Attempt to do a rndis device halt */ - request = hv_rndis_request(device, REMOTE_NDIS_HALT_MSG, - RNDIS_MESSAGE_SIZE(rndis_halt_request)); - if (request == NULL) { - return (-1); + xact = vmbus_xact_get(sc->hn_xact, sizeof(*halt)); + if (xact == NULL) { + if_printf(sc->hn_ifp, "no xact for RNDIS halt\n"); + return (ENXIO); } + halt = vmbus_xact_req_data(xact); + halt->rm_type = REMOTE_NDIS_HALT_MSG; + halt->rm_len = sizeof(*halt); + halt->rm_rid = hn_rndis_rid(sc); - /* initialize "poor man's semaphore" */ - request->halt_complete_flag = 0; + /* No RNDIS completion; rely on NVS message send completion */ + hn_send_ctx_init_simple(&sndc, hn_nvs_sent_xact, xact); + hn_rndis_xact_exec1(sc, xact, sizeof(*halt), &sndc, &comp_len); - ret = hv_rf_send_request(device, request, REMOTE_NDIS_HALT_MSG); - if (ret != 0) { - return (-1); - } - - /* - * Wait for halt response from halt callback. We must wait for - * the transaction response before freeing the request and other - * resources. - */ - for (i=HALT_COMPLETION_WAIT_COUNT; i > 0; i--) { - if (request->halt_complete_flag != 0) { - break; - } - DELAY(400); - } - if (i == 0) { - return (-1); - } - - device->state = RNDIS_DEV_UNINITIALIZED; - - hv_put_rndis_request(device, request); - + vmbus_xact_put(xact); + if (bootverbose) + if_printf(sc->hn_ifp, "RNDIS halt done\n"); return (0); } /* - * RNDIS filter open device - */ -static int -hv_rf_open_device(rndis_device *device) -{ - int ret; - - if (device->state != RNDIS_DEV_INITIALIZED) { - return (0); - } - - if (hv_promisc_mode != 1) { - ret = hv_rf_set_packet_filter(device, - NDIS_PACKET_TYPE_BROADCAST | - NDIS_PACKET_TYPE_ALL_MULTICAST | - NDIS_PACKET_TYPE_DIRECTED); - } else { - ret = hv_rf_set_packet_filter(device, - NDIS_PACKET_TYPE_PROMISCUOUS); - } - - if (ret == 0) { - device->state = RNDIS_DEV_DATAINITIALIZED; - } - - return (ret); -} - -/* - * RNDIS filter close device - */ -static int -hv_rf_close_device(rndis_device *device) -{ - int ret; - - if (device->state != RNDIS_DEV_DATAINITIALIZED) { - return (0); - } - - ret = hv_rf_set_packet_filter(device, 0); - if (ret == 0) { - device->state = RNDIS_DEV_INITIALIZED; - } - - return (ret); -} - -/* * RNDIS filter on device add */ int hv_rf_on_device_add(struct hn_softc *sc, void *additl_info, int *nchan0, struct hn_rx_ring *rxr) { int ret; - rndis_device *rndis_dev; netvsc_device_info *dev_info = (netvsc_device_info *)additl_info; device_t dev = sc->hn_dev; struct hn_nvs_subch_req *req; const struct hn_nvs_subch_resp *resp; size_t resp_len; struct vmbus_xact *xact = NULL; uint32_t status, nsubch; int nchan = *nchan0; int rxr_cnt; - rndis_dev = hv_get_rndis_device(); - if (rndis_dev == NULL) { - return (ENOMEM); - } - sc->rndis_dev = rndis_dev; - rndis_dev->sc = sc; - /* * Let the inner driver handle this first to create the netvsc channel * NOTE! Once the channel is created, we may get a receive callback * (hv_rf_on_receive()) before this call is completed. * Note: Earlier code used a function pointer here. */ ret = hv_nv_on_device_add(sc, rxr); - if (ret != 0) { - hv_put_rndis_device(rndis_dev); + if (ret != 0) return (ret); - } /* * Initialize the rndis device */ /* Send the rndis initialization message */ - ret = hv_rf_init_device(rndis_dev); + ret = hv_rf_init_device(sc); if (ret != 0) { /* * TODO: If rndis init failed, we will need to shut down * the channel */ } /* Get the mac address */ - ret = hv_rf_query_device_mac(rndis_dev); + ret = hv_rf_query_device_mac(sc, dev_info->mac_addr); if (ret != 0) { /* TODO: shut down rndis device and the channel */ } /* Configure NDIS offload settings */ hn_rndis_conf_offload(sc); - - memcpy(dev_info->mac_addr, rndis_dev->hw_mac_addr, ETHER_ADDR_LEN); - hv_rf_query_device_link_status(rndis_dev); - - dev_info->link_state = rndis_dev->link_status; + hv_rf_query_device_link_status(sc, &dev_info->link_state); if (sc->hn_ndis_ver < NDIS_VERSION_6_30 || nchan == 1) { /* * Either RSS is not supported, or multiple RX/TX rings * are not requested. */ *nchan0 = 1; return (0); } /* * Get RSS capabilities, e.g. # of RX rings, and # of indirect * table entries. */ ret = hn_rndis_get_rsscaps(sc, &rxr_cnt); if (ret) { /* No RSS; this is benign. */ *nchan0 = 1; return (0); } if (nchan > rxr_cnt) nchan = rxr_cnt; if_printf(sc->hn_ifp, "RX rings offered %u, requested %d\n", rxr_cnt, nchan); if (nchan == 1) { device_printf(dev, "only 1 channel is supported, no vRSS\n"); goto out; } /* * Ask NVS to allocate sub-channels. */ xact = vmbus_xact_get(sc->hn_xact, sizeof(*req)); if (xact == NULL) { if_printf(sc->hn_ifp, "no xact for nvs subch req\n"); ret = ENXIO; goto out; } req = vmbus_xact_req_data(xact); req->nvs_type = HN_NVS_TYPE_SUBCH_REQ; req->nvs_op = HN_NVS_SUBCH_OP_ALLOC; req->nvs_nsubch = nchan - 1; resp = hn_nvs_xact_execute(sc, xact, req, sizeof(*req), &resp_len); if (resp == NULL) { if_printf(sc->hn_ifp, "exec subch failed\n"); ret = EIO; goto out; } if (resp_len < sizeof(*resp)) { if_printf(sc->hn_ifp, "invalid subch resp length %zu\n", resp_len); ret = EINVAL; goto out; } if (resp->nvs_type != HN_NVS_TYPE_SUBCH_RESP) { if_printf(sc->hn_ifp, "not subch resp, type %u\n", resp->nvs_type); ret = EINVAL; goto out; } status = resp->nvs_status; nsubch = resp->nvs_nsubch; vmbus_xact_put(xact); xact = NULL; if (status != HN_NVS_STATUS_OK) { if_printf(sc->hn_ifp, "subch req failed: %x\n", status); ret = EIO; goto out; } if (nsubch > nchan - 1) { if_printf(sc->hn_ifp, "%u subchans are allocated, requested %u\n", nsubch, nchan - 1); nsubch = nchan - 1; } nchan = nsubch + 1; ret = hn_rndis_conf_rss(sc, nchan); if (ret != 0) *nchan0 = 1; else *nchan0 = nchan; out: if (xact != NULL) vmbus_xact_put(xact); return (ret); } /* * RNDIS filter on device remove */ int -hv_rf_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel) +hv_rf_on_device_remove(struct hn_softc *sc) { - rndis_device *rndis_dev = sc->rndis_dev; int ret; /* Halt and release the rndis device */ - ret = hv_rf_halt_device(rndis_dev); + ret = hv_rf_halt_device(sc); - sc->rndis_dev = NULL; - hv_put_rndis_device(rndis_dev); - /* Pass control to inner driver to remove the device */ - ret |= hv_nv_on_device_remove(sc, destroy_channel); + ret |= hv_nv_on_device_remove(sc); return (ret); } /* * RNDIS filter on open */ int hv_rf_on_open(struct hn_softc *sc) { + uint32_t filter; - return (hv_rf_open_device(sc->rndis_dev)); + /* XXX */ + if (hv_promisc_mode != 1) { + filter = NDIS_PACKET_TYPE_BROADCAST | + NDIS_PACKET_TYPE_ALL_MULTICAST | + NDIS_PACKET_TYPE_DIRECTED; + } else { + filter = NDIS_PACKET_TYPE_PROMISCUOUS; + } + return (hn_rndis_set_rxfilter(sc, filter)); } /* * RNDIS filter on close */ int hv_rf_on_close(struct hn_softc *sc) { - return (hv_rf_close_device(sc->rndis_dev)); -} - -static void -hn_rndis_sent_cb(struct hn_send_ctx *sndc, struct hn_softc *sc, - struct vmbus_channel *chan __unused, const void *data __unused, - int dlen __unused) -{ - if (sndc->hn_chim_idx != HN_NVS_CHIM_IDX_INVALID) - hn_chim_free(sc, sndc->hn_chim_idx); -} - -static void -hn_rndis_sent_halt(struct hn_send_ctx *sndc, struct hn_softc *sc, - struct vmbus_channel *chan __unused, const void *data __unused, - int dlen __unused) -{ - rndis_request *request = sndc->hn_cbarg; - - if (sndc->hn_chim_idx != HN_NVS_CHIM_IDX_INVALID) - hn_chim_free(sc, sndc->hn_chim_idx); - - /* - * Notify hv_rf_halt_device() about halt completion. - * The halt code must wait for completion before freeing - * the transaction resources. - */ - request->halt_complete_flag = 1; + return (hn_rndis_set_rxfilter(sc, 0)); } void hv_rf_channel_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr) { netvsc_channel_rollup(rxr, txr); } Index: projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis_filter.h =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis_filter.h (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/hv_rndis_filter.h (revision 305081) @@ -1,127 +1,53 @@ /*- * Copyright (c) 2009-2012,2016 Microsoft Corp. * Copyright (c) 2010-2012 Citrix Inc. * Copyright (c) 2012 NetApp Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __HV_RNDIS_FILTER_H__ #define __HV_RNDIS_FILTER_H__ #include #include #include /* - * Defines - */ - -/* Destroy or preserve channel on filter/netvsc teardown */ -#define HV_RF_NV_DESTROY_CHANNEL TRUE -#define HV_RF_NV_RETAIN_CHANNEL FALSE - -/* - * Number of page buffers to reserve for the RNDIS filter packet in the - * transmitted message. - */ -#define HV_RF_NUM_TX_RESERVED_PAGE_BUFS 1 - - -/* - * Data types - */ - -typedef enum { - RNDIS_DEV_UNINITIALIZED = 0, - RNDIS_DEV_INITIALIZING, - RNDIS_DEV_INITIALIZED, - RNDIS_DEV_DATAINITIALIZED, -} rndis_device_state; - -typedef struct rndis_request_ { - STAILQ_ENTRY(rndis_request_) mylist_entry; - struct sema wait_sema; - - /* - * The max response size is sizeof(rndis_msg) + PAGE_SIZE. - * - * XXX - * This is ugly and should be cleaned up once we busdma-fy - * RNDIS request bits. - */ - rndis_msg response_msg; - uint8_t buf_resp[PAGE_SIZE]; - - /* Simplify allocation by having a netvsc packet inline */ - struct hn_send_ctx send_ctx; - - /* - * The max request size is sizeof(rndis_msg) + PAGE_SIZE. - * - * NOTE: - * This is required for the large request like RSS settings. - * - * XXX - * This is ugly and should be cleaned up once we busdma-fy - * RNDIS request bits. - */ - rndis_msg request_msg; - uint8_t buf_req[PAGE_SIZE]; - - /* Fixme: Poor man's semaphore. */ - uint32_t halt_complete_flag; -} rndis_request; - -typedef struct rndis_device_ { - struct hn_softc *sc; - - rndis_device_state state; - uint32_t link_status; - uint32_t new_request_id; - - struct mtx req_lock; - - STAILQ_HEAD(RQ, rndis_request_) myrequest_list; - - uint8_t hw_mac_addr[ETHER_ADDR_LEN]; -} rndis_device; - -/* * Externs */ struct hn_rx_ring; int hv_rf_on_receive(struct hn_softc *sc, struct hn_rx_ring *rxr, const void *data, int dlen); void hv_rf_channel_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr); int hv_rf_on_device_add(struct hn_softc *sc, void *additl_info, int *nchan, struct hn_rx_ring *rxr); -int hv_rf_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel); +int hv_rf_on_device_remove(struct hn_softc *sc); int hv_rf_on_open(struct hn_softc *sc); int hv_rf_on_close(struct hn_softc *sc); #endif /* __HV_RNDIS_FILTER_H__ */ Index: projects/clang390-import/sys/dev/hyperv/netvsc/if_hnvar.h =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/if_hnvar.h (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/if_hnvar.h (revision 305081) @@ -1,120 +1,122 @@ /*- * Copyright (c) 2016 Microsoft Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _IF_HNVAR_H_ #define _IF_HNVAR_H_ #include #include #include struct hn_softc; struct vmbus_channel; struct hn_send_ctx; typedef void (*hn_sent_callback_t) (struct hn_send_ctx *, struct hn_softc *, struct vmbus_channel *, const void *, int); struct hn_send_ctx { hn_sent_callback_t hn_cb; void *hn_cbarg; uint32_t hn_chim_idx; int hn_chim_sz; }; struct rndis_hash_info; struct rndix_hash_value; struct ndis_8021q_info_; struct rndis_tcp_ip_csum_info_; struct hn_recvinfo { const struct ndis_8021q_info_ *vlan_info; const struct rndis_tcp_ip_csum_info_ *csum_info; const struct rndis_hash_info *hash_info; const struct rndis_hash_value *hash_value; }; #define HN_SEND_CTX_INITIALIZER(cb, cbarg) \ { \ .hn_cb = cb, \ .hn_cbarg = cbarg, \ .hn_chim_idx = HN_NVS_CHIM_IDX_INVALID, \ .hn_chim_sz = 0 \ } static __inline void hn_send_ctx_init(struct hn_send_ctx *sndc, hn_sent_callback_t cb, void *cbarg, uint32_t chim_idx, int chim_sz) { sndc->hn_cb = cb; sndc->hn_cbarg = cbarg; sndc->hn_chim_idx = chim_idx; sndc->hn_chim_sz = chim_sz; } static __inline void hn_send_ctx_init_simple(struct hn_send_ctx *sndc, hn_sent_callback_t cb, void *cbarg) { hn_send_ctx_init(sndc, cb, cbarg, HN_NVS_CHIM_IDX_INVALID, 0); } static __inline int hn_nvs_send(struct vmbus_channel *chan, uint16_t flags, void *nvs_msg, int nvs_msglen, struct hn_send_ctx *sndc) { return (vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND, flags, nvs_msg, nvs_msglen, (uint64_t)(uintptr_t)sndc)); } static __inline int hn_nvs_send_sglist(struct vmbus_channel *chan, struct vmbus_gpa sg[], int sglen, void *nvs_msg, int nvs_msglen, struct hn_send_ctx *sndc) { return (vmbus_chan_send_sglist(chan, sg, sglen, nvs_msg, nvs_msglen, (uint64_t)(uintptr_t)sndc)); } struct vmbus_xact; const void *hn_nvs_xact_execute(struct hn_softc *sc, struct vmbus_xact *xact, void *req, int reqlen, size_t *resp_len); +void hn_nvs_sent_xact(struct hn_send_ctx *sndc, struct hn_softc *sc, + struct vmbus_channel *chan, const void *data, int dlen); uint32_t hn_chim_alloc(struct hn_softc *sc); void hn_chim_free(struct hn_softc *sc, uint32_t chim_idx); extern struct hn_send_ctx hn_send_ctx_none; #endif /* !_IF_HNVAR_H_ */ Index: projects/clang390-import/sys/dev/hyperv/netvsc/ndis.h =================================================================== --- projects/clang390-import/sys/dev/hyperv/netvsc/ndis.h (revision 305080) +++ projects/clang390-import/sys/dev/hyperv/netvsc/ndis.h (revision 305081) @@ -1,217 +1,206 @@ /*- * Copyright (c) 2016 Microsoft Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _NET_NDIS_H_ #define _NET_NDIS_H_ -#define NDIS_MEDIA_STATE_CONNECTED 0 -#define NDIS_MEDIA_STATE_DISCONNECTED 1 +#define NDIS_MEDIA_STATE_CONNECTED 0 +#define NDIS_MEDIA_STATE_DISCONNECTED 1 -#define OID_GEN_RSS_CAPABILITIES 0x00010203 -#define OID_GEN_RSS_PARAMETERS 0x00010204 -#define OID_TCP_OFFLOAD_PARAMETERS 0xFC01020C +#define NDIS_OFFLOAD_SET_NOCHG 0 +#define NDIS_OFFLOAD_SET_ON 1 +#define NDIS_OFFLOAD_SET_OFF 2 -#define NDIS_OBJTYPE_DEFAULT 0x80 -#define NDIS_OBJTYPE_RSS_CAPS 0x88 -#define NDIS_OBJTYPE_RSS_PARAMS 0x89 - -/* common_set */ -#define NDIS_OFFLOAD_SET_NOCHG 0 -#define NDIS_OFFLOAD_SET_ON 1 -#define NDIS_OFFLOAD_SET_OFF 2 - /* a.k.a GRE MAC */ -#define NDIS_ENCAP_TYPE_NVGRE 0x00000001 +#define NDIS_ENCAP_TYPE_NVGRE 0x00000001 -#define NDIS_HASH_FUNCTION_MASK 0x000000FF /* see hash function */ -#define NDIS_HASH_TYPE_MASK 0x00FFFF00 /* see hash type */ +#define NDIS_HASH_FUNCTION_MASK 0x000000FF /* see hash function */ +#define NDIS_HASH_TYPE_MASK 0x00FFFF00 /* see hash type */ /* hash function */ -#define NDIS_HASH_FUNCTION_TOEPLITZ 0x00000001 +#define NDIS_HASH_FUNCTION_TOEPLITZ 0x00000001 /* hash type */ -#define NDIS_HASH_IPV4 0x00000100 -#define NDIS_HASH_TCP_IPV4 0x00000200 -#define NDIS_HASH_IPV6 0x00000400 -#define NDIS_HASH_IPV6_EX 0x00000800 -#define NDIS_HASH_TCP_IPV6 0x00001000 -#define NDIS_HASH_TCP_IPV6_EX 0x00002000 +#define NDIS_HASH_IPV4 0x00000100 +#define NDIS_HASH_TCP_IPV4 0x00000200 +#define NDIS_HASH_IPV6 0x00000400 +#define NDIS_HASH_IPV6_EX 0x00000800 +#define NDIS_HASH_TCP_IPV6 0x00001000 +#define NDIS_HASH_TCP_IPV6_EX 0x00002000 -#define NDIS_HASH_KEYSIZE_TOEPLITZ 40 -#define NDIS_HASH_INDCNT 128 +#define NDIS_HASH_KEYSIZE_TOEPLITZ 40 +#define NDIS_HASH_INDCNT 128 +#define NDIS_OBJTYPE_DEFAULT 0x80 +#define NDIS_OBJTYPE_RSS_CAPS 0x88 +#define NDIS_OBJTYPE_RSS_PARAMS 0x89 + struct ndis_object_hdr { - uint8_t ndis_type; /* NDIS_OBJTYPE_ */ - uint8_t ndis_rev; /* type specific */ - uint16_t ndis_size; /* incl. this hdr */ + uint8_t ndis_type; /* NDIS_OBJTYPE_ */ + uint8_t ndis_rev; /* type specific */ + uint16_t ndis_size; /* incl. this hdr */ }; /* * OID_TCP_OFFLOAD_PARAMETERS * ndis_type: NDIS_OBJTYPE_DEFAULT */ struct ndis_offload_params { struct ndis_object_hdr ndis_hdr; - uint8_t ndis_ip4csum; /* param_set */ - uint8_t ndis_tcp4csum; /* param_set */ - uint8_t ndis_udp4csum; /* param_set */ - uint8_t ndis_tcp6csum; /* param_set */ - uint8_t ndis_udp6csum; /* param_set */ - uint8_t ndis_lsov1; /* lsov1_set */ - uint8_t ndis_ipsecv1; /* ipsecv1_set */ - uint8_t ndis_lsov2_ip4; /* lsov2_set */ - uint8_t ndis_lsov2_ip6; /* lsov2_set */ - uint8_t ndis_tcp4conn; /* PARAM_NOCHG */ - uint8_t ndis_tcp6conn; /* PARAM_NOCHG */ - uint32_t ndis_flags; /* 0 */ + uint8_t ndis_ip4csum; /* NDIS_OFFLOAD_PARAM_ */ + uint8_t ndis_tcp4csum; /* NDIS_OFFLOAD_PARAM_ */ + uint8_t ndis_udp4csum; /* NDIS_OFFLOAD_PARAM_ */ + uint8_t ndis_tcp6csum; /* NDIS_OFFLOAD_PARAM_ */ + uint8_t ndis_udp6csum; /* NDIS_OFFLOAD_PARAM_ */ + uint8_t ndis_lsov1; /* NDIS_OFFLOAD_PARAM_ */ + uint8_t ndis_ipsecv1; /* NDIS_OFFLOAD_IPSECV1_ */ + uint8_t ndis_lsov2_ip4; /* NDIS_OFFLOAD_LSOV2_ */ + uint8_t ndis_lsov2_ip6; /* NDIS_OFFLOAD_LSOV2_ */ + uint8_t ndis_tcp4conn; /* 0 */ + uint8_t ndis_tcp6conn; /* 0 */ + uint32_t ndis_flags; /* 0 */ /* NDIS >= 6.1 */ - uint8_t ndis_ipsecv2; /* ipsecv2_set */ - uint8_t ndis_ipsecv2_ip4; /* ipsecv2_set */ + uint8_t ndis_ipsecv2; /* NDIS_OFFLOAD_IPSECV2_ */ + uint8_t ndis_ipsecv2_ip4;/* NDIS_OFFLOAD_IPSECV2_ */ /* NDIS >= 6.30 */ - uint8_t ndis_rsc_ip4; /* rsc_set */ - uint8_t ndis_rsc_ip6; /* rsc_set */ - uint8_t ndis_encap; /* common_set */ - uint8_t ndis_encap_types; /* NDIS_ENCAP_TYPE_ */ + uint8_t ndis_rsc_ip4; /* NDIS_OFFLOAD_RSC_ */ + uint8_t ndis_rsc_ip6; /* NDIS_OFFLOAD_RSC_ */ + uint8_t ndis_encap; /* NDIS_OFFLOAD_SET_ */ + uint8_t ndis_encap_types;/* NDIS_ENCAP_TYPE_ */ }; -#define NDIS_OFFLOAD_PARAMS_SIZE sizeof(struct ndis_offload_params) -#define NDIS_OFFLOAD_PARAMS_SIZE_6_1 \ +#define NDIS_OFFLOAD_PARAMS_SIZE sizeof(struct ndis_offload_params) +#define NDIS_OFFLOAD_PARAMS_SIZE_6_1 \ __offsetof(struct ndis_offload_params, ndis_rsc_ip4) -#define NDIS_OFFLOAD_PARAMS_REV_2 2 /* NDIS 6.1 */ -#define NDIS_OFFLOAD_PARAMS_REV_3 3 /* NDIS 6.30 */ +#define NDIS_OFFLOAD_PARAMS_REV_2 2 /* NDIS 6.1 */ +#define NDIS_OFFLOAD_PARAMS_REV_3 3 /* NDIS 6.30 */ -/* param_set */ -#define NDIS_OFFLOAD_PARAM_NOCHG 0 /* common to all sets */ -#define NDIS_OFFLOAD_PARAM_OFF 1 -#define NDIS_OFFLOAD_PARAM_TX 2 -#define NDIS_OFFLOAD_PARAM_RX 3 -#define NDIS_OFFLOAD_PARAM_TXRX 4 +#define NDIS_OFFLOAD_PARAM_NOCHG 0 /* common */ +#define NDIS_OFFLOAD_PARAM_OFF 1 +#define NDIS_OFFLOAD_PARAM_TX 2 +#define NDIS_OFFLOAD_PARAM_RX 3 +#define NDIS_OFFLOAD_PARAM_TXRX 4 -/* lsov1_set */ /* NDIS_OFFLOAD_PARAM_NOCHG */ -#define NDIS_OFFLOAD_LSOV1_OFF 1 -#define NDIS_OFFLOAD_LSOV1_ON 2 +#define NDIS_OFFLOAD_LSOV1_OFF 1 +#define NDIS_OFFLOAD_LSOV1_ON 2 -/* ipsecv1_set */ /* NDIS_OFFLOAD_PARAM_NOCHG */ -#define NDIS_OFFLOAD_IPSECV1_OFF 1 -#define NDIS_OFFLOAD_IPSECV1_AH 2 -#define NDIS_OFFLOAD_IPSECV1_ESP 3 -#define NDIS_OFFLOAD_IPSECV1_AH_ESP 4 +#define NDIS_OFFLOAD_IPSECV1_OFF 1 +#define NDIS_OFFLOAD_IPSECV1_AH 2 +#define NDIS_OFFLOAD_IPSECV1_ESP 3 +#define NDIS_OFFLOAD_IPSECV1_AH_ESP 4 -/* lsov2_set */ /* NDIS_OFFLOAD_PARAM_NOCHG */ -#define NDIS_OFFLOAD_LSOV2_OFF 1 -#define NDIS_OFFLOAD_LSOV2_ON 2 +#define NDIS_OFFLOAD_LSOV2_OFF 1 +#define NDIS_OFFLOAD_LSOV2_ON 2 -/* ipsecv2_set */ /* NDIS_OFFLOAD_PARAM_NOCHG */ -#define NDIS_OFFLOAD_IPSECV2_OFF 1 -#define NDIS_OFFLOAD_IPSECV2_AH 2 -#define NDIS_OFFLOAD_IPSECV2_ESP 3 -#define NDIS_OFFLOAD_IPSECV2_AH_ESP 4 +#define NDIS_OFFLOAD_IPSECV2_OFF 1 +#define NDIS_OFFLOAD_IPSECV2_AH 2 +#define NDIS_OFFLOAD_IPSECV2_ESP 3 +#define NDIS_OFFLOAD_IPSECV2_AH_ESP 4 -/* rsc_set */ /* NDIS_OFFLOAD_PARAM_NOCHG */ -#define NDIS_OFFLOAD_RSC_OFF 1 -#define NDIS_OFFLOAD_RSC_ON 2 +#define NDIS_OFFLOAD_RSC_OFF 1 +#define NDIS_OFFLOAD_RSC_ON 2 /* - * OID_GEN_RSS_CAPABILITIES + * OID_GEN_RECEIVE_SCALE_CAPABILITIES * ndis_type: NDIS_OBJTYPE_RSS_CAPS */ struct ndis_rss_caps { struct ndis_object_hdr ndis_hdr; uint32_t ndis_flags; /* NDIS_RSS_CAP_ */ uint32_t ndis_nmsi; /* # of MSIs */ uint32_t ndis_nrxr; /* # of RX rings */ /* NDIS >= 6.30 */ uint16_t ndis_nind; /* # of indtbl ent. */ uint16_t ndis_pad; }; -#define NDIS_RSS_CAPS_SIZE \ +#define NDIS_RSS_CAPS_SIZE \ __offsetof(struct ndis_rss_caps, ndis_pad) -#define NDIS_RSS_CAPS_SIZE_6_0 \ +#define NDIS_RSS_CAPS_SIZE_6_0 \ __offsetof(struct ndis_rss_caps, ndis_nind) -#define NDIS_RSS_CAPS_REV_1 1 /* NDIS 6.{0,1,20} */ -#define NDIS_RSS_CAPS_REV_2 2 /* NDIS 6.30 */ +#define NDIS_RSS_CAPS_REV_1 1 /* NDIS 6.{0,1,20} */ +#define NDIS_RSS_CAPS_REV_2 2 /* NDIS 6.30 */ -#define NDIS_RSS_CAP_MSI 0x01000000 -#define NDIS_RSS_CAP_CLASSIFY_ISR 0x02000000 -#define NDIS_RSS_CAP_CLASSIFY_DPC 0x04000000 -#define NDIS_RSS_CAP_MSIX 0x08000000 -#define NDIS_RSS_CAP_IPV4 0x00000100 -#define NDIS_RSS_CAP_IPV6 0x00000200 -#define NDIS_RSS_CAP_IPV6_EX 0x00000400 -#define NDIS_RSS_CAP_HASH_TOEPLITZ 0x00000001 +#define NDIS_RSS_CAP_MSI 0x01000000 +#define NDIS_RSS_CAP_CLASSIFY_ISR 0x02000000 +#define NDIS_RSS_CAP_CLASSIFY_DPC 0x04000000 +#define NDIS_RSS_CAP_MSIX 0x08000000 +#define NDIS_RSS_CAP_IPV4 0x00000100 +#define NDIS_RSS_CAP_IPV6 0x00000200 +#define NDIS_RSS_CAP_IPV6_EX 0x00000400 +#define NDIS_RSS_CAP_HASH_TOEPLITZ 0x00000001 /* - * OID_GEN_RSS_PARAMETERS + * OID_GEN_RECEIVE_SCALE_PARAMETERS * ndis_type: NDIS_OBJTYPE_RSS_PARAMS */ struct ndis_rss_params { struct ndis_object_hdr ndis_hdr; uint16_t ndis_flags; /* NDIS_RSS_FLAG_ */ uint16_t ndis_bcpu; /* base cpu 0 */ uint32_t ndis_hash; /* NDIS_HASH_ */ uint16_t ndis_indsize; /* indirect table */ uint32_t ndis_indoffset; uint16_t ndis_keysize; /* hash key */ uint32_t ndis_keyoffset; /* NDIS >= 6.20 */ uint32_t ndis_cpumaskoffset; uint32_t ndis_cpumaskcnt; uint32_t ndis_cpumaskentsz; }; -#define NDIS_RSS_PARAMS_SIZE sizeof(struct ndis_rss_params) -#define NDIS_RSS_PARAMS_SIZE_6_0 \ +#define NDIS_RSS_PARAMS_SIZE sizeof(struct ndis_rss_params) +#define NDIS_RSS_PARAMS_SIZE_6_0 \ __offsetof(struct ndis_rss_params, ndis_cpumaskoffset) -#define NDIS_RSS_PARAMS_REV_1 1 /* NDIS 6.0 */ -#define NDIS_RSS_PARAMS_REV_2 2 /* NDIS 6.20 */ +#define NDIS_RSS_PARAMS_REV_1 1 /* NDIS 6.0 */ +#define NDIS_RSS_PARAMS_REV_2 2 /* NDIS 6.20 */ -#define NDIS_RSS_FLAG_BCPU_UNCHG 0x0001 -#define NDIS_RSS_FLAG_HASH_UNCHG 0x0002 -#define NDIS_RSS_FLAG_IND_UNCHG 0x0004 -#define NDIS_RSS_FLAG_KEY_UNCHG 0x0008 -#define NDIS_RSS_FLAG_DISABLE 0x0010 +#define NDIS_RSS_FLAG_BCPU_UNCHG 0x0001 +#define NDIS_RSS_FLAG_HASH_UNCHG 0x0002 +#define NDIS_RSS_FLAG_IND_UNCHG 0x0004 +#define NDIS_RSS_FLAG_KEY_UNCHG 0x0008 +#define NDIS_RSS_FLAG_DISABLE 0x0010 /* non-standard convenient struct */ struct ndis_rssprm_toeplitz { struct ndis_rss_params rss_params; /* Toeplitz hash key */ uint8_t rss_key[NDIS_HASH_KEYSIZE_TOEPLITZ]; /* Indirect table */ uint32_t rss_ind[NDIS_HASH_INDCNT]; }; #endif /* !_NET_NDIS_H_ */ Index: projects/clang390-import/sys/dev/iscsi_initiator/iscsi.h =================================================================== --- projects/clang390-import/sys/dev/iscsi_initiator/iscsi.h (revision 305080) +++ projects/clang390-import/sys/dev/iscsi_initiator/iscsi.h (revision 305081) @@ -1,500 +1,500 @@ /*- * Copyright (c) 2005-2010 Daniel Braniss * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* | $Id: iscsi.h 743 2009-08-08 10:54:53Z danny $ */ #define TRUE 1 #define FALSE 0 #ifndef _KERNEL typedef int boolean_t; #endif #include #define ISCSIDEV "iscsi" #define ISCSI_MAX_TARGETS 64 /* | iSCSI commands */ /* | Initiator Opcodes: */ #define ISCSI_NOP_OUT 0x00 #define ISCSI_SCSI_CMD 0x01 #define ISCSI_TASK_CMD 0x02 #define ISCSI_LOGIN_CMD 0x03 #define ISCSI_TEXT_CMD 0x04 #define ISCSI_WRITE_DATA 0x05 #define ISCSI_LOGOUT_CMD 0x06 #define ISCSI_SNACK 0x10 /* | Target Opcodes: */ #define ISCSI_NOP_IN 0x20 #define ISCSI_SCSI_RSP 0x21 #define ISCSI_TASK_RSP 0x22 #define ISCSI_LOGIN_RSP 0x23 #define ISCSI_TEXT_RSP 0x24 #define ISCSI_READ_DATA 0x25 #define ISCSI_LOGOUT_RSP 0x26 #define ISCSI_R2T 0x31 #define ISCSI_ASYNC 0x32 #define ISCSI_REJECT 0x3f /* | PDU stuff */ /* | BHS Basic Header Segment */ typedef struct bhs { // the order is network byte order! u_char opcode:6; u_char I:1; u_char _:1; u_char __:7; u_char F:1; // Final bit u_char ___[2]; u_int AHSLength:8; // in 4byte words u_int DSLength:24; // in bytes u_int LUN[2]; // or Opcode-specific fields u_int itt; u_int OpcodeSpecificFields[7]; #define CmdSN OpcodeSpecificFields[1] #define ExpStSN OpcodeSpecificFields[2] #define MaxCmdSN OpcodeSpecificFields[3] } bhs_t; typedef struct ahs { u_int len:16; u_int type:8; u_int spec:8; char data[0]; } ahs_t; typedef struct { // Sequence Numbers // (computers were invented to count, right?) int cmd; int expcmd; int maxcmd; } req_sn_t; typedef struct { // Sequence Numbers // (computers were invented to count, right?) int stat; int expcmd; int maxcmd; } rsp_sn_t; typedef struct scsi_req { u_char opcode:6; // 0x01 u_char I:1; u_char _:1; u_char attr:3; u_char _0:2; u_char W:1; u_char R:1; u_char F:1; #define iSCSI_TASK_UNTAGGED 0 #define iSCSI_TASK_SIMPLE 1 #define iSCSI_TASK_ORDER 2 #define iSCSI_TASK_HOFQ 3 #define iSCSI_TASK_ACA 4 char _1[2]; int len; int lun[2]; int itt; int edtlen; // expectect data transfere length int cmdSN; int extStatSN; int cdb[4]; } scsi_req_t; typedef struct scsi_rsp { char opcode; // 0x21 u_char flag; u_char response; u_char status; int len; int _[2]; int itt; int stag; rsp_sn_t sn; int expdatasn; int bdrcnt; // bidirectional residual count int rcnt; // residual count } scsi_rsp_t; typedef struct nop_out { // the order is network byte order! u_char opcode:6; u_char I:1; u_char _:1; u_char __:7; u_char F:1; // Final bit u_char ___[2]; u_int len; u_int lun[2]; u_int itt; u_int ttt; req_sn_t sn; u_int mbz[3]; } nop_out_t; typedef struct nop_in { // the order is network byte order! u_char opcode:6; u_char I:1; u_char _:1; u_char __:7; u_char F:1; // Final bit u_char ___[2]; u_int len; u_int lun[2]; u_int itt; u_int ttt; rsp_sn_t sn; u_int ____[2]; } nop_in_t; typedef struct r2t { u_char opcode:6; u_char I:1; u_char _:1; u_char __:7; u_char F:1; // Final bit u_char ___[2]; u_int len; u_int lun[2]; u_int itt; u_int ttt; rsp_sn_t sn; u_int r2tSN; u_int bo; u_int ddtl; } r2t_t; typedef struct data_out { u_char opcode:6; u_char I:1; u_char _:1; u_char __:7; u_char F:1; // Final bit u_char ___[2]; u_int len; u_int lun[2]; u_int itt; u_int ttt; rsp_sn_t sn; u_int dsn; // data seq. number u_int bo; u_int ____; } data_out_t; typedef struct data_in { u_char opcode:6; u_char I:1; u_char _:1; u_char S:1; u_char U:1; u_char O:1; u_char __:3; u_char A:1; u_char F:1; // Final bit u_char ___[1]; u_char status; u_int len; u_int lun[2]; u_int itt; u_int ttt; rsp_sn_t sn; u_int dataSN; u_int bo; u_int ____; } data_in_t; typedef struct reject { u_char opcode:6; u_char _:2; u_char F:1; u_char __:7; u_char reason; u_char ___; u_int len; u_int ____[2]; u_int tt[2]; // must be -1 rsp_sn_t sn; u_int dataSN; // or R2TSN or reserved u_int _____[2]; } reject_t; typedef struct async { u_char opcode:6; u_char _:2; u_char F:1; u_char __:7; u_char ___[2]; u_int len; u_int lun[2]; u_int itt; // must be -1 u_int ____; rsp_sn_t sn; u_char asyncEvent; u_char asyncVCode; u_char param1[2]; u_char param2[2]; u_char param3[2]; u_int _____; } async_t; typedef struct login_req { char cmd; // 0x03 u_char NSG:2; u_char CSG:2; u_char _:2; u_char C:1; u_char T:1; char v_max; char v_min; int len; // remapped via standard bhs char isid[6]; short tsih; int itt; // Initiator Task Tag; int CID:16; int rsv:16; int cmdSN; int expStatSN; int unused[4]; } login_req_t; typedef struct login_rsp { char cmd; // 0x23 u_char NSG:2; u_char CSG:2; u_char _1:2; u_char C:1; u_char T:1; char v_max; char v_act; int len; // remapped via standard bhs char isid[6]; short tsih; int itt; // Initiator Task Tag; int _2; rsp_sn_t sn; int status:16; int _3:16; int _4[2]; } login_rsp_t; typedef struct text_req { char cmd; // 0x04 u_char _1:6; u_char C:1; // Continuation u_char F:1; // Final char _2[2]; int len; int itt; // Initiator Task Tag int LUN[2]; int ttt; // Target Transfer Tag int cmdSN; int expStatSN; int unused[4]; } text_req_t; typedef struct logout_req { char cmd; // 0x06 - char reason; // 0 - close session + u_char reason; // 0 - close session // 1 - close connection // 2 - remove the connection for recovery char _2[2]; int len; int _r[2]; int itt; // Initiator Task Tag; u_int CID:16; u_int rsv:16; int cmdSN; int expStatSN; int unused[4]; } logout_req_t; typedef struct logout_rsp { char cmd; // 0x26 char cbits; char _1[2]; int len; int _2[2]; int itt; int _3; rsp_sn_t sn; short time2wait; short time2retain; int _4; } logout_rsp_t; union ipdu_u { bhs_t bhs; scsi_req_t scsi_req; scsi_rsp_t scsi_rsp; nop_out_t nop_out; nop_in_t nop_in; r2t_t r2t; data_out_t data_out; data_in_t data_in; reject_t reject; async_t async; }; /* | Sequence Numbers */ typedef struct { u_int itt; u_int cmd; u_int expCmd; u_int maxCmd; u_int stat; u_int expStat; u_int data; } sn_t; /* | in-core version of a Protocol Data Unit */ typedef struct { union ipdu_u ipdu; u_int hdr_dig; // header digest ahs_t *ahs_addr; u_int ahs_len; u_int ahs_size; // the allocated size u_char *ds_addr; u_int ds_len; u_int ds_size; // the allocated size u_int ds_dig; // data digest } pdu_t; typedef struct opvals { int port; int tags; int maxluns; int sockbufsize; int maxConnections; int maxRecvDataSegmentLength; int maxXmitDataSegmentLength; // pseudo ... int maxBurstLength; int firstBurstLength; int defaultTime2Wait; int defaultTime2Retain; int maxOutstandingR2T; int errorRecoveryLevel; int targetPortalGroupTag; boolean_t initialR2T; boolean_t immediateData; boolean_t dataPDUInOrder; boolean_t dataSequenceInOrder; char *headerDigest; char *dataDigest; char *sessionType; char *sendTargets; char *targetAddress; char *targetAlias; char *targetName; char *initiatorName; char *initiatorAlias; char *authMethod; char *chapSecret; char *chapIName; char *chapDigest; char *tgtChapName; char *tgtChapSecret; int tgtChallengeLen; u_char tgtChapID; char *tgtChapDigest; char *iqn; char *pidfile; } isc_opt_t; /* | ioctl */ #define ISCSISETSES _IOR('i', 1, int) #define ISCSISETSOC _IOW('i', 2, int) #define ISCSISETOPT _IOW('i', 5, isc_opt_t) #define ISCSIGETOPT _IOR('i', 6, isc_opt_t) #define ISCSISEND _IOW('i', 10, pdu_t) #define ISCSIRECV _IOWR('i', 11, pdu_t) #define ISCSIPING _IO('i', 20) #define ISCSISIGNAL _IOW('i', 21, int *) #define ISCSISTART _IO('i', 30) #define ISCSIRESTART _IO('i', 31) #define ISCSISTOP _IO('i', 32) typedef struct iscsi_cam { path_id_t path_id; target_id_t target_id; int target_nluns; } iscsi_cam_t; #define ISCSIGETCAM _IOR('i', 33, iscsi_cam_t) Index: projects/clang390-import/sys/dev/kbdmux/kbdmux.c =================================================================== --- projects/clang390-import/sys/dev/kbdmux/kbdmux.c (revision 305080) +++ projects/clang390-import/sys/dev/kbdmux/kbdmux.c (revision 305081) @@ -1,1398 +1,1399 @@ /* * kbdmux.c */ /*- * Copyright (c) 2005 Maksim Yevmenkin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $Id: kbdmux.c,v 1.4 2005/07/14 17:38:35 max Exp $ * $FreeBSD$ */ #include "opt_compat.h" #include "opt_kbd.h" #include "opt_kbdmux.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* the initial key map, accent map and fkey strings */ #ifdef KBDMUX_DFLT_KEYMAP #define KBD_DFLT_KEYMAP #include "kbdmuxmap.h" #endif #include #define KEYBOARD_NAME "kbdmux" MALLOC_DECLARE(M_KBDMUX); MALLOC_DEFINE(M_KBDMUX, KEYBOARD_NAME, "Keyboard multiplexor"); /***************************************************************************** ***************************************************************************** ** Keyboard state ***************************************************************************** *****************************************************************************/ #define KBDMUX_Q_SIZE 512 /* input queue size */ /* * XXX * For now rely on Giant mutex to protect our data structures. * Just like the rest of keyboard drivers and syscons(4) do. * Note that callout is initialized as not MP-safe to make sure * Giant is held. */ #if 0 /* not yet */ #define KBDMUX_LOCK_DECL_GLOBAL \ struct mtx ks_lock #define KBDMUX_LOCK_INIT(s) \ mtx_init(&(s)->ks_lock, "kbdmux", NULL, MTX_DEF|MTX_RECURSE) #define KBDMUX_LOCK_DESTROY(s) \ mtx_destroy(&(s)->ks_lock) #define KBDMUX_LOCK(s) \ mtx_lock(&(s)->ks_lock) #define KBDMUX_UNLOCK(s) \ mtx_unlock(&(s)->ks_lock) #define KBDMUX_LOCK_ASSERT(s, w) \ mtx_assert(&(s)->ks_lock, (w)) #define KBDMUX_SLEEP(s, f, d, t) \ msleep(&(s)->f, &(s)->ks_lock, PCATCH | (PZERO + 1), (d), (t)) #define KBDMUX_CALLOUT_INIT(s) \ callout_init_mtx(&(s)->ks_timo, &(s)->ks_lock, 0) #define KBDMUX_QUEUE_INTR(s) \ taskqueue_enqueue(taskqueue_swi_giant, &(s)->ks_task) #else #define KBDMUX_LOCK_DECL_GLOBAL #define KBDMUX_LOCK_INIT(s) #define KBDMUX_LOCK_DESTROY(s) #define KBDMUX_LOCK(s) #define KBDMUX_UNLOCK(s) #define KBDMUX_LOCK_ASSERT(s, w) #define KBDMUX_SLEEP(s, f, d, t) \ tsleep(&(s)->f, PCATCH | (PZERO + 1), (d), (t)) #define KBDMUX_CALLOUT_INIT(s) \ callout_init(&(s)->ks_timo, 0) #define KBDMUX_QUEUE_INTR(s) \ taskqueue_enqueue(taskqueue_swi_giant, &(s)->ks_task) #endif /* not yet */ /* * kbdmux keyboard */ struct kbdmux_kbd { keyboard_t *kbd; /* keyboard */ SLIST_ENTRY(kbdmux_kbd) next; /* link to next */ }; typedef struct kbdmux_kbd kbdmux_kbd_t; /* * kbdmux state */ struct kbdmux_state { char ks_inq[KBDMUX_Q_SIZE]; /* input chars queue */ unsigned int ks_inq_start; unsigned int ks_inq_length; struct task ks_task; /* interrupt task */ struct callout ks_timo; /* timeout handler */ #define TICKS (hz) /* rate */ int ks_flags; /* flags */ #define COMPOSE (1 << 0) /* compose char flag */ -#define POLLING (1 << 1) /* polling */ #define TASK (1 << 2) /* interrupt task queued */ + int ks_polling; /* poll nesting count */ int ks_mode; /* K_XLATE, K_RAW, K_CODE */ int ks_state; /* state */ int ks_accents; /* accent key index (> 0) */ u_int ks_composed_char; /* composed char code */ u_char ks_prefix; /* AT scan code prefix */ SLIST_HEAD(, kbdmux_kbd) ks_kbds; /* keyboards */ KBDMUX_LOCK_DECL_GLOBAL; }; typedef struct kbdmux_state kbdmux_state_t; /***************************************************************************** ***************************************************************************** ** Helper functions ***************************************************************************** *****************************************************************************/ static task_fn_t kbdmux_kbd_intr; static timeout_t kbdmux_kbd_intr_timo; static kbd_callback_func_t kbdmux_kbd_event; static void kbdmux_kbd_putc(kbdmux_state_t *state, char c) { unsigned int p; if (state->ks_inq_length == KBDMUX_Q_SIZE) return; p = (state->ks_inq_start + state->ks_inq_length) % KBDMUX_Q_SIZE; state->ks_inq[p] = c; state->ks_inq_length++; } static int kbdmux_kbd_getc(kbdmux_state_t *state) { unsigned char c; if (state->ks_inq_length == 0) return (-1); c = state->ks_inq[state->ks_inq_start]; state->ks_inq_start = (state->ks_inq_start + 1) % KBDMUX_Q_SIZE; state->ks_inq_length--; return (c); } /* * Interrupt handler task */ void kbdmux_kbd_intr(void *xkbd, int pending) { keyboard_t *kbd = (keyboard_t *) xkbd; kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; kbdd_intr(kbd, NULL); KBDMUX_LOCK(state); state->ks_flags &= ~TASK; wakeup(&state->ks_task); KBDMUX_UNLOCK(state); } /* * Schedule interrupt handler on timeout. Called with locked state. */ void kbdmux_kbd_intr_timo(void *xstate) { kbdmux_state_t *state = (kbdmux_state_t *) xstate; KBDMUX_LOCK_ASSERT(state, MA_OWNED); if (callout_pending(&state->ks_timo)) return; /* callout was reset */ if (!callout_active(&state->ks_timo)) return; /* callout was stopped */ callout_deactivate(&state->ks_timo); /* queue interrupt task if needed */ if (state->ks_inq_length > 0 && !(state->ks_flags & TASK) && KBDMUX_QUEUE_INTR(state) == 0) state->ks_flags |= TASK; /* re-schedule timeout */ callout_reset(&state->ks_timo, TICKS, kbdmux_kbd_intr_timo, state); } /* * Process event from one of our keyboards */ static int kbdmux_kbd_event(keyboard_t *kbd, int event, void *arg) { kbdmux_state_t *state = (kbdmux_state_t *) arg; switch (event) { case KBDIO_KEYINPUT: { int c; KBDMUX_LOCK(state); /* * Read all chars from the keyboard * * Turns out that atkbd(4) check_char() method may return * "true" while read_char() method returns NOKEY. If this * happens we could stuck in the loop below. Avoid this * by breaking out of the loop if read_char() method returns * NOKEY. */ while (kbdd_check_char(kbd)) { c = kbdd_read_char(kbd, 0); if (c == NOKEY) break; if (c == ERRKEY) continue; /* XXX ring bell */ if (!KBD_IS_BUSY(kbd)) continue; /* not open - discard the input */ kbdmux_kbd_putc(state, c); } /* queue interrupt task if needed */ if (state->ks_inq_length > 0 && !(state->ks_flags & TASK) && KBDMUX_QUEUE_INTR(state) == 0) state->ks_flags |= TASK; KBDMUX_UNLOCK(state); } break; case KBDIO_UNLOADING: { kbdmux_kbd_t *k; KBDMUX_LOCK(state); SLIST_FOREACH(k, &state->ks_kbds, next) if (k->kbd == kbd) break; if (k != NULL) { kbd_release(k->kbd, &k->kbd); SLIST_REMOVE(&state->ks_kbds, k, kbdmux_kbd, next); k->kbd = NULL; free(k, M_KBDMUX); } KBDMUX_UNLOCK(state); } break; default: return (EINVAL); /* NOT REACHED */ } return (0); } /**************************************************************************** **************************************************************************** ** Keyboard driver **************************************************************************** ****************************************************************************/ static int kbdmux_configure(int flags); static kbd_probe_t kbdmux_probe; static kbd_init_t kbdmux_init; static kbd_term_t kbdmux_term; static kbd_intr_t kbdmux_intr; static kbd_test_if_t kbdmux_test_if; static kbd_enable_t kbdmux_enable; static kbd_disable_t kbdmux_disable; static kbd_read_t kbdmux_read; static kbd_check_t kbdmux_check; static kbd_read_char_t kbdmux_read_char; static kbd_check_char_t kbdmux_check_char; static kbd_ioctl_t kbdmux_ioctl; static kbd_lock_t kbdmux_lock; static void kbdmux_clear_state_locked(kbdmux_state_t *state); static kbd_clear_state_t kbdmux_clear_state; static kbd_get_state_t kbdmux_get_state; static kbd_set_state_t kbdmux_set_state; static kbd_poll_mode_t kbdmux_poll; static keyboard_switch_t kbdmuxsw = { .probe = kbdmux_probe, .init = kbdmux_init, .term = kbdmux_term, .intr = kbdmux_intr, .test_if = kbdmux_test_if, .enable = kbdmux_enable, .disable = kbdmux_disable, .read = kbdmux_read, .check = kbdmux_check, .read_char = kbdmux_read_char, .check_char = kbdmux_check_char, .ioctl = kbdmux_ioctl, .lock = kbdmux_lock, .clear_state = kbdmux_clear_state, .get_state = kbdmux_get_state, .set_state = kbdmux_set_state, .get_fkeystr = genkbd_get_fkeystr, .poll = kbdmux_poll, .diag = genkbd_diag, }; /* * Return the number of found keyboards */ static int kbdmux_configure(int flags) { return (1); } /* * Detect a keyboard */ static int kbdmux_probe(int unit, void *arg, int flags) { if (resource_disabled(KEYBOARD_NAME, unit)) return (ENXIO); return (0); } /* * Reset and initialize the keyboard (stolen from atkbd.c) */ static int kbdmux_init(int unit, keyboard_t **kbdp, void *arg, int flags) { keyboard_t *kbd = NULL; kbdmux_state_t *state = NULL; keymap_t *keymap = NULL; accentmap_t *accmap = NULL; fkeytab_t *fkeymap = NULL; int error, needfree, fkeymap_size, delay[2]; if (*kbdp == NULL) { *kbdp = kbd = malloc(sizeof(*kbd), M_KBDMUX, M_NOWAIT | M_ZERO); state = malloc(sizeof(*state), M_KBDMUX, M_NOWAIT | M_ZERO); keymap = malloc(sizeof(key_map), M_KBDMUX, M_NOWAIT); accmap = malloc(sizeof(accent_map), M_KBDMUX, M_NOWAIT); fkeymap = malloc(sizeof(fkey_tab), M_KBDMUX, M_NOWAIT); fkeymap_size = sizeof(fkey_tab)/sizeof(fkey_tab[0]); needfree = 1; if ((kbd == NULL) || (state == NULL) || (keymap == NULL) || (accmap == NULL) || (fkeymap == NULL)) { error = ENOMEM; goto bad; } KBDMUX_LOCK_INIT(state); TASK_INIT(&state->ks_task, 0, kbdmux_kbd_intr, (void *) kbd); KBDMUX_CALLOUT_INIT(state); SLIST_INIT(&state->ks_kbds); } else if (KBD_IS_INITIALIZED(*kbdp) && KBD_IS_CONFIGURED(*kbdp)) { return (0); } else { kbd = *kbdp; state = (kbdmux_state_t *) kbd->kb_data; keymap = kbd->kb_keymap; accmap = kbd->kb_accentmap; fkeymap = kbd->kb_fkeytab; fkeymap_size = kbd->kb_fkeytab_size; needfree = 0; } if (!KBD_IS_PROBED(kbd)) { /* XXX assume 101/102 keys keyboard */ kbd_init_struct(kbd, KEYBOARD_NAME, KB_101, unit, flags, 0, 0); bcopy(&key_map, keymap, sizeof(key_map)); bcopy(&accent_map, accmap, sizeof(accent_map)); bcopy(fkey_tab, fkeymap, imin(fkeymap_size*sizeof(fkeymap[0]), sizeof(fkey_tab))); kbd_set_maps(kbd, keymap, accmap, fkeymap, fkeymap_size); kbd->kb_data = (void *)state; KBD_FOUND_DEVICE(kbd); KBD_PROBE_DONE(kbd); KBDMUX_LOCK(state); kbdmux_clear_state_locked(state); state->ks_mode = K_XLATE; KBDMUX_UNLOCK(state); } if (!KBD_IS_INITIALIZED(kbd) && !(flags & KB_CONF_PROBE_ONLY)) { kbd->kb_config = flags & ~KB_CONF_PROBE_ONLY; kbdmux_ioctl(kbd, KDSETLED, (caddr_t)&state->ks_state); delay[0] = kbd->kb_delay1; delay[1] = kbd->kb_delay2; kbdmux_ioctl(kbd, KDSETREPEAT, (caddr_t)delay); KBD_INIT_DONE(kbd); } if (!KBD_IS_CONFIGURED(kbd)) { if (kbd_register(kbd) < 0) { error = ENXIO; goto bad; } KBD_CONFIG_DONE(kbd); KBDMUX_LOCK(state); callout_reset(&state->ks_timo, TICKS, kbdmux_kbd_intr_timo, state); KBDMUX_UNLOCK(state); } return (0); bad: if (needfree) { if (state != NULL) free(state, M_KBDMUX); if (keymap != NULL) free(keymap, M_KBDMUX); if (accmap != NULL) free(accmap, M_KBDMUX); if (fkeymap != NULL) free(fkeymap, M_KBDMUX); if (kbd != NULL) { free(kbd, M_KBDMUX); *kbdp = NULL; /* insure ref doesn't leak to caller */ } } return (error); } /* * Finish using this keyboard */ static int kbdmux_term(keyboard_t *kbd) { kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; kbdmux_kbd_t *k; KBDMUX_LOCK(state); /* kill callout */ callout_stop(&state->ks_timo); /* wait for interrupt task */ while (state->ks_flags & TASK) KBDMUX_SLEEP(state, ks_task, "kbdmuxc", 0); /* release all keyboards from the mux */ while ((k = SLIST_FIRST(&state->ks_kbds)) != NULL) { kbd_release(k->kbd, &k->kbd); SLIST_REMOVE_HEAD(&state->ks_kbds, next); k->kbd = NULL; free(k, M_KBDMUX); } KBDMUX_UNLOCK(state); kbd_unregister(kbd); KBDMUX_LOCK_DESTROY(state); bzero(state, sizeof(*state)); free(state, M_KBDMUX); free(kbd->kb_keymap, M_KBDMUX); free(kbd->kb_accentmap, M_KBDMUX); free(kbd->kb_fkeytab, M_KBDMUX); free(kbd, M_KBDMUX); return (0); } /* * Keyboard interrupt routine */ static int kbdmux_intr(keyboard_t *kbd, void *arg) { int c; if (KBD_IS_ACTIVE(kbd) && KBD_IS_BUSY(kbd)) { /* let the callback function to process the input */ (*kbd->kb_callback.kc_func)(kbd, KBDIO_KEYINPUT, kbd->kb_callback.kc_arg); } else { /* read and discard the input; no one is waiting for input */ do { c = kbdmux_read_char(kbd, FALSE); } while (c != NOKEY); } return (0); } /* * Test the interface to the device */ static int kbdmux_test_if(keyboard_t *kbd) { return (0); } /* * Enable the access to the device; until this function is called, * the client cannot read from the keyboard. */ static int kbdmux_enable(keyboard_t *kbd) { KBD_ACTIVATE(kbd); return (0); } /* * Disallow the access to the device */ static int kbdmux_disable(keyboard_t *kbd) { KBD_DEACTIVATE(kbd); return (0); } /* * Read one byte from the keyboard if it's allowed */ static int kbdmux_read(keyboard_t *kbd, int wait) { kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; int c; KBDMUX_LOCK(state); c = kbdmux_kbd_getc(state); KBDMUX_UNLOCK(state); if (c != -1) kbd->kb_count ++; return (KBD_IS_ACTIVE(kbd)? c : -1); } /* * Check if data is waiting */ static int kbdmux_check(keyboard_t *kbd) { kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; int ready; if (!KBD_IS_ACTIVE(kbd)) return (FALSE); KBDMUX_LOCK(state); ready = (state->ks_inq_length > 0) ? TRUE : FALSE; KBDMUX_UNLOCK(state); return (ready); } /* * Read char from the keyboard (stolen from atkbd.c) */ static u_int kbdmux_read_char(keyboard_t *kbd, int wait) { kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; u_int action; int scancode, keycode; KBDMUX_LOCK(state); next_code: /* do we have a composed char to return? */ if (!(state->ks_flags & COMPOSE) && (state->ks_composed_char > 0)) { action = state->ks_composed_char; state->ks_composed_char = 0; if (action > UCHAR_MAX) { KBDMUX_UNLOCK(state); return (ERRKEY); } KBDMUX_UNLOCK(state); return (action); } /* see if there is something in the keyboard queue */ scancode = kbdmux_kbd_getc(state); if (scancode == -1) { - if (state->ks_flags & POLLING) { + if (state->ks_polling != 0) { kbdmux_kbd_t *k; SLIST_FOREACH(k, &state->ks_kbds, next) { while (kbdd_check_char(k->kbd)) { scancode = kbdd_read_char(k->kbd, 0); if (scancode == NOKEY) break; if (scancode == ERRKEY) continue; if (!KBD_IS_BUSY(k->kbd)) continue; kbdmux_kbd_putc(state, scancode); } } if (state->ks_inq_length > 0) goto next_code; } KBDMUX_UNLOCK(state); return (NOKEY); } /* XXX FIXME: check for -1 if wait == 1! */ kbd->kb_count ++; /* return the byte as is for the K_RAW mode */ if (state->ks_mode == K_RAW) { KBDMUX_UNLOCK(state); return (scancode); } /* translate the scan code into a keycode */ keycode = scancode & 0x7F; switch (state->ks_prefix) { case 0x00: /* normal scancode */ switch(scancode) { case 0xB8: /* left alt (compose key) released */ if (state->ks_flags & COMPOSE) { state->ks_flags &= ~COMPOSE; if (state->ks_composed_char > UCHAR_MAX) state->ks_composed_char = 0; } break; case 0x38: /* left alt (compose key) pressed */ if (!(state->ks_flags & COMPOSE)) { state->ks_flags |= COMPOSE; state->ks_composed_char = 0; } break; case 0xE0: case 0xE1: state->ks_prefix = scancode; goto next_code; } break; case 0xE0: /* 0xE0 prefix */ state->ks_prefix = 0; switch (keycode) { case 0x1C: /* right enter key */ keycode = 0x59; break; case 0x1D: /* right ctrl key */ keycode = 0x5A; break; case 0x35: /* keypad divide key */ keycode = 0x5B; break; case 0x37: /* print scrn key */ keycode = 0x5C; break; case 0x38: /* right alt key (alt gr) */ keycode = 0x5D; break; case 0x46: /* ctrl-pause/break on AT 101 (see below) */ keycode = 0x68; break; case 0x47: /* grey home key */ keycode = 0x5E; break; case 0x48: /* grey up arrow key */ keycode = 0x5F; break; case 0x49: /* grey page up key */ keycode = 0x60; break; case 0x4B: /* grey left arrow key */ keycode = 0x61; break; case 0x4D: /* grey right arrow key */ keycode = 0x62; break; case 0x4F: /* grey end key */ keycode = 0x63; break; case 0x50: /* grey down arrow key */ keycode = 0x64; break; case 0x51: /* grey page down key */ keycode = 0x65; break; case 0x52: /* grey insert key */ keycode = 0x66; break; case 0x53: /* grey delete key */ keycode = 0x67; break; /* the following 3 are only used on the MS "Natural" keyboard */ case 0x5b: /* left Window key */ keycode = 0x69; break; case 0x5c: /* right Window key */ keycode = 0x6a; break; case 0x5d: /* menu key */ keycode = 0x6b; break; case 0x5e: /* power key */ keycode = 0x6d; break; case 0x5f: /* sleep key */ keycode = 0x6e; break; case 0x63: /* wake key */ keycode = 0x6f; break; case 0x64: /* [JP106USB] backslash, underscore */ keycode = 0x73; break; default: /* ignore everything else */ goto next_code; } break; case 0xE1: /* 0xE1 prefix */ /* * The pause/break key on the 101 keyboard produces: * E1-1D-45 E1-9D-C5 * Ctrl-pause/break produces: * E0-46 E0-C6 (See above.) */ state->ks_prefix = 0; if (keycode == 0x1D) state->ks_prefix = 0x1D; goto next_code; /* NOT REACHED */ case 0x1D: /* pause / break */ state->ks_prefix = 0; if (keycode != 0x45) goto next_code; keycode = 0x68; break; } /* XXX assume 101/102 keys AT keyboard */ switch (keycode) { case 0x5c: /* print screen */ if (state->ks_flags & ALTS) keycode = 0x54; /* sysrq */ break; case 0x68: /* pause/break */ if (state->ks_flags & CTLS) keycode = 0x6c; /* break */ break; } /* return the key code in the K_CODE mode */ if (state->ks_mode == K_CODE) { KBDMUX_UNLOCK(state); return (keycode | (scancode & 0x80)); } /* compose a character code */ if (state->ks_flags & COMPOSE) { switch (keycode | (scancode & 0x80)) { /* key pressed, process it */ case 0x47: case 0x48: case 0x49: /* keypad 7,8,9 */ state->ks_composed_char *= 10; state->ks_composed_char += keycode - 0x40; if (state->ks_composed_char > UCHAR_MAX) { KBDMUX_UNLOCK(state); return (ERRKEY); } goto next_code; case 0x4B: case 0x4C: case 0x4D: /* keypad 4,5,6 */ state->ks_composed_char *= 10; state->ks_composed_char += keycode - 0x47; if (state->ks_composed_char > UCHAR_MAX) { KBDMUX_UNLOCK(state); return (ERRKEY); } goto next_code; case 0x4F: case 0x50: case 0x51: /* keypad 1,2,3 */ state->ks_composed_char *= 10; state->ks_composed_char += keycode - 0x4E; if (state->ks_composed_char > UCHAR_MAX) { KBDMUX_UNLOCK(state); return (ERRKEY); } goto next_code; case 0x52: /* keypad 0 */ state->ks_composed_char *= 10; if (state->ks_composed_char > UCHAR_MAX) { KBDMUX_UNLOCK(state); return (ERRKEY); } goto next_code; /* key released, no interest here */ case 0xC7: case 0xC8: case 0xC9: /* keypad 7,8,9 */ case 0xCB: case 0xCC: case 0xCD: /* keypad 4,5,6 */ case 0xCF: case 0xD0: case 0xD1: /* keypad 1,2,3 */ case 0xD2: /* keypad 0 */ goto next_code; case 0x38: /* left alt key */ break; default: if (state->ks_composed_char > 0) { state->ks_flags &= ~COMPOSE; state->ks_composed_char = 0; KBDMUX_UNLOCK(state); return (ERRKEY); } break; } } /* keycode to key action */ action = genkbd_keyaction(kbd, keycode, scancode & 0x80, &state->ks_state, &state->ks_accents); if (action == NOKEY) goto next_code; KBDMUX_UNLOCK(state); return (action); } /* * Check if char is waiting */ static int kbdmux_check_char(keyboard_t *kbd) { kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; int ready; if (!KBD_IS_ACTIVE(kbd)) return (FALSE); KBDMUX_LOCK(state); if (!(state->ks_flags & COMPOSE) && (state->ks_composed_char != 0)) ready = TRUE; else ready = (state->ks_inq_length > 0) ? TRUE : FALSE; KBDMUX_UNLOCK(state); return (ready); } /* * Keyboard ioctl's */ static int kbdmux_ioctl(keyboard_t *kbd, u_long cmd, caddr_t arg) { static int delays[] = { 250, 500, 750, 1000 }; static int rates[] = { 34, 38, 42, 46, 50, 55, 59, 63, 68, 76, 84, 92, 100, 110, 118, 126, 136, 152, 168, 184, 200, 220, 236, 252, 272, 304, 336, 368, 400, 440, 472, 504 }; kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; kbdmux_kbd_t *k; keyboard_info_t *ki; int error = 0, mode; #ifdef COMPAT_FREEBSD6 int ival; #endif if (state == NULL) return (ENXIO); switch (cmd) { case KBADDKBD: /* add keyboard to the mux */ ki = (keyboard_info_t *) arg; if (ki == NULL || ki->kb_unit < 0 || ki->kb_name[0] == '\0' || strcmp(ki->kb_name, "*") == 0) return (EINVAL); /* bad input */ KBDMUX_LOCK(state); SLIST_FOREACH(k, &state->ks_kbds, next) if (k->kbd->kb_unit == ki->kb_unit && strcmp(k->kbd->kb_name, ki->kb_name) == 0) break; if (k != NULL) { KBDMUX_UNLOCK(state); return (0); /* keyboard already in the mux */ } k = malloc(sizeof(*k), M_KBDMUX, M_NOWAIT | M_ZERO); if (k == NULL) { KBDMUX_UNLOCK(state); return (ENOMEM); /* out of memory */ } k->kbd = kbd_get_keyboard( kbd_allocate( ki->kb_name, ki->kb_unit, (void *) &k->kbd, kbdmux_kbd_event, (void *) state)); if (k->kbd == NULL) { KBDMUX_UNLOCK(state); free(k, M_KBDMUX); return (EINVAL); /* bad keyboard */ } kbdd_enable(k->kbd); kbdd_clear_state(k->kbd); /* set K_RAW mode on slave keyboard */ mode = K_RAW; error = kbdd_ioctl(k->kbd, KDSKBMODE, (caddr_t)&mode); if (error == 0) { /* set lock keys state on slave keyboard */ mode = state->ks_state & LOCK_MASK; error = kbdd_ioctl(k->kbd, KDSKBSTATE, (caddr_t)&mode); } if (error != 0) { KBDMUX_UNLOCK(state); kbd_release(k->kbd, &k->kbd); k->kbd = NULL; free(k, M_KBDMUX); return (error); /* could not set mode */ } SLIST_INSERT_HEAD(&state->ks_kbds, k, next); KBDMUX_UNLOCK(state); break; case KBRELKBD: /* release keyboard from the mux */ ki = (keyboard_info_t *) arg; if (ki == NULL || ki->kb_unit < 0 || ki->kb_name[0] == '\0' || strcmp(ki->kb_name, "*") == 0) return (EINVAL); /* bad input */ KBDMUX_LOCK(state); SLIST_FOREACH(k, &state->ks_kbds, next) if (k->kbd->kb_unit == ki->kb_unit && strcmp(k->kbd->kb_name, ki->kb_name) == 0) break; if (k != NULL) { error = kbd_release(k->kbd, &k->kbd); if (error == 0) { SLIST_REMOVE(&state->ks_kbds, k, kbdmux_kbd, next); k->kbd = NULL; free(k, M_KBDMUX); } } else error = ENXIO; /* keyboard is not in the mux */ KBDMUX_UNLOCK(state); break; case KDGKBMODE: /* get kyboard mode */ KBDMUX_LOCK(state); *(int *)arg = state->ks_mode; KBDMUX_UNLOCK(state); break; #ifdef COMPAT_FREEBSD6 case _IO('K', 7): ival = IOCPARM_IVAL(arg); arg = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSKBMODE: /* set keyboard mode */ KBDMUX_LOCK(state); switch (*(int *)arg) { case K_XLATE: if (state->ks_mode != K_XLATE) { /* make lock key state and LED state match */ state->ks_state &= ~LOCK_MASK; state->ks_state |= KBD_LED_VAL(kbd); } /* FALLTHROUGH */ case K_RAW: case K_CODE: if (state->ks_mode != *(int *)arg) { kbdmux_clear_state_locked(state); state->ks_mode = *(int *)arg; } break; default: error = EINVAL; break; } KBDMUX_UNLOCK(state); break; case KDGETLED: /* get keyboard LED */ KBDMUX_LOCK(state); *(int *)arg = KBD_LED_VAL(kbd); KBDMUX_UNLOCK(state); break; #ifdef COMPAT_FREEBSD6 case _IO('K', 66): ival = IOCPARM_IVAL(arg); arg = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSETLED: /* set keyboard LED */ KBDMUX_LOCK(state); /* NOTE: lock key state in ks_state won't be changed */ if (*(int *)arg & ~LOCK_MASK) { KBDMUX_UNLOCK(state); return (EINVAL); } KBD_LED_VAL(kbd) = *(int *)arg; /* KDSETLED on all slave keyboards */ SLIST_FOREACH(k, &state->ks_kbds, next) (void)kbdd_ioctl(k->kbd, KDSETLED, arg); KBDMUX_UNLOCK(state); break; case KDGKBSTATE: /* get lock key state */ KBDMUX_LOCK(state); *(int *)arg = state->ks_state & LOCK_MASK; KBDMUX_UNLOCK(state); break; #ifdef COMPAT_FREEBSD6 case _IO('K', 20): ival = IOCPARM_IVAL(arg); arg = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSKBSTATE: /* set lock key state */ KBDMUX_LOCK(state); if (*(int *)arg & ~LOCK_MASK) { KBDMUX_UNLOCK(state); return (EINVAL); } state->ks_state &= ~LOCK_MASK; state->ks_state |= *(int *)arg; /* KDSKBSTATE on all slave keyboards */ SLIST_FOREACH(k, &state->ks_kbds, next) (void)kbdd_ioctl(k->kbd, KDSKBSTATE, arg); KBDMUX_UNLOCK(state); return (kbdmux_ioctl(kbd, KDSETLED, arg)); /* NOT REACHED */ #ifdef COMPAT_FREEBSD6 case _IO('K', 67): cmd = KDSETRAD; ival = IOCPARM_IVAL(arg); arg = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSETREPEAT: /* set keyboard repeat rate (new interface) */ case KDSETRAD: /* set keyboard repeat rate (old interface) */ KBDMUX_LOCK(state); if (cmd == KDSETREPEAT) { int i; /* lookup delay */ for (i = sizeof(delays)/sizeof(delays[0]) - 1; i > 0; i --) if (((int *)arg)[0] >= delays[i]) break; mode = i << 5; /* lookup rate */ for (i = sizeof(rates)/sizeof(rates[0]) - 1; i > 0; i --) if (((int *)arg)[1] >= rates[i]) break; mode |= i; } else mode = *(int *)arg; if (mode & ~0x7f) { KBDMUX_UNLOCK(state); return (EINVAL); } kbd->kb_delay1 = delays[(mode >> 5) & 3]; kbd->kb_delay2 = rates[mode & 0x1f]; /* perform command on all slave keyboards */ SLIST_FOREACH(k, &state->ks_kbds, next) (void)kbdd_ioctl(k->kbd, cmd, arg); KBDMUX_UNLOCK(state); break; case PIO_KEYMAP: /* set keyboard translation table */ case OPIO_KEYMAP: /* set keyboard translation table (compat) */ case PIO_KEYMAPENT: /* set keyboard translation table entry */ case PIO_DEADKEYMAP: /* set accent key translation table */ KBDMUX_LOCK(state); state->ks_accents = 0; /* perform command on all slave keyboards */ SLIST_FOREACH(k, &state->ks_kbds, next) (void)kbdd_ioctl(k->kbd, cmd, arg); KBDMUX_UNLOCK(state); /* FALLTHROUGH */ default: error = genkbd_commonioctl(kbd, cmd, arg); break; } return (error); } /* * Lock the access to the keyboard */ static int kbdmux_lock(keyboard_t *kbd, int lock) { return (1); /* XXX */ } /* * Clear the internal state of the keyboard */ static void kbdmux_clear_state_locked(kbdmux_state_t *state) { KBDMUX_LOCK_ASSERT(state, MA_OWNED); - state->ks_flags &= ~(COMPOSE|POLLING); + state->ks_flags &= ~COMPOSE; + state->ks_polling = 0; state->ks_state &= LOCK_MASK; /* preserve locking key state */ state->ks_accents = 0; state->ks_composed_char = 0; /* state->ks_prefix = 0; XXX */ state->ks_inq_length = 0; } static void kbdmux_clear_state(keyboard_t *kbd) { kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; KBDMUX_LOCK(state); kbdmux_clear_state_locked(state); KBDMUX_UNLOCK(state); } /* * Save the internal state */ static int kbdmux_get_state(keyboard_t *kbd, void *buf, size_t len) { if (len == 0) return (sizeof(kbdmux_state_t)); if (len < sizeof(kbdmux_state_t)) return (-1); bcopy(kbd->kb_data, buf, sizeof(kbdmux_state_t)); /* XXX locking? */ return (0); } /* * Set the internal state */ static int kbdmux_set_state(keyboard_t *kbd, void *buf, size_t len) { if (len < sizeof(kbdmux_state_t)) return (ENOMEM); bcopy(buf, kbd->kb_data, sizeof(kbdmux_state_t)); /* XXX locking? */ return (0); } /* * Set polling */ static int kbdmux_poll(keyboard_t *kbd, int on) { kbdmux_state_t *state = (kbdmux_state_t *) kbd->kb_data; kbdmux_kbd_t *k; KBDMUX_LOCK(state); if (on) - state->ks_flags |= POLLING; + state->ks_polling++; else - state->ks_flags &= ~POLLING; + state->ks_polling--; /* set poll on slave keyboards */ SLIST_FOREACH(k, &state->ks_kbds, next) kbdd_poll(k->kbd, on); KBDMUX_UNLOCK(state); return (0); } /***************************************************************************** ***************************************************************************** ** Module ***************************************************************************** *****************************************************************************/ KEYBOARD_DRIVER(kbdmux, kbdmuxsw, kbdmux_configure); static int kbdmux_modevent(module_t mod, int type, void *data) { keyboard_switch_t *sw; keyboard_t *kbd; int error; switch (type) { case MOD_LOAD: if ((error = kbd_add_driver(&kbdmux_kbd_driver)) != 0) break; if ((sw = kbd_get_switch(KEYBOARD_NAME)) == NULL) { kbd_delete_driver(&kbdmux_kbd_driver); error = ENXIO; break; } kbd = NULL; if ((error = (*sw->probe)(0, NULL, 0)) != 0 || (error = (*sw->init)(0, &kbd, NULL, 0)) != 0) { kbd_delete_driver(&kbdmux_kbd_driver); break; } #ifdef KBD_INSTALL_CDEV if ((error = kbd_attach(kbd)) != 0) { (*sw->term)(kbd); kbd_delete_driver(&kbdmux_kbd_driver); break; } #endif if ((error = (*sw->enable)(kbd)) != 0) { (*sw->disable)(kbd); #ifdef KBD_INSTALL_CDEV kbd_detach(kbd); #endif (*sw->term)(kbd); kbd_delete_driver(&kbdmux_kbd_driver); break; } break; case MOD_UNLOAD: if ((sw = kbd_get_switch(KEYBOARD_NAME)) == NULL) panic("kbd_get_switch(" KEYBOARD_NAME ") == NULL"); kbd = kbd_get_keyboard(kbd_find_keyboard(KEYBOARD_NAME, 0)); if (kbd != NULL) { (*sw->disable)(kbd); #ifdef KBD_INSTALL_CDEV kbd_detach(kbd); #endif (*sw->term)(kbd); kbd_delete_driver(&kbdmux_kbd_driver); } error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } DEV_MODULE(kbdmux, kbdmux_modevent, NULL); Index: projects/clang390-import/sys/dev/syscons/syscons.c =================================================================== --- projects/clang390-import/sys/dev/syscons/syscons.c (revision 305080) +++ projects/clang390-import/sys/dev/syscons/syscons.c (revision 305081) @@ -1,4004 +1,4011 @@ /*- * Copyright (c) 1992-1998 Søren Schmidt * All rights reserved. * * This code is derived from software contributed to The DragonFly Project * by Sascha Wildner * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include "opt_syscons.h" #include "opt_splash.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__arm__) || defined(__mips__) || \ defined(__powerpc__) || defined(__sparc64__) #include #else #include #endif #if defined( __i386__) || defined(__amd64__) #include #include #endif #include #include #include #include #include #define COLD 0 #define WARM 1 #define DEFAULT_BLANKTIME (5*60) /* 5 minutes */ #define MAX_BLANKTIME (7*24*60*60) /* 7 days!? */ #define KEYCODE_BS 0x0e /* "<-- Backspace" key, XXX */ /* NULL-safe version of "tty_opened()" */ #define tty_opened_ns(tp) ((tp) != NULL && tty_opened(tp)) typedef struct default_attr { int std_color; /* normal hardware color */ int rev_color; /* reverse hardware color */ } default_attr; static default_attr user_default = { SC_NORM_ATTR, SC_NORM_REV_ATTR, }; static int sc_console_unit = -1; static int sc_saver_keyb_only = 1; static scr_stat *sc_console; static struct consdev *sc_consptr; static scr_stat main_console; static struct tty *main_devs[MAXCONS]; static char init_done = COLD; static int shutdown_in_progress = FALSE; static int suspend_in_progress = FALSE; static char sc_malloc = FALSE; static int saver_mode = CONS_NO_SAVER; /* LKM/user saver */ static int run_scrn_saver = FALSE; /* should run the saver? */ static int enable_bell = TRUE; /* enable beeper */ #ifndef SC_DISABLE_REBOOT static int enable_reboot = TRUE; /* enable keyboard reboot */ #endif #ifndef SC_DISABLE_KDBKEY static int enable_kdbkey = TRUE; /* enable keyboard debug */ #endif static long scrn_blank_time = 0; /* screen saver timeout value */ #ifdef DEV_SPLASH static int scrn_blanked; /* # of blanked screen */ static int sticky_splash = FALSE; static void none_saver(sc_softc_t *sc, int blank) { } static void (*current_saver)(sc_softc_t *, int) = none_saver; #endif #ifdef SC_NO_SUSPEND_VTYSWITCH static int sc_no_suspend_vtswitch = 1; #else static int sc_no_suspend_vtswitch = 0; #endif static int sc_susp_scr; static SYSCTL_NODE(_hw, OID_AUTO, syscons, CTLFLAG_RD, 0, "syscons"); static SYSCTL_NODE(_hw_syscons, OID_AUTO, saver, CTLFLAG_RD, 0, "saver"); SYSCTL_INT(_hw_syscons_saver, OID_AUTO, keybonly, CTLFLAG_RW, &sc_saver_keyb_only, 0, "screen saver interrupted by input only"); SYSCTL_INT(_hw_syscons, OID_AUTO, bell, CTLFLAG_RW, &enable_bell, 0, "enable bell"); #ifndef SC_DISABLE_REBOOT SYSCTL_INT(_hw_syscons, OID_AUTO, kbd_reboot, CTLFLAG_RW|CTLFLAG_SECURE, &enable_reboot, 0, "enable keyboard reboot"); #endif #ifndef SC_DISABLE_KDBKEY SYSCTL_INT(_hw_syscons, OID_AUTO, kbd_debug, CTLFLAG_RW|CTLFLAG_SECURE, &enable_kdbkey, 0, "enable keyboard debug"); #endif SYSCTL_INT(_hw_syscons, OID_AUTO, sc_no_suspend_vtswitch, CTLFLAG_RWTUN, &sc_no_suspend_vtswitch, 0, "Disable VT switch before suspend."); #if !defined(SC_NO_FONT_LOADING) && defined(SC_DFLT_FONT) #include "font.h" #endif tsw_ioctl_t *sc_user_ioctl; static bios_values_t bios_value; static int enable_panic_key; SYSCTL_INT(_machdep, OID_AUTO, enable_panic_key, CTLFLAG_RW, &enable_panic_key, 0, "Enable panic via keypress specified in kbdmap(5)"); #define SC_CONSOLECTL 255 #define VTY_WCHAN(sc, vty) (&SC_DEV(sc, vty)) /* prototypes */ static int sc_allocate_keyboard(sc_softc_t *sc, int unit); static int scvidprobe(int unit, int flags, int cons); static int sckbdprobe(int unit, int flags, int cons); static void scmeminit(void *arg); static int scdevtounit(struct tty *tp); static kbd_callback_func_t sckbdevent; static void scinit(int unit, int flags); static scr_stat *sc_get_stat(struct tty *tp); static void scterm(int unit, int flags); static void scshutdown(void *, int); static void scsuspend(void *); static void scresume(void *); static u_int scgetc(sc_softc_t *sc, u_int flags, struct sc_cnstate *sp); static void sc_puts(scr_stat *scp, u_char *buf, int len, int kernel); #define SCGETC_CN 1 #define SCGETC_NONBLOCK 2 static void sccnupdate(scr_stat *scp); static scr_stat *alloc_scp(sc_softc_t *sc, int vty); static void init_scp(sc_softc_t *sc, int vty, scr_stat *scp); static timeout_t scrn_timer; static int and_region(int *s1, int *e1, int s2, int e2); static void scrn_update(scr_stat *scp, int show_cursor); #ifdef DEV_SPLASH static int scsplash_callback(int event, void *arg); static void scsplash_saver(sc_softc_t *sc, int show); static int add_scrn_saver(void (*this_saver)(sc_softc_t *, int)); static int remove_scrn_saver(void (*this_saver)(sc_softc_t *, int)); static int set_scrn_saver_mode(scr_stat *scp, int mode, u_char *pal, int border); static int restore_scrn_saver_mode(scr_stat *scp, int changemode); static void stop_scrn_saver(sc_softc_t *sc, void (*saver)(sc_softc_t *, int)); static int wait_scrn_saver_stop(sc_softc_t *sc); #define scsplash_stick(stick) (sticky_splash = (stick)) #else /* !DEV_SPLASH */ #define scsplash_stick(stick) #endif /* DEV_SPLASH */ static int do_switch_scr(sc_softc_t *sc, int s); static int vt_proc_alive(scr_stat *scp); static int signal_vt_rel(scr_stat *scp); static int signal_vt_acq(scr_stat *scp); static int finish_vt_rel(scr_stat *scp, int release, int *s); static int finish_vt_acq(scr_stat *scp); static void exchange_scr(sc_softc_t *sc); static void update_cursor_image(scr_stat *scp); static void change_cursor_shape(scr_stat *scp, int flags, int base, int height); static void update_font(scr_stat *); static int save_kbd_state(scr_stat *scp); static int update_kbd_state(scr_stat *scp, int state, int mask); static int update_kbd_leds(scr_stat *scp, int which); static timeout_t blink_screen; static struct tty *sc_alloc_tty(int, int); static cn_probe_t sc_cnprobe; static cn_init_t sc_cninit; static cn_term_t sc_cnterm; static cn_getc_t sc_cngetc; static cn_putc_t sc_cnputc; static cn_grab_t sc_cngrab; static cn_ungrab_t sc_cnungrab; CONSOLE_DRIVER(sc); static tsw_open_t sctty_open; static tsw_close_t sctty_close; static tsw_outwakeup_t sctty_outwakeup; static tsw_ioctl_t sctty_ioctl; static tsw_mmap_t sctty_mmap; static struct ttydevsw sc_ttydevsw = { .tsw_open = sctty_open, .tsw_close = sctty_close, .tsw_outwakeup = sctty_outwakeup, .tsw_ioctl = sctty_ioctl, .tsw_mmap = sctty_mmap, }; static d_ioctl_t consolectl_ioctl; static d_close_t consolectl_close; static struct cdevsw consolectl_devsw = { .d_version = D_VERSION, .d_flags = D_NEEDGIANT | D_TRACKCLOSE, .d_ioctl = consolectl_ioctl, .d_close = consolectl_close, .d_name = "consolectl", }; int sc_probe_unit(int unit, int flags) { if (!vty_enabled(VTY_SC)) return ENXIO; if (!scvidprobe(unit, flags, FALSE)) { if (bootverbose) printf("%s%d: no video adapter found.\n", SC_DRIVER_NAME, unit); return ENXIO; } /* syscons will be attached even when there is no keyboard */ sckbdprobe(unit, flags, FALSE); return 0; } /* probe video adapters, return TRUE if found */ static int scvidprobe(int unit, int flags, int cons) { /* * Access the video adapter driver through the back door! * Video adapter drivers need to be configured before syscons. * However, when syscons is being probed as the low-level console, * they have not been initialized yet. We force them to initialize * themselves here. XXX */ vid_configure(cons ? VIO_PROBE_ONLY : 0); return (vid_find_adapter("*", unit) >= 0); } /* probe the keyboard, return TRUE if found */ static int sckbdprobe(int unit, int flags, int cons) { /* access the keyboard driver through the backdoor! */ kbd_configure(cons ? KB_CONF_PROBE_ONLY : 0); return (kbd_find_keyboard("*", unit) >= 0); } static char *adapter_name(video_adapter_t *adp) { static struct { int type; char *name[2]; } names[] = { { KD_MONO, { "MDA", "MDA" } }, { KD_HERCULES, { "Hercules", "Hercules" } }, { KD_CGA, { "CGA", "CGA" } }, { KD_EGA, { "EGA", "EGA (mono)" } }, { KD_VGA, { "VGA", "VGA (mono)" } }, { KD_PC98, { "PC-98x1", "PC-98x1" } }, { KD_TGA, { "TGA", "TGA" } }, { -1, { "Unknown", "Unknown" } }, }; int i; for (i = 0; names[i].type != -1; ++i) if (names[i].type == adp->va_type) break; return names[i].name[(adp->va_flags & V_ADP_COLOR) ? 0 : 1]; } static void sctty_outwakeup(struct tty *tp) { size_t len; u_char buf[PCBURST]; scr_stat *scp = sc_get_stat(tp); if (scp->status & SLKED || (scp == scp->sc->cur_scp && scp->sc->blink_in_progress)) return; for (;;) { len = ttydisc_getc(tp, buf, sizeof buf); if (len == 0) break; SC_VIDEO_LOCK(scp->sc); sc_puts(scp, buf, len, 0); SC_VIDEO_UNLOCK(scp->sc); } } static struct tty * sc_alloc_tty(int index, int devnum) { struct sc_ttysoftc *stc; struct tty *tp; /* Allocate TTY object and softc to store unit number. */ stc = malloc(sizeof(struct sc_ttysoftc), M_DEVBUF, M_WAITOK); stc->st_index = index; stc->st_stat = NULL; tp = tty_alloc_mutex(&sc_ttydevsw, stc, &Giant); /* Create device node. */ tty_makedev(tp, NULL, "v%r", devnum); return (tp); } #ifdef SC_PIXEL_MODE static void sc_set_vesa_mode(scr_stat *scp, sc_softc_t *sc, int unit) { video_info_t info; u_char *font; int depth; int fontsize; int i; int vmode; vmode = 0; (void)resource_int_value("sc", unit, "vesa_mode", &vmode); if (vmode < M_VESA_BASE || vmode > M_VESA_MODE_MAX || vidd_get_info(sc->adp, vmode, &info) != 0 || !sc_support_pixel_mode(&info)) vmode = 0; /* * If the mode is unset or unsupported, search for an available * 800x600 graphics mode with the highest color depth. */ if (vmode == 0) { for (depth = 0, i = M_VESA_BASE; i <= M_VESA_MODE_MAX; i++) if (vidd_get_info(sc->adp, i, &info) == 0 && info.vi_width == 800 && info.vi_height == 600 && sc_support_pixel_mode(&info) && info.vi_depth > depth) { vmode = i; depth = info.vi_depth; } if (vmode == 0) return; vidd_get_info(sc->adp, vmode, &info); } #if !defined(SC_NO_FONT_LOADING) && defined(SC_DFLT_FONT) fontsize = info.vi_cheight; #else fontsize = scp->font_size; #endif if (fontsize < 14) fontsize = 8; else if (fontsize >= 16) fontsize = 16; else fontsize = 14; #ifndef SC_NO_FONT_LOADING switch (fontsize) { case 8: if ((sc->fonts_loaded & FONT_8) == 0) return; font = sc->font_8; break; case 14: if ((sc->fonts_loaded & FONT_14) == 0) return; font = sc->font_14; break; case 16: if ((sc->fonts_loaded & FONT_16) == 0) return; font = sc->font_16; break; } #else font = NULL; #endif #ifdef DEV_SPLASH if ((sc->flags & SC_SPLASH_SCRN) != 0) splash_term(sc->adp); #endif #ifndef SC_NO_HISTORY if (scp->history != NULL) { sc_vtb_append(&scp->vtb, 0, scp->history, scp->ypos * scp->xsize + scp->xpos); scp->history_pos = sc_vtb_tail(scp->history); } #endif vidd_set_mode(sc->adp, vmode); scp->status |= (UNKNOWN_MODE | PIXEL_MODE | MOUSE_HIDDEN); scp->status &= ~(GRAPHICS_MODE | MOUSE_VISIBLE); scp->xpixel = info.vi_width; scp->ypixel = info.vi_height; scp->xsize = scp->xpixel / 8; scp->ysize = scp->ypixel / fontsize; scp->xpos = 0; scp->ypos = scp->ysize - 1; scp->xoff = scp->yoff = 0; scp->font = font; scp->font_size = fontsize; scp->font_width = 8; scp->start = scp->xsize * scp->ysize - 1; scp->end = 0; scp->cursor_pos = scp->cursor_oldpos = scp->xsize * scp->xsize; scp->mode = sc->initial_mode = vmode; #ifndef __sparc64__ sc_vtb_init(&scp->scr, VTB_FRAMEBUFFER, scp->xsize, scp->ysize, (void *)sc->adp->va_window, FALSE); #endif sc_alloc_scr_buffer(scp, FALSE, FALSE); sc_init_emulator(scp, NULL); #ifndef SC_NO_CUTPASTE sc_alloc_cut_buffer(scp, FALSE); #endif #ifndef SC_NO_HISTORY sc_alloc_history_buffer(scp, 0, 0, FALSE); #endif sc_set_border(scp, scp->border); sc_set_cursor_image(scp); scp->status &= ~UNKNOWN_MODE; #ifdef DEV_SPLASH if ((sc->flags & SC_SPLASH_SCRN) != 0) splash_init(sc->adp, scsplash_callback, sc); #endif } #endif int sc_attach_unit(int unit, int flags) { sc_softc_t *sc; scr_stat *scp; struct cdev *dev; int vc; if (!vty_enabled(VTY_SC)) return ENXIO; flags &= ~SC_KERNEL_CONSOLE; if (sc_console_unit == unit) { /* * If this unit is being used as the system console, we need to * adjust some variables and buffers before and after scinit(). */ /* assert(sc_console != NULL) */ flags |= SC_KERNEL_CONSOLE; scmeminit(NULL); } scinit(unit, flags); sc = sc_get_softc(unit, flags & SC_KERNEL_CONSOLE); sc->config = flags; callout_init(&sc->ctimeout, 0); callout_init(&sc->cblink, 0); scp = sc_get_stat(sc->dev[0]); if (sc_console == NULL) /* sc_console_unit < 0 */ sc_console = scp; #ifdef SC_PIXEL_MODE if ((sc->config & SC_VESAMODE) != 0) sc_set_vesa_mode(scp, sc, unit); #endif /* SC_PIXEL_MODE */ /* initialize cursor */ if (!ISGRAPHSC(scp)) update_cursor_image(scp); /* get screen update going */ scrn_timer(sc); /* set up the keyboard */ (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&scp->kbd_mode); update_kbd_state(scp, scp->status, LOCK_MASK); printf("%s%d: %s <%d virtual consoles, flags=0x%x>\n", SC_DRIVER_NAME, unit, adapter_name(sc->adp), sc->vtys, sc->config); if (bootverbose) { printf("%s%d:", SC_DRIVER_NAME, unit); if (sc->adapter >= 0) printf(" fb%d", sc->adapter); if (sc->keyboard >= 0) printf(", kbd%d", sc->keyboard); if (scp->tsw) printf(", terminal emulator: %s (%s)", scp->tsw->te_name, scp->tsw->te_desc); printf("\n"); } /* Register suspend/resume/shutdown callbacks for the kernel console. */ if (sc_console_unit == unit) { EVENTHANDLER_REGISTER(power_suspend_early, scsuspend, NULL, EVENTHANDLER_PRI_ANY); EVENTHANDLER_REGISTER(power_resume, scresume, NULL, EVENTHANDLER_PRI_ANY); EVENTHANDLER_REGISTER(shutdown_pre_sync, scshutdown, NULL, SHUTDOWN_PRI_DEFAULT); } for (vc = 0; vc < sc->vtys; vc++) { if (sc->dev[vc] == NULL) { sc->dev[vc] = sc_alloc_tty(vc, vc + unit * MAXCONS); if (vc == 0 && sc->dev == main_devs) SC_STAT(sc->dev[0]) = &main_console; } /* * The first vty already has struct tty and scr_stat initialized * in scinit(). The other vtys will have these structs when * first opened. */ } dev = make_dev(&consolectl_devsw, 0, UID_ROOT, GID_WHEEL, 0600, "consolectl"); dev->si_drv1 = sc->dev[0]; return 0; } static void scmeminit(void *arg) { if (!vty_enabled(VTY_SC)) return; if (sc_malloc) return; sc_malloc = TRUE; /* * As soon as malloc() becomes functional, we had better allocate * various buffers for the kernel console. */ if (sc_console_unit < 0) /* sc_console == NULL */ return; /* copy the temporary buffer to the final buffer */ sc_alloc_scr_buffer(sc_console, FALSE, FALSE); #ifndef SC_NO_CUTPASTE sc_alloc_cut_buffer(sc_console, FALSE); #endif #ifndef SC_NO_HISTORY /* initialize history buffer & pointers */ sc_alloc_history_buffer(sc_console, 0, 0, FALSE); #endif } /* XXX */ SYSINIT(sc_mem, SI_SUB_KMEM, SI_ORDER_ANY, scmeminit, NULL); static int scdevtounit(struct tty *tp) { int vty = SC_VTY(tp); if (vty == SC_CONSOLECTL) return ((sc_console != NULL) ? sc_console->sc->unit : -1); else if ((vty < 0) || (vty >= MAXCONS*sc_max_unit())) return -1; else return vty/MAXCONS; } static int sctty_open(struct tty *tp) { int unit = scdevtounit(tp); sc_softc_t *sc; scr_stat *scp; #ifndef __sparc64__ keyarg_t key; #endif DPRINTF(5, ("scopen: dev:%s, unit:%d, vty:%d\n", devtoname(tp->t_dev), unit, SC_VTY(tp))); sc = sc_get_softc(unit, (sc_console_unit == unit) ? SC_KERNEL_CONSOLE : 0); if (sc == NULL) return ENXIO; if (!tty_opened(tp)) { /* Use the current setting of the <-- key as default VERASE. */ /* If the Delete key is preferable, an stty is necessary */ #ifndef __sparc64__ if (sc->kbd != NULL) { key.keynum = KEYCODE_BS; (void)kbdd_ioctl(sc->kbd, GIO_KEYMAPENT, (caddr_t)&key); tp->t_termios.c_cc[VERASE] = key.key.map[0]; } #endif } scp = sc_get_stat(tp); if (scp == NULL) { scp = SC_STAT(tp) = alloc_scp(sc, SC_VTY(tp)); if (ISGRAPHSC(scp)) sc_set_pixel_mode(scp, NULL, 0, 0, 16, 8); } if (!tp->t_winsize.ws_col && !tp->t_winsize.ws_row) { tp->t_winsize.ws_col = scp->xsize; tp->t_winsize.ws_row = scp->ysize; } return (0); } static void sctty_close(struct tty *tp) { scr_stat *scp; int s; if (SC_VTY(tp) != SC_CONSOLECTL) { scp = sc_get_stat(tp); /* were we in the middle of the VT switching process? */ DPRINTF(5, ("sc%d: scclose(), ", scp->sc->unit)); s = spltty(); if ((scp == scp->sc->cur_scp) && (scp->sc->unit == sc_console_unit)) cnavailable(sc_consptr, TRUE); if (finish_vt_rel(scp, TRUE, &s) == 0) /* force release */ DPRINTF(5, ("reset WAIT_REL, ")); if (finish_vt_acq(scp) == 0) /* force acknowledge */ DPRINTF(5, ("reset WAIT_ACQ, ")); #ifdef not_yet_done if (scp == &main_console) { scp->pid = 0; scp->proc = NULL; scp->smode.mode = VT_AUTO; } else { sc_vtb_destroy(&scp->vtb); #ifndef __sparc64__ sc_vtb_destroy(&scp->scr); #endif sc_free_history_buffer(scp, scp->ysize); SC_STAT(tp) = NULL; free(scp, M_DEVBUF); } #else scp->pid = 0; scp->proc = NULL; scp->smode.mode = VT_AUTO; #endif scp->kbd_mode = K_XLATE; if (scp == scp->sc->cur_scp) (void)kbdd_ioctl(scp->sc->kbd, KDSKBMODE, (caddr_t)&scp->kbd_mode); DPRINTF(5, ("done.\n")); } } #if 0 /* XXX mpsafetty: fix screensaver. What about outwakeup? */ static int scread(struct cdev *dev, struct uio *uio, int flag) { if (!sc_saver_keyb_only) sc_touch_scrn_saver(); return ttyread(dev, uio, flag); } #endif static int sckbdevent(keyboard_t *thiskbd, int event, void *arg) { sc_softc_t *sc; struct tty *cur_tty; int c, error = 0; size_t len; const u_char *cp; sc = (sc_softc_t *)arg; /* assert(thiskbd == sc->kbd) */ mtx_lock(&Giant); switch (event) { case KBDIO_KEYINPUT: break; case KBDIO_UNLOADING: sc->kbd = NULL; sc->keyboard = -1; kbd_release(thiskbd, (void *)&sc->keyboard); goto done; default: error = EINVAL; goto done; } /* * Loop while there is still input to get from the keyboard. * I don't think this is nessesary, and it doesn't fix * the Xaccel-2.1 keyboard hang, but it can't hurt. XXX */ while ((c = scgetc(sc, SCGETC_NONBLOCK, NULL)) != NOKEY) { cur_tty = SC_DEV(sc, sc->cur_scp->index); if (!tty_opened_ns(cur_tty)) continue; if ((*sc->cur_scp->tsw->te_input)(sc->cur_scp, c, cur_tty)) continue; switch (KEYFLAGS(c)) { case 0x0000: /* normal key */ ttydisc_rint(cur_tty, KEYCHAR(c), 0); break; case FKEY: /* function key, return string */ cp = (*sc->cur_scp->tsw->te_fkeystr)(sc->cur_scp, c); if (cp != NULL) { ttydisc_rint_simple(cur_tty, cp, strlen(cp)); break; } cp = kbdd_get_fkeystr(thiskbd, KEYCHAR(c), &len); if (cp != NULL) ttydisc_rint_simple(cur_tty, cp, len); break; case MKEY: /* meta is active, prepend ESC */ ttydisc_rint(cur_tty, 0x1b, 0); ttydisc_rint(cur_tty, KEYCHAR(c), 0); break; case BKEY: /* backtab fixed sequence (esc [ Z) */ ttydisc_rint_simple(cur_tty, "\x1B[Z", 3); break; } ttydisc_rint_done(cur_tty); } sc->cur_scp->status |= MOUSE_HIDDEN; done: mtx_unlock(&Giant); return (error); } static int sctty_ioctl(struct tty *tp, u_long cmd, caddr_t data, struct thread *td) { int error; int i; sc_softc_t *sc; scr_stat *scp; int s; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) int ival; #endif /* If there is a user_ioctl function call that first */ if (sc_user_ioctl) { error = (*sc_user_ioctl)(tp, cmd, data, td); if (error != ENOIOCTL) return error; } error = sc_vid_ioctl(tp, cmd, data, td); if (error != ENOIOCTL) return error; #ifndef SC_NO_HISTORY error = sc_hist_ioctl(tp, cmd, data, td); if (error != ENOIOCTL) return error; #endif #ifndef SC_NO_SYSMOUSE error = sc_mouse_ioctl(tp, cmd, data, td); if (error != ENOIOCTL) return error; #endif scp = sc_get_stat(tp); /* assert(scp != NULL) */ /* scp is sc_console, if SC_VTY(dev) == SC_CONSOLECTL. */ sc = scp->sc; if (scp->tsw) { error = (*scp->tsw->te_ioctl)(scp, tp, cmd, data, td); if (error != ENOIOCTL) return error; } switch (cmd) { /* process console hardware related ioctl's */ case GIO_ATTR: /* get current attributes */ /* this ioctl is not processed here, but in the terminal emulator */ return ENOTTY; case GIO_COLOR: /* is this a color console ? */ *(int *)data = (sc->adp->va_flags & V_ADP_COLOR) ? 1 : 0; return 0; case CONS_BLANKTIME: /* set screen saver timeout (0 = no saver) */ if (*(int *)data < 0 || *(int *)data > MAX_BLANKTIME) return EINVAL; s = spltty(); scrn_blank_time = *(int *)data; run_scrn_saver = (scrn_blank_time != 0); splx(s); return 0; case CONS_CURSORTYPE: /* set cursor type (obsolete) */ s = spltty(); *(int *)data &= CONS_CURSOR_ATTRS; sc_change_cursor_shape(scp, *(int *)data, -1, -1); splx(s); return 0; case CONS_GETCURSORSHAPE: /* get cursor shape (new interface) */ if (((int *)data)[0] & CONS_LOCAL_CURSOR) { ((int *)data)[0] = scp->curr_curs_attr.flags; ((int *)data)[1] = scp->curr_curs_attr.base; ((int *)data)[2] = scp->curr_curs_attr.height; } else { ((int *)data)[0] = sc->curs_attr.flags; ((int *)data)[1] = sc->curs_attr.base; ((int *)data)[2] = sc->curs_attr.height; } return 0; case CONS_SETCURSORSHAPE: /* set cursor shape (new interface) */ s = spltty(); sc_change_cursor_shape(scp, ((int *)data)[0], ((int *)data)[1], ((int *)data)[2]); splx(s); return 0; case CONS_BELLTYPE: /* set bell type sound/visual */ if ((*(int *)data) & CONS_VISUAL_BELL) sc->flags |= SC_VISUAL_BELL; else sc->flags &= ~SC_VISUAL_BELL; if ((*(int *)data) & CONS_QUIET_BELL) sc->flags |= SC_QUIET_BELL; else sc->flags &= ~SC_QUIET_BELL; return 0; case CONS_GETINFO: /* get current (virtual) console info */ { vid_info_t *ptr = (vid_info_t*)data; if (ptr->size == sizeof(struct vid_info)) { ptr->m_num = sc->cur_scp->index; ptr->font_size = scp->font_size; ptr->mv_col = scp->xpos; ptr->mv_row = scp->ypos; ptr->mv_csz = scp->xsize; ptr->mv_rsz = scp->ysize; ptr->mv_hsz = (scp->history != NULL) ? scp->history->vtb_rows : 0; /* * The following fields are filled by the terminal emulator. XXX * * ptr->mv_norm.fore * ptr->mv_norm.back * ptr->mv_rev.fore * ptr->mv_rev.back */ ptr->mv_grfc.fore = 0; /* not supported */ ptr->mv_grfc.back = 0; /* not supported */ ptr->mv_ovscan = scp->border; if (scp == sc->cur_scp) save_kbd_state(scp); ptr->mk_keylock = scp->status & LOCK_MASK; return 0; } return EINVAL; } case CONS_GETVERS: /* get version number */ *(int*)data = 0x200; /* version 2.0 */ return 0; case CONS_IDLE: /* see if the screen has been idle */ /* * When the screen is in the GRAPHICS_MODE or UNKNOWN_MODE, * the user process may have been writing something on the * screen and syscons is not aware of it. Declare the screen * is NOT idle if it is in one of these modes. But there is * an exception to it; if a screen saver is running in the * graphics mode in the current screen, we should say that the * screen has been idle. */ *(int *)data = (sc->flags & SC_SCRN_IDLE) && (!ISGRAPHSC(sc->cur_scp) || (sc->cur_scp->status & SAVER_RUNNING)); return 0; case CONS_SAVERMODE: /* set saver mode */ switch(*(int *)data) { case CONS_NO_SAVER: case CONS_USR_SAVER: /* if a LKM screen saver is running, stop it first. */ scsplash_stick(FALSE); saver_mode = *(int *)data; s = spltty(); #ifdef DEV_SPLASH if ((error = wait_scrn_saver_stop(NULL))) { splx(s); return error; } #endif run_scrn_saver = TRUE; if (saver_mode == CONS_USR_SAVER) scp->status |= SAVER_RUNNING; else scp->status &= ~SAVER_RUNNING; scsplash_stick(TRUE); splx(s); break; case CONS_LKM_SAVER: s = spltty(); if ((saver_mode == CONS_USR_SAVER) && (scp->status & SAVER_RUNNING)) scp->status &= ~SAVER_RUNNING; saver_mode = *(int *)data; splx(s); break; default: return EINVAL; } return 0; case CONS_SAVERSTART: /* immediately start/stop the screen saver */ /* * Note that this ioctl does not guarantee the screen saver * actually starts or stops. It merely attempts to do so... */ s = spltty(); run_scrn_saver = (*(int *)data != 0); if (run_scrn_saver) sc->scrn_time_stamp -= scrn_blank_time; splx(s); return 0; case CONS_SCRSHOT: /* get a screen shot */ { int retval, hist_rsz; size_t lsize, csize; vm_offset_t frbp, hstp; unsigned lnum; scrshot_t *ptr = (scrshot_t *)data; void *outp = ptr->buf; if (ptr->x < 0 || ptr->y < 0 || ptr->xsize < 0 || ptr->ysize < 0) return EINVAL; s = spltty(); if (ISGRAPHSC(scp)) { splx(s); return EOPNOTSUPP; } hist_rsz = (scp->history != NULL) ? scp->history->vtb_rows : 0; if (((u_int)ptr->x + ptr->xsize) > scp->xsize || ((u_int)ptr->y + ptr->ysize) > (scp->ysize + hist_rsz)) { splx(s); return EINVAL; } lsize = scp->xsize * sizeof(u_int16_t); csize = ptr->xsize * sizeof(u_int16_t); /* Pointer to the last line of framebuffer */ frbp = scp->vtb.vtb_buffer + scp->ysize * lsize + ptr->x * sizeof(u_int16_t); /* Pointer to the last line of target buffer */ outp = (char *)outp + ptr->ysize * csize; /* Pointer to the last line of history buffer */ if (scp->history != NULL) hstp = scp->history->vtb_buffer + sc_vtb_tail(scp->history) * sizeof(u_int16_t) + ptr->x * sizeof(u_int16_t); else hstp = 0; retval = 0; for (lnum = 0; lnum < (ptr->y + ptr->ysize); lnum++) { if (lnum < scp->ysize) { frbp -= lsize; } else { hstp -= lsize; if (hstp < scp->history->vtb_buffer) hstp += scp->history->vtb_rows * lsize; frbp = hstp; } if (lnum < ptr->y) continue; outp = (char *)outp - csize; retval = copyout((void *)frbp, outp, csize); if (retval != 0) break; } splx(s); return retval; } case VT_SETMODE: /* set screen switcher mode */ { struct vt_mode *mode; struct proc *p1; mode = (struct vt_mode *)data; DPRINTF(5, ("%s%d: VT_SETMODE ", SC_DRIVER_NAME, sc->unit)); if (scp->smode.mode == VT_PROCESS) { p1 = pfind(scp->pid); if (scp->proc == p1 && scp->proc != td->td_proc) { if (p1) PROC_UNLOCK(p1); DPRINTF(5, ("error EPERM\n")); return EPERM; } if (p1) PROC_UNLOCK(p1); } s = spltty(); if (mode->mode == VT_AUTO) { scp->smode.mode = VT_AUTO; scp->proc = NULL; scp->pid = 0; DPRINTF(5, ("VT_AUTO, ")); if ((scp == sc->cur_scp) && (sc->unit == sc_console_unit)) cnavailable(sc_consptr, TRUE); /* were we in the middle of the vty switching process? */ if (finish_vt_rel(scp, TRUE, &s) == 0) DPRINTF(5, ("reset WAIT_REL, ")); if (finish_vt_acq(scp) == 0) DPRINTF(5, ("reset WAIT_ACQ, ")); } else { if (!ISSIGVALID(mode->relsig) || !ISSIGVALID(mode->acqsig) || !ISSIGVALID(mode->frsig)) { splx(s); DPRINTF(5, ("error EINVAL\n")); return EINVAL; } DPRINTF(5, ("VT_PROCESS %d, ", td->td_proc->p_pid)); bcopy(data, &scp->smode, sizeof(struct vt_mode)); scp->proc = td->td_proc; scp->pid = scp->proc->p_pid; if ((scp == sc->cur_scp) && (sc->unit == sc_console_unit)) cnavailable(sc_consptr, FALSE); } splx(s); DPRINTF(5, ("\n")); return 0; } case VT_GETMODE: /* get screen switcher mode */ bcopy(&scp->smode, data, sizeof(struct vt_mode)); return 0; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('v', 4): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case VT_RELDISP: /* screen switcher ioctl */ s = spltty(); /* * This must be the current vty which is in the VT_PROCESS * switching mode... */ if ((scp != sc->cur_scp) || (scp->smode.mode != VT_PROCESS)) { splx(s); return EINVAL; } /* ...and this process is controlling it. */ if (scp->proc != td->td_proc) { splx(s); return EPERM; } error = EINVAL; switch(*(int *)data) { case VT_FALSE: /* user refuses to release screen, abort */ if ((error = finish_vt_rel(scp, FALSE, &s)) == 0) DPRINTF(5, ("%s%d: VT_FALSE\n", SC_DRIVER_NAME, sc->unit)); break; case VT_TRUE: /* user has released screen, go on */ if ((error = finish_vt_rel(scp, TRUE, &s)) == 0) DPRINTF(5, ("%s%d: VT_TRUE\n", SC_DRIVER_NAME, sc->unit)); break; case VT_ACKACQ: /* acquire acknowledged, switch completed */ if ((error = finish_vt_acq(scp)) == 0) DPRINTF(5, ("%s%d: VT_ACKACQ\n", SC_DRIVER_NAME, sc->unit)); break; default: break; } splx(s); return error; case VT_OPENQRY: /* return free virtual console */ for (i = sc->first_vty; i < sc->first_vty + sc->vtys; i++) { tp = SC_DEV(sc, i); if (!tty_opened_ns(tp)) { *(int *)data = i + 1; return 0; } } return EINVAL; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('v', 5): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case VT_ACTIVATE: /* switch to screen *data */ i = (*(int *)data == 0) ? scp->index : (*(int *)data - 1); s = spltty(); error = sc_clean_up(sc->cur_scp); splx(s); if (error) return error; error = sc_switch_scr(sc, i); return (error); #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('v', 6): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case VT_WAITACTIVE: /* wait for switch to occur */ i = (*(int *)data == 0) ? scp->index : (*(int *)data - 1); if ((i < sc->first_vty) || (i >= sc->first_vty + sc->vtys)) return EINVAL; if (i == sc->cur_scp->index) return 0; error = tsleep(VTY_WCHAN(sc, i), (PZERO + 1) | PCATCH, "waitvt", 0); return error; case VT_GETACTIVE: /* get active vty # */ *(int *)data = sc->cur_scp->index + 1; return 0; case VT_GETINDEX: /* get this vty # */ *(int *)data = scp->index + 1; return 0; case VT_LOCKSWITCH: /* prevent vty switching */ if ((*(int *)data) & 0x01) sc->flags |= SC_SCRN_VTYLOCK; else sc->flags &= ~SC_SCRN_VTYLOCK; return 0; case KDENABIO: /* allow io operations */ error = priv_check(td, PRIV_IO); if (error != 0) return error; error = securelevel_gt(td->td_ucred, 0); if (error != 0) return error; #ifdef __i386__ td->td_frame->tf_eflags |= PSL_IOPL; #elif defined(__amd64__) td->td_frame->tf_rflags |= PSL_IOPL; #endif return 0; case KDDISABIO: /* disallow io operations (default) */ #ifdef __i386__ td->td_frame->tf_eflags &= ~PSL_IOPL; #elif defined(__amd64__) td->td_frame->tf_rflags &= ~PSL_IOPL; #endif return 0; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('K', 20): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSKBSTATE: /* set keyboard state (locks) */ if (*(int *)data & ~LOCK_MASK) return EINVAL; scp->status &= ~LOCK_MASK; scp->status |= *(int *)data; if (scp == sc->cur_scp) update_kbd_state(scp, scp->status, LOCK_MASK); return 0; case KDGKBSTATE: /* get keyboard state (locks) */ if (scp == sc->cur_scp) save_kbd_state(scp); *(int *)data = scp->status & LOCK_MASK; return 0; case KDGETREPEAT: /* get keyboard repeat & delay rates */ case KDSETREPEAT: /* set keyboard repeat & delay rates (new) */ error = kbdd_ioctl(sc->kbd, cmd, data); if (error == ENOIOCTL) error = ENODEV; return error; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('K', 67): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSETRAD: /* set keyboard repeat & delay rates (old) */ if (*(int *)data & ~0x7f) return EINVAL; error = kbdd_ioctl(sc->kbd, KDSETRAD, data); if (error == ENOIOCTL) error = ENODEV; return error; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('K', 7): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSKBMODE: /* set keyboard mode */ switch (*(int *)data) { case K_XLATE: /* switch to XLT ascii mode */ case K_RAW: /* switch to RAW scancode mode */ case K_CODE: /* switch to CODE mode */ scp->kbd_mode = *(int *)data; if (scp == sc->cur_scp) (void)kbdd_ioctl(sc->kbd, KDSKBMODE, data); return 0; default: return EINVAL; } /* NOT REACHED */ case KDGKBMODE: /* get keyboard mode */ *(int *)data = scp->kbd_mode; return 0; case KDGKBINFO: error = kbdd_ioctl(sc->kbd, cmd, data); if (error == ENOIOCTL) error = ENODEV; return error; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('K', 8): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDMKTONE: /* sound the bell */ if (*(int*)data) sc_bell(scp, (*(int*)data)&0xffff, (((*(int*)data)>>16)&0xffff)*hz/1000); else sc_bell(scp, scp->bell_pitch, scp->bell_duration); return 0; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('K', 63): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KIOCSOUND: /* make tone (*data) hz */ if (scp == sc->cur_scp) { if (*(int *)data) return sc_tone(*(int *)data); else return sc_tone(0); } return 0; case KDGKBTYPE: /* get keyboard type */ error = kbdd_ioctl(sc->kbd, cmd, data); if (error == ENOIOCTL) { /* always return something? XXX */ *(int *)data = 0; } return 0; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('K', 66): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case KDSETLED: /* set keyboard LED status */ if (*(int *)data & ~LED_MASK) /* FIXME: LOCK_MASK? */ return EINVAL; scp->status &= ~LED_MASK; scp->status |= *(int *)data; if (scp == sc->cur_scp) update_kbd_leds(scp, scp->status); return 0; case KDGETLED: /* get keyboard LED status */ if (scp == sc->cur_scp) save_kbd_state(scp); *(int *)data = scp->status & LED_MASK; return 0; case KBADDKBD: /* add/remove keyboard to/from mux */ case KBRELKBD: error = kbdd_ioctl(sc->kbd, cmd, data); if (error == ENOIOCTL) error = ENODEV; return error; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) case _IO('c', 110): ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; /* FALLTHROUGH */ #endif case CONS_SETKBD: /* set the new keyboard */ { keyboard_t *newkbd; s = spltty(); newkbd = kbd_get_keyboard(*(int *)data); if (newkbd == NULL) { splx(s); return EINVAL; } error = 0; if (sc->kbd != newkbd) { i = kbd_allocate(newkbd->kb_name, newkbd->kb_unit, (void *)&sc->keyboard, sckbdevent, sc); /* i == newkbd->kb_index */ if (i >= 0) { if (sc->kbd != NULL) { save_kbd_state(sc->cur_scp); kbd_release(sc->kbd, (void *)&sc->keyboard); } sc->kbd = kbd_get_keyboard(i); /* sc->kbd == newkbd */ sc->keyboard = i; (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&sc->cur_scp->kbd_mode); update_kbd_state(sc->cur_scp, sc->cur_scp->status, LOCK_MASK); } else { error = EPERM; /* XXX */ } } splx(s); return error; } case CONS_RELKBD: /* release the current keyboard */ s = spltty(); error = 0; if (sc->kbd != NULL) { save_kbd_state(sc->cur_scp); error = kbd_release(sc->kbd, (void *)&sc->keyboard); if (error == 0) { sc->kbd = NULL; sc->keyboard = -1; } } splx(s); return error; case CONS_GETTERM: /* get the current terminal emulator info */ { sc_term_sw_t *sw; if (((term_info_t *)data)->ti_index == 0) { sw = scp->tsw; } else { sw = sc_term_match_by_number(((term_info_t *)data)->ti_index); } if (sw != NULL) { strncpy(((term_info_t *)data)->ti_name, sw->te_name, sizeof(((term_info_t *)data)->ti_name)); strncpy(((term_info_t *)data)->ti_desc, sw->te_desc, sizeof(((term_info_t *)data)->ti_desc)); ((term_info_t *)data)->ti_flags = 0; return 0; } else { ((term_info_t *)data)->ti_name[0] = '\0'; ((term_info_t *)data)->ti_desc[0] = '\0'; ((term_info_t *)data)->ti_flags = 0; return EINVAL; } } case CONS_SETTERM: /* set the current terminal emulator */ s = spltty(); error = sc_init_emulator(scp, ((term_info_t *)data)->ti_name); /* FIXME: what if scp == sc_console! XXX */ splx(s); return error; case GIO_SCRNMAP: /* get output translation table */ bcopy(&sc->scr_map, data, sizeof(sc->scr_map)); return 0; case PIO_SCRNMAP: /* set output translation table */ bcopy(data, &sc->scr_map, sizeof(sc->scr_map)); for (i=0; iscr_map); i++) { sc->scr_rmap[sc->scr_map[i]] = i; } return 0; case GIO_KEYMAP: /* get keyboard translation table */ case PIO_KEYMAP: /* set keyboard translation table */ case OGIO_KEYMAP: /* get keyboard translation table (compat) */ case OPIO_KEYMAP: /* set keyboard translation table (compat) */ case GIO_DEADKEYMAP: /* get accent key translation table */ case PIO_DEADKEYMAP: /* set accent key translation table */ case GETFKEY: /* get function key string */ case SETFKEY: /* set function key string */ error = kbdd_ioctl(sc->kbd, cmd, data); if (error == ENOIOCTL) error = ENODEV; return error; #ifndef SC_NO_FONT_LOADING case PIO_FONT8x8: /* set 8x8 dot font */ if (!ISFONTAVAIL(sc->adp->va_flags)) return ENXIO; bcopy(data, sc->font_8, 8*256); sc->fonts_loaded |= FONT_8; /* * FONT KLUDGE * Always use the font page #0. XXX * Don't load if the current font size is not 8x8. */ if (ISTEXTSC(sc->cur_scp) && (sc->cur_scp->font_size < 14)) sc_load_font(sc->cur_scp, 0, 8, 8, sc->font_8, 0, 256); return 0; case GIO_FONT8x8: /* get 8x8 dot font */ if (!ISFONTAVAIL(sc->adp->va_flags)) return ENXIO; if (sc->fonts_loaded & FONT_8) { bcopy(sc->font_8, data, 8*256); return 0; } else return ENXIO; case PIO_FONT8x14: /* set 8x14 dot font */ if (!ISFONTAVAIL(sc->adp->va_flags)) return ENXIO; bcopy(data, sc->font_14, 14*256); sc->fonts_loaded |= FONT_14; /* * FONT KLUDGE * Always use the font page #0. XXX * Don't load if the current font size is not 8x14. */ if (ISTEXTSC(sc->cur_scp) && (sc->cur_scp->font_size >= 14) && (sc->cur_scp->font_size < 16)) sc_load_font(sc->cur_scp, 0, 14, 8, sc->font_14, 0, 256); return 0; case GIO_FONT8x14: /* get 8x14 dot font */ if (!ISFONTAVAIL(sc->adp->va_flags)) return ENXIO; if (sc->fonts_loaded & FONT_14) { bcopy(sc->font_14, data, 14*256); return 0; } else return ENXIO; case PIO_FONT8x16: /* set 8x16 dot font */ if (!ISFONTAVAIL(sc->adp->va_flags)) return ENXIO; bcopy(data, sc->font_16, 16*256); sc->fonts_loaded |= FONT_16; /* * FONT KLUDGE * Always use the font page #0. XXX * Don't load if the current font size is not 8x16. */ if (ISTEXTSC(sc->cur_scp) && (sc->cur_scp->font_size >= 16)) sc_load_font(sc->cur_scp, 0, 16, 8, sc->font_16, 0, 256); return 0; case GIO_FONT8x16: /* get 8x16 dot font */ if (!ISFONTAVAIL(sc->adp->va_flags)) return ENXIO; if (sc->fonts_loaded & FONT_16) { bcopy(sc->font_16, data, 16*256); return 0; } else return ENXIO; #endif /* SC_NO_FONT_LOADING */ default: break; } return (ENOIOCTL); } static int consolectl_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { return sctty_ioctl(dev->si_drv1, cmd, data, td); } static int consolectl_close(struct cdev *dev, int flags, int mode, struct thread *td) { #ifndef SC_NO_SYSMOUSE mouse_info_t info; memset(&info, 0, sizeof(info)); info.operation = MOUSE_ACTION; /* * Make sure all buttons are released when moused and other * console daemons exit, so that no buttons are left pressed. */ (void) sctty_ioctl(dev->si_drv1, CONS_MOUSECTL, (caddr_t)&info, td); #endif return (0); } static void sc_cnprobe(struct consdev *cp) { int unit; int flags; if (!vty_enabled(VTY_SC)) { cp->cn_pri = CN_DEAD; return; } cp->cn_pri = sc_get_cons_priority(&unit, &flags); /* a video card is always required */ if (!scvidprobe(unit, flags, TRUE)) cp->cn_pri = CN_DEAD; /* syscons will become console even when there is no keyboard */ sckbdprobe(unit, flags, TRUE); if (cp->cn_pri == CN_DEAD) return; /* initialize required fields */ strcpy(cp->cn_name, "ttyv0"); } static void sc_cninit(struct consdev *cp) { int unit; int flags; sc_get_cons_priority(&unit, &flags); scinit(unit, flags | SC_KERNEL_CONSOLE); sc_console_unit = unit; sc_console = sc_get_stat(sc_get_softc(unit, SC_KERNEL_CONSOLE)->dev[0]); sc_consptr = cp; } static void sc_cnterm(struct consdev *cp) { /* we are not the kernel console any more, release everything */ if (sc_console_unit < 0) return; /* shouldn't happen */ #if 0 /* XXX */ sc_clear_screen(sc_console); sccnupdate(sc_console); #endif scterm(sc_console_unit, SC_KERNEL_CONSOLE); sc_console_unit = -1; sc_console = NULL; } static void sccnclose(sc_softc_t *sc, struct sc_cnstate *sp); +static int sc_cngetc_locked(struct sc_cnstate *sp); static void sccnopen(sc_softc_t *sc, struct sc_cnstate *sp, int flags); static void sccnscrlock(sc_softc_t *sc, struct sc_cnstate *sp); static void sccnscrunlock(sc_softc_t *sc, struct sc_cnstate *sp); static void sccnscrlock(sc_softc_t *sc, struct sc_cnstate *sp) { SC_VIDEO_LOCK(sc); } static void sccnscrunlock(sc_softc_t *sc, struct sc_cnstate *sp) { SC_VIDEO_UNLOCK(sc); } static void sccnopen(sc_softc_t *sc, struct sc_cnstate *sp, int flags) { int kbd_mode; /* assert(sc_console_unit >= 0) */ sp->kbd_opened = FALSE; sp->scr_opened = FALSE; /* Opening the keyboard is optional. */ if (!(flags & 1) || sc->kbd == NULL) goto over_keyboard; /* * Make sure the keyboard is accessible even when the kbd device * driver is disabled. */ kbdd_enable(sc->kbd); /* Switch the keyboard to console mode (K_XLATE, polled) on all scp's. */ kbd_mode = K_XLATE; (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&kbd_mode); sc->kbd_open_level++; kbdd_poll(sc->kbd, TRUE); sp->kbd_opened = TRUE; over_keyboard: ; /* The screen is opened iff locking it succeeds. */ sccnscrlock(sc, sp); sp->scr_opened = TRUE; /* The screen switch is optional. */ if (!(flags & 2)) return; /* try to switch to the kernel console screen */ if (!cold && sc->cur_scp->index != sc_console->index && sc->cur_scp->smode.mode == VT_AUTO && sc_console->smode.mode == VT_AUTO) sc_switch_scr(sc, sc_console->index); } static void sccnclose(sc_softc_t *sc, struct sc_cnstate *sp) { sp->scr_opened = FALSE; sccnscrunlock(sc, sp); if (!sp->kbd_opened) return; /* Restore keyboard mode (for the current, possibly-changed scp). */ kbdd_poll(sc->kbd, FALSE); if (--sc->kbd_open_level == 0) (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&sc->cur_scp->kbd_mode); kbdd_disable(sc->kbd); sp->kbd_opened = FALSE; } /* * Grabbing switches the screen and keyboard focus to sc_console and the * keyboard mode to (K_XLATE, polled). Only switching to polled mode is * essential (for preventing the interrupt handler from eating input * between polls). Focus is part of the UI, and the other switches are * work just was well when they are done on every entry and exit. * * Screen switches while grabbed are supported, and to maintain focus for * this ungrabbing and closing only restore the polling state and then * the keyboard mode if on the original screen. */ static void sc_cngrab(struct consdev *cp) { sc_softc_t *sc; int lev; sc = sc_console->sc; lev = atomic_fetchadd_int(&sc->grab_level, 1); if (lev >= 0 && lev < 2) { sccnopen(sc, &sc->grab_state[lev], 1 | 2); sccnscrunlock(sc, &sc->grab_state[lev]); } } static void sc_cnungrab(struct consdev *cp) { sc_softc_t *sc; int lev; sc = sc_console->sc; lev = atomic_load_acq_int(&sc->grab_level) - 1; if (lev >= 0 && lev < 2) { sccnscrlock(sc, &sc->grab_state[lev]); sccnclose(sc, &sc->grab_state[lev]); } atomic_add_int(&sc->grab_level, -1); } static void sc_cnputc(struct consdev *cd, int c) { struct sc_cnstate st; u_char buf[1]; scr_stat *scp = sc_console; #ifndef SC_NO_HISTORY #if 0 struct tty *tp; #endif #endif /* !SC_NO_HISTORY */ int s; /* assert(sc_console != NULL) */ sccnopen(scp->sc, &st, 0); #ifndef SC_NO_HISTORY if (scp == scp->sc->cur_scp && scp->status & SLKED) { scp->status &= ~SLKED; update_kbd_state(scp, scp->status, SLKED); if (scp->status & BUFFER_SAVED) { if (!sc_hist_restore(scp)) sc_remove_cutmarking(scp); scp->status &= ~BUFFER_SAVED; scp->status |= CURSOR_ENABLED; sc_draw_cursor_image(scp); } #if 0 /* * XXX: Now that TTY's have their own locks, we cannot process * any data after disabling scroll lock. cnputs already holds a * spinlock. */ tp = SC_DEV(scp->sc, scp->index); /* XXX "tp" can be NULL */ tty_lock(tp); if (tty_opened(tp)) sctty_outwakeup(tp); tty_unlock(tp); #endif } #endif /* !SC_NO_HISTORY */ buf[0] = c; sc_puts(scp, buf, 1, 1); s = spltty(); /* block sckbdevent and scrn_timer */ sccnupdate(scp); splx(s); sccnclose(scp->sc, &st); } static int sc_cngetc(struct consdev *cd) { + int c, s; + + /* assert(sc_console != NULL) */ + s = spltty(); /* block sckbdevent and scrn_timer while we poll */ + if (sc_console->sc->kbd == NULL) { + splx(s); + return -1; + } + c = sc_cngetc_locked(NULL); + splx(s); + return c; +} + +static int +sc_cngetc_locked(struct sc_cnstate *sp) +{ static struct fkeytab fkey; static int fkeycp; scr_stat *scp; const u_char *p; - int s = spltty(); /* block sckbdevent and scrn_timer while we poll */ int c; - /* assert(sc_console != NULL) */ - /* * Stop the screen saver and update the screen if necessary. * What if we have been running in the screen saver code... XXX */ sc_touch_scrn_saver(); scp = sc_console->sc->cur_scp; /* XXX */ sccnupdate(scp); - if (fkeycp < fkey.len) { - splx(s); + if (fkeycp < fkey.len) return fkey.str[fkeycp++]; - } - - if (scp->sc->kbd == NULL) { - splx(s); - return -1; - } c = scgetc(scp->sc, SCGETC_CN | SCGETC_NONBLOCK, NULL); switch (KEYFLAGS(c)) { case 0: /* normal char */ return KEYCHAR(c); case FKEY: /* function key */ p = (*scp->tsw->te_fkeystr)(scp, c); if (p != NULL) { fkey.len = strlen(p); bcopy(p, fkey.str, fkey.len); fkeycp = 1; return fkey.str[0]; } p = kbdd_get_fkeystr(scp->sc->kbd, KEYCHAR(c), (size_t *)&fkeycp); fkey.len = fkeycp; if ((p != NULL) && (fkey.len > 0)) { bcopy(p, fkey.str, fkey.len); fkeycp = 1; return fkey.str[0]; } return c; /* XXX */ case NOKEY: case ERRKEY: default: return -1; } /* NOT REACHED */ } static void sccnupdate(scr_stat *scp) { /* this is a cut-down version of scrn_timer()... */ if (suspend_in_progress || scp->sc->font_loading_in_progress) return; if (kdb_active || panicstr || shutdown_in_progress) { sc_touch_scrn_saver(); } else if (scp != scp->sc->cur_scp) { return; } if (!run_scrn_saver) scp->sc->flags &= ~SC_SCRN_IDLE; #ifdef DEV_SPLASH if ((saver_mode != CONS_LKM_SAVER) || !(scp->sc->flags & SC_SCRN_IDLE)) if (scp->sc->flags & SC_SCRN_BLANKED) stop_scrn_saver(scp->sc, current_saver); #endif if (scp != scp->sc->cur_scp || scp->sc->blink_in_progress || scp->sc->switch_in_progress) return; /* * FIXME: unlike scrn_timer(), we call scrn_update() from here even * when write_in_progress is non-zero. XXX */ if (!ISGRAPHSC(scp) && !(scp->sc->flags & SC_SCRN_BLANKED)) scrn_update(scp, TRUE); } static void scrn_timer(void *arg) { #ifndef PC98 static time_t kbd_time_stamp = 0; #endif sc_softc_t *sc; scr_stat *scp; int again, rate; again = (arg != NULL); if (arg != NULL) sc = (sc_softc_t *)arg; else if (sc_console != NULL) sc = sc_console->sc; else return; /* find the vty to update */ scp = sc->cur_scp; /* don't do anything when we are performing some I/O operations */ if (suspend_in_progress || sc->font_loading_in_progress) goto done; #ifndef PC98 if ((sc->kbd == NULL) && (sc->config & SC_AUTODETECT_KBD)) { /* try to allocate a keyboard automatically */ if (kbd_time_stamp != time_uptime) { kbd_time_stamp = time_uptime; sc->keyboard = sc_allocate_keyboard(sc, -1); if (sc->keyboard >= 0) { sc->kbd = kbd_get_keyboard(sc->keyboard); (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&sc->cur_scp->kbd_mode); update_kbd_state(sc->cur_scp, sc->cur_scp->status, LOCK_MASK); } } } #endif /* PC98 */ /* should we stop the screen saver? */ if (kdb_active || panicstr || shutdown_in_progress) sc_touch_scrn_saver(); if (run_scrn_saver) { if (time_uptime > sc->scrn_time_stamp + scrn_blank_time) sc->flags |= SC_SCRN_IDLE; else sc->flags &= ~SC_SCRN_IDLE; } else { sc->scrn_time_stamp = time_uptime; sc->flags &= ~SC_SCRN_IDLE; if (scrn_blank_time > 0) run_scrn_saver = TRUE; } #ifdef DEV_SPLASH if ((saver_mode != CONS_LKM_SAVER) || !(sc->flags & SC_SCRN_IDLE)) if (sc->flags & SC_SCRN_BLANKED) stop_scrn_saver(sc, current_saver); #endif /* should we just return ? */ if (sc->blink_in_progress || sc->switch_in_progress || sc->write_in_progress) goto done; /* Update the screen */ scp = sc->cur_scp; /* cur_scp may have changed... */ if (!ISGRAPHSC(scp) && !(sc->flags & SC_SCRN_BLANKED)) scrn_update(scp, TRUE); #ifdef DEV_SPLASH /* should we activate the screen saver? */ if ((saver_mode == CONS_LKM_SAVER) && (sc->flags & SC_SCRN_IDLE)) if (!ISGRAPHSC(scp) || (sc->flags & SC_SCRN_BLANKED)) (*current_saver)(sc, TRUE); #endif done: if (again) { /* * Use reduced "refresh" rate if we are in graphics and that is not a * graphical screen saver. In such case we just have nothing to do. */ if (ISGRAPHSC(scp) && !(sc->flags & SC_SCRN_BLANKED)) rate = 2; else rate = 30; callout_reset_sbt(&sc->ctimeout, SBT_1S / rate, 0, scrn_timer, sc, C_PREL(1)); } } static int and_region(int *s1, int *e1, int s2, int e2) { if (*e1 < s2 || e2 < *s1) return FALSE; *s1 = imax(*s1, s2); *e1 = imin(*e1, e2); return TRUE; } static void scrn_update(scr_stat *scp, int show_cursor) { int start; int end; int s; int e; /* assert(scp == scp->sc->cur_scp) */ SC_VIDEO_LOCK(scp->sc); #ifndef SC_NO_CUTPASTE /* remove the previous mouse pointer image if necessary */ if (scp->status & MOUSE_VISIBLE) { s = scp->mouse_pos; e = scp->mouse_pos + scp->xsize + 1; if ((scp->status & (MOUSE_MOVED | MOUSE_HIDDEN)) || and_region(&s, &e, scp->start, scp->end) || ((scp->status & CURSOR_ENABLED) && (scp->cursor_pos != scp->cursor_oldpos) && (and_region(&s, &e, scp->cursor_pos, scp->cursor_pos) || and_region(&s, &e, scp->cursor_oldpos, scp->cursor_oldpos)))) { sc_remove_mouse_image(scp); if (scp->end >= scp->xsize*scp->ysize) scp->end = scp->xsize*scp->ysize - 1; } } #endif /* !SC_NO_CUTPASTE */ #if 1 /* debug: XXX */ if (scp->end >= scp->xsize*scp->ysize) { printf("scrn_update(): scp->end %d > size_of_screen!!\n", scp->end); scp->end = scp->xsize*scp->ysize - 1; } if (scp->start < 0) { printf("scrn_update(): scp->start %d < 0\n", scp->start); scp->start = 0; } #endif /* update screen image */ if (scp->start <= scp->end) { if (scp->mouse_cut_end >= 0) { /* there is a marked region for cut & paste */ if (scp->mouse_cut_start <= scp->mouse_cut_end) { start = scp->mouse_cut_start; end = scp->mouse_cut_end; } else { start = scp->mouse_cut_end; end = scp->mouse_cut_start - 1; } s = start; e = end; /* does the cut-mark region overlap with the update region? */ if (and_region(&s, &e, scp->start, scp->end)) { (*scp->rndr->draw)(scp, s, e - s + 1, TRUE); s = 0; e = start - 1; if (and_region(&s, &e, scp->start, scp->end)) (*scp->rndr->draw)(scp, s, e - s + 1, FALSE); s = end + 1; e = scp->xsize*scp->ysize - 1; if (and_region(&s, &e, scp->start, scp->end)) (*scp->rndr->draw)(scp, s, e - s + 1, FALSE); } else { (*scp->rndr->draw)(scp, scp->start, scp->end - scp->start + 1, FALSE); } } else { (*scp->rndr->draw)(scp, scp->start, scp->end - scp->start + 1, FALSE); } } /* we are not to show the cursor and the mouse pointer... */ if (!show_cursor) { scp->end = 0; scp->start = scp->xsize*scp->ysize - 1; SC_VIDEO_UNLOCK(scp->sc); return; } /* update cursor image */ if (scp->status & CURSOR_ENABLED) { s = scp->start; e = scp->end; /* did cursor move since last time ? */ if (scp->cursor_pos != scp->cursor_oldpos) { /* do we need to remove old cursor image ? */ if (!and_region(&s, &e, scp->cursor_oldpos, scp->cursor_oldpos)) sc_remove_cursor_image(scp); sc_draw_cursor_image(scp); } else { if (and_region(&s, &e, scp->cursor_pos, scp->cursor_pos)) /* cursor didn't move, but has been overwritten */ sc_draw_cursor_image(scp); else if (scp->curs_attr.flags & CONS_BLINK_CURSOR) /* if it's a blinking cursor, update it */ (*scp->rndr->blink_cursor)(scp, scp->cursor_pos, sc_inside_cutmark(scp, scp->cursor_pos)); } } #ifndef SC_NO_CUTPASTE /* update "pseudo" mouse pointer image */ if (scp->sc->flags & SC_MOUSE_ENABLED) { if (!(scp->status & (MOUSE_VISIBLE | MOUSE_HIDDEN))) { scp->status &= ~MOUSE_MOVED; sc_draw_mouse_image(scp); } } #endif /* SC_NO_CUTPASTE */ scp->end = 0; scp->start = scp->xsize*scp->ysize - 1; SC_VIDEO_UNLOCK(scp->sc); } #ifdef DEV_SPLASH static int scsplash_callback(int event, void *arg) { sc_softc_t *sc; int error; sc = (sc_softc_t *)arg; switch (event) { case SPLASH_INIT: if (add_scrn_saver(scsplash_saver) == 0) { sc->flags &= ~SC_SAVER_FAILED; run_scrn_saver = TRUE; if (cold && !(boothowto & RB_VERBOSE)) { scsplash_stick(TRUE); (*current_saver)(sc, TRUE); } } return 0; case SPLASH_TERM: if (current_saver == scsplash_saver) { scsplash_stick(FALSE); error = remove_scrn_saver(scsplash_saver); if (error) return error; } return 0; default: return EINVAL; } } static void scsplash_saver(sc_softc_t *sc, int show) { static int busy = FALSE; scr_stat *scp; if (busy) return; busy = TRUE; scp = sc->cur_scp; if (show) { if (!(sc->flags & SC_SAVER_FAILED)) { if (!(sc->flags & SC_SCRN_BLANKED)) set_scrn_saver_mode(scp, -1, NULL, 0); switch (splash(sc->adp, TRUE)) { case 0: /* succeeded */ break; case EAGAIN: /* try later */ restore_scrn_saver_mode(scp, FALSE); sc_touch_scrn_saver(); /* XXX */ break; default: sc->flags |= SC_SAVER_FAILED; scsplash_stick(FALSE); restore_scrn_saver_mode(scp, TRUE); printf("scsplash_saver(): failed to put up the image\n"); break; } } } else if (!sticky_splash) { if ((sc->flags & SC_SCRN_BLANKED) && (splash(sc->adp, FALSE) == 0)) restore_scrn_saver_mode(scp, TRUE); } busy = FALSE; } static int add_scrn_saver(void (*this_saver)(sc_softc_t *, int)) { #if 0 int error; if (current_saver != none_saver) { error = remove_scrn_saver(current_saver); if (error) return error; } #endif if (current_saver != none_saver) return EBUSY; run_scrn_saver = FALSE; saver_mode = CONS_LKM_SAVER; current_saver = this_saver; return 0; } static int remove_scrn_saver(void (*this_saver)(sc_softc_t *, int)) { if (current_saver != this_saver) return EINVAL; #if 0 /* * In order to prevent `current_saver' from being called by * the timeout routine `scrn_timer()' while we manipulate * the saver list, we shall set `current_saver' to `none_saver' * before stopping the current saver, rather than blocking by `splXX()'. */ current_saver = none_saver; if (scrn_blanked) stop_scrn_saver(this_saver); #endif /* unblank all blanked screens */ wait_scrn_saver_stop(NULL); if (scrn_blanked) return EBUSY; current_saver = none_saver; return 0; } static int set_scrn_saver_mode(scr_stat *scp, int mode, u_char *pal, int border) { int s; /* assert(scp == scp->sc->cur_scp) */ s = spltty(); if (!ISGRAPHSC(scp)) sc_remove_cursor_image(scp); scp->splash_save_mode = scp->mode; scp->splash_save_status = scp->status & (GRAPHICS_MODE | PIXEL_MODE); scp->status &= ~(GRAPHICS_MODE | PIXEL_MODE); scp->status |= (UNKNOWN_MODE | SAVER_RUNNING); scp->sc->flags |= SC_SCRN_BLANKED; ++scrn_blanked; splx(s); if (mode < 0) return 0; scp->mode = mode; if (set_mode(scp) == 0) { if (scp->sc->adp->va_info.vi_flags & V_INFO_GRAPHICS) scp->status |= GRAPHICS_MODE; #ifndef SC_NO_PALETTE_LOADING if (pal != NULL) vidd_load_palette(scp->sc->adp, pal); #endif sc_set_border(scp, border); return 0; } else { s = spltty(); scp->mode = scp->splash_save_mode; scp->status &= ~(UNKNOWN_MODE | SAVER_RUNNING); scp->status |= scp->splash_save_status; splx(s); return 1; } } static int restore_scrn_saver_mode(scr_stat *scp, int changemode) { int mode; int status; int s; /* assert(scp == scp->sc->cur_scp) */ s = spltty(); mode = scp->mode; status = scp->status; scp->mode = scp->splash_save_mode; scp->status &= ~(UNKNOWN_MODE | SAVER_RUNNING); scp->status |= scp->splash_save_status; scp->sc->flags &= ~SC_SCRN_BLANKED; if (!changemode) { if (!ISGRAPHSC(scp)) sc_draw_cursor_image(scp); --scrn_blanked; splx(s); return 0; } if (set_mode(scp) == 0) { #ifndef SC_NO_PALETTE_LOADING #ifdef SC_PIXEL_MODE if (scp->sc->adp->va_info.vi_mem_model == V_INFO_MM_DIRECT) vidd_load_palette(scp->sc->adp, scp->sc->palette2); else #endif vidd_load_palette(scp->sc->adp, scp->sc->palette); #endif --scrn_blanked; splx(s); return 0; } else { scp->mode = mode; scp->status = status; splx(s); return 1; } } static void stop_scrn_saver(sc_softc_t *sc, void (*saver)(sc_softc_t *, int)) { (*saver)(sc, FALSE); run_scrn_saver = FALSE; /* the screen saver may have chosen not to stop after all... */ if (sc->flags & SC_SCRN_BLANKED) return; mark_all(sc->cur_scp); if (sc->delayed_next_scr) sc_switch_scr(sc, sc->delayed_next_scr - 1); if (!kdb_active) wakeup(&scrn_blanked); } static int wait_scrn_saver_stop(sc_softc_t *sc) { int error = 0; while (scrn_blanked > 0) { run_scrn_saver = FALSE; if (sc && !(sc->flags & SC_SCRN_BLANKED)) { error = 0; break; } error = tsleep(&scrn_blanked, PZERO | PCATCH, "scrsav", 0); if ((error != 0) && (error != ERESTART)) break; } run_scrn_saver = FALSE; return error; } #endif /* DEV_SPLASH */ void sc_touch_scrn_saver(void) { scsplash_stick(FALSE); run_scrn_saver = FALSE; } int sc_switch_scr(sc_softc_t *sc, u_int next_scr) { scr_stat *cur_scp; struct tty *tp; struct proc *p; int s; DPRINTF(5, ("sc0: sc_switch_scr() %d ", next_scr + 1)); if (sc->cur_scp == NULL) return (0); /* prevent switch if previously requested */ if (sc->flags & SC_SCRN_VTYLOCK) { sc_bell(sc->cur_scp, sc->cur_scp->bell_pitch, sc->cur_scp->bell_duration); return EPERM; } /* delay switch if the screen is blanked or being updated */ if ((sc->flags & SC_SCRN_BLANKED) || sc->write_in_progress || sc->blink_in_progress) { sc->delayed_next_scr = next_scr + 1; sc_touch_scrn_saver(); DPRINTF(5, ("switch delayed\n")); return 0; } sc->delayed_next_scr = 0; s = spltty(); cur_scp = sc->cur_scp; /* we are in the middle of the vty switching process... */ if (sc->switch_in_progress && (cur_scp->smode.mode == VT_PROCESS) && cur_scp->proc) { p = pfind(cur_scp->pid); if (cur_scp->proc != p) { if (p) PROC_UNLOCK(p); /* * The controlling process has died!!. Do some clean up. * NOTE:`cur_scp->proc' and `cur_scp->smode.mode' * are not reset here yet; they will be cleared later. */ DPRINTF(5, ("cur_scp controlling process %d died, ", cur_scp->pid)); if (cur_scp->status & SWITCH_WAIT_REL) { /* * Force the previous switch to finish, but return now * with error. */ DPRINTF(5, ("reset WAIT_REL, ")); finish_vt_rel(cur_scp, TRUE, &s); splx(s); DPRINTF(5, ("finishing previous switch\n")); return EINVAL; } else if (cur_scp->status & SWITCH_WAIT_ACQ) { /* let's assume screen switch has been completed. */ DPRINTF(5, ("reset WAIT_ACQ, ")); finish_vt_acq(cur_scp); } else { /* * We are in between screen release and acquisition, and * reached here via scgetc() or scrn_timer() which has * interrupted exchange_scr(). Don't do anything stupid. */ DPRINTF(5, ("waiting nothing, ")); } } else { if (p) PROC_UNLOCK(p); /* * The controlling process is alive, but not responding... * It is either buggy or it may be just taking time. * The following code is a gross kludge to cope with this * problem for which there is no clean solution. XXX */ if (cur_scp->status & SWITCH_WAIT_REL) { switch (sc->switch_in_progress++) { case 1: break; case 2: DPRINTF(5, ("sending relsig again, ")); signal_vt_rel(cur_scp); break; case 3: break; case 4: default: /* * Act as if the controlling program returned * VT_FALSE. */ DPRINTF(5, ("force reset WAIT_REL, ")); finish_vt_rel(cur_scp, FALSE, &s); splx(s); DPRINTF(5, ("act as if VT_FALSE was seen\n")); return EINVAL; } } else if (cur_scp->status & SWITCH_WAIT_ACQ) { switch (sc->switch_in_progress++) { case 1: break; case 2: DPRINTF(5, ("sending acqsig again, ")); signal_vt_acq(cur_scp); break; case 3: break; case 4: default: /* clear the flag and finish the previous switch */ DPRINTF(5, ("force reset WAIT_ACQ, ")); finish_vt_acq(cur_scp); break; } } } } /* * Return error if an invalid argument is given, or vty switch * is still in progress. */ if ((next_scr < sc->first_vty) || (next_scr >= sc->first_vty + sc->vtys) || sc->switch_in_progress) { splx(s); sc_bell(cur_scp, bios_value.bell_pitch, BELL_DURATION); DPRINTF(5, ("error 1\n")); return EINVAL; } /* * Don't allow switching away from the graphics mode vty * if the switch mode is VT_AUTO, unless the next vty is the same * as the current or the current vty has been closed (but showing). */ tp = SC_DEV(sc, cur_scp->index); if ((cur_scp->index != next_scr) && tty_opened_ns(tp) && (cur_scp->smode.mode == VT_AUTO) && ISGRAPHSC(cur_scp)) { splx(s); sc_bell(cur_scp, bios_value.bell_pitch, BELL_DURATION); DPRINTF(5, ("error, graphics mode\n")); return EINVAL; } /* * Is the wanted vty open? Don't allow switching to a closed vty. * If we are in DDB, don't switch to a vty in the VT_PROCESS mode. * Note that we always allow the user to switch to the kernel * console even if it is closed. */ if ((sc_console == NULL) || (next_scr != sc_console->index)) { tp = SC_DEV(sc, next_scr); if (!tty_opened_ns(tp)) { splx(s); sc_bell(cur_scp, bios_value.bell_pitch, BELL_DURATION); DPRINTF(5, ("error 2, requested vty isn't open!\n")); return EINVAL; } if (kdb_active && SC_STAT(tp)->smode.mode == VT_PROCESS) { splx(s); DPRINTF(5, ("error 3, requested vty is in the VT_PROCESS mode\n")); return EINVAL; } } /* this is the start of vty switching process... */ ++sc->switch_in_progress; sc->old_scp = cur_scp; sc->new_scp = sc_get_stat(SC_DEV(sc, next_scr)); if (sc->new_scp == sc->old_scp) { sc->switch_in_progress = 0; /* * XXX wakeup() locks the scheduler lock which will hang if * the lock is in an in-between state, e.g., when we stop at * a breakpoint at fork_exit. It has always been wrong to call * wakeup() when the debugger is active. In RELENG_4, wakeup() * is supposed to be locked by splhigh(), but the debugger may * be invoked at splhigh(). */ if (!kdb_active) wakeup(VTY_WCHAN(sc,next_scr)); splx(s); DPRINTF(5, ("switch done (new == old)\n")); return 0; } /* has controlling process died? */ vt_proc_alive(sc->old_scp); vt_proc_alive(sc->new_scp); /* wait for the controlling process to release the screen, if necessary */ if (signal_vt_rel(sc->old_scp)) { splx(s); return 0; } /* go set up the new vty screen */ splx(s); exchange_scr(sc); s = spltty(); /* wake up processes waiting for this vty */ if (!kdb_active) wakeup(VTY_WCHAN(sc,next_scr)); /* wait for the controlling process to acknowledge, if necessary */ if (signal_vt_acq(sc->cur_scp)) { splx(s); return 0; } sc->switch_in_progress = 0; if (sc->unit == sc_console_unit) cnavailable(sc_consptr, TRUE); splx(s); DPRINTF(5, ("switch done\n")); return 0; } static int do_switch_scr(sc_softc_t *sc, int s) { vt_proc_alive(sc->new_scp); splx(s); exchange_scr(sc); s = spltty(); /* sc->cur_scp == sc->new_scp */ wakeup(VTY_WCHAN(sc,sc->cur_scp->index)); /* wait for the controlling process to acknowledge, if necessary */ if (!signal_vt_acq(sc->cur_scp)) { sc->switch_in_progress = 0; if (sc->unit == sc_console_unit) cnavailable(sc_consptr, TRUE); } return s; } static int vt_proc_alive(scr_stat *scp) { struct proc *p; if (scp->proc) { if ((p = pfind(scp->pid)) != NULL) PROC_UNLOCK(p); if (scp->proc == p) return TRUE; scp->proc = NULL; scp->smode.mode = VT_AUTO; DPRINTF(5, ("vt controlling process %d died\n", scp->pid)); } return FALSE; } static int signal_vt_rel(scr_stat *scp) { if (scp->smode.mode != VT_PROCESS) return FALSE; scp->status |= SWITCH_WAIT_REL; PROC_LOCK(scp->proc); kern_psignal(scp->proc, scp->smode.relsig); PROC_UNLOCK(scp->proc); DPRINTF(5, ("sending relsig to %d\n", scp->pid)); return TRUE; } static int signal_vt_acq(scr_stat *scp) { if (scp->smode.mode != VT_PROCESS) return FALSE; if (scp->sc->unit == sc_console_unit) cnavailable(sc_consptr, FALSE); scp->status |= SWITCH_WAIT_ACQ; PROC_LOCK(scp->proc); kern_psignal(scp->proc, scp->smode.acqsig); PROC_UNLOCK(scp->proc); DPRINTF(5, ("sending acqsig to %d\n", scp->pid)); return TRUE; } static int finish_vt_rel(scr_stat *scp, int release, int *s) { if (scp == scp->sc->old_scp && scp->status & SWITCH_WAIT_REL) { scp->status &= ~SWITCH_WAIT_REL; if (release) *s = do_switch_scr(scp->sc, *s); else scp->sc->switch_in_progress = 0; return 0; } return EINVAL; } static int finish_vt_acq(scr_stat *scp) { if (scp == scp->sc->new_scp && scp->status & SWITCH_WAIT_ACQ) { scp->status &= ~SWITCH_WAIT_ACQ; scp->sc->switch_in_progress = 0; return 0; } return EINVAL; } static void exchange_scr(sc_softc_t *sc) { scr_stat *scp; /* save the current state of video and keyboard */ sc_move_cursor(sc->old_scp, sc->old_scp->xpos, sc->old_scp->ypos); if (!ISGRAPHSC(sc->old_scp)) sc_remove_cursor_image(sc->old_scp); if (sc->old_scp->kbd_mode == K_XLATE) save_kbd_state(sc->old_scp); /* set up the video for the new screen */ scp = sc->cur_scp = sc->new_scp; #ifdef PC98 if (sc->old_scp->mode != scp->mode || ISUNKNOWNSC(sc->old_scp) || ISUNKNOWNSC(sc->new_scp)) #else if (sc->old_scp->mode != scp->mode || ISUNKNOWNSC(sc->old_scp)) #endif set_mode(scp); #ifndef __sparc64__ else sc_vtb_init(&scp->scr, VTB_FRAMEBUFFER, scp->xsize, scp->ysize, (void *)sc->adp->va_window, FALSE); #endif scp->status |= MOUSE_HIDDEN; sc_move_cursor(scp, scp->xpos, scp->ypos); if (!ISGRAPHSC(scp)) sc_set_cursor_image(scp); #ifndef SC_NO_PALETTE_LOADING if (ISGRAPHSC(sc->old_scp)) { #ifdef SC_PIXEL_MODE if (sc->adp->va_info.vi_mem_model == V_INFO_MM_DIRECT) vidd_load_palette(sc->adp, sc->palette2); else #endif vidd_load_palette(sc->adp, sc->palette); } #endif sc_set_border(scp, scp->border); /* set up the keyboard for the new screen */ if (sc->kbd_open_level == 0 && sc->old_scp->kbd_mode != scp->kbd_mode) (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&scp->kbd_mode); update_kbd_state(scp, scp->status, LOCK_MASK); mark_all(scp); } static void sc_puts(scr_stat *scp, u_char *buf, int len, int kernel) { #ifdef DEV_SPLASH /* make screensaver happy */ if (!sticky_splash && scp == scp->sc->cur_scp && !sc_saver_keyb_only) run_scrn_saver = FALSE; #endif if (scp->tsw) (*scp->tsw->te_puts)(scp, buf, len, kernel); if (scp->sc->delayed_next_scr) sc_switch_scr(scp->sc, scp->sc->delayed_next_scr - 1); } void sc_draw_cursor_image(scr_stat *scp) { /* assert(scp == scp->sc->cur_scp); */ SC_VIDEO_LOCK(scp->sc); (*scp->rndr->draw_cursor)(scp, scp->cursor_pos, scp->curs_attr.flags & CONS_BLINK_CURSOR, TRUE, sc_inside_cutmark(scp, scp->cursor_pos)); scp->cursor_oldpos = scp->cursor_pos; SC_VIDEO_UNLOCK(scp->sc); } void sc_remove_cursor_image(scr_stat *scp) { /* assert(scp == scp->sc->cur_scp); */ SC_VIDEO_LOCK(scp->sc); (*scp->rndr->draw_cursor)(scp, scp->cursor_oldpos, scp->curs_attr.flags & CONS_BLINK_CURSOR, FALSE, sc_inside_cutmark(scp, scp->cursor_oldpos)); SC_VIDEO_UNLOCK(scp->sc); } static void update_cursor_image(scr_stat *scp) { /* assert(scp == scp->sc->cur_scp); */ sc_remove_cursor_image(scp); sc_set_cursor_image(scp); sc_draw_cursor_image(scp); } void sc_set_cursor_image(scr_stat *scp) { scp->curs_attr.flags = scp->curr_curs_attr.flags; if (scp->curs_attr.flags & CONS_HIDDEN_CURSOR) { /* hidden cursor is internally represented as zero-height underline */ scp->curs_attr.flags = CONS_CHAR_CURSOR; scp->curs_attr.base = scp->curs_attr.height = 0; } else if (scp->curs_attr.flags & CONS_CHAR_CURSOR) { scp->curs_attr.base = imin(scp->curr_curs_attr.base, scp->font_size - 1); scp->curs_attr.height = imin(scp->curr_curs_attr.height, scp->font_size - scp->curs_attr.base); } else { /* block cursor */ scp->curs_attr.base = 0; scp->curs_attr.height = scp->font_size; } /* assert(scp == scp->sc->cur_scp); */ SC_VIDEO_LOCK(scp->sc); (*scp->rndr->set_cursor)(scp, scp->curs_attr.base, scp->curs_attr.height, scp->curs_attr.flags & CONS_BLINK_CURSOR); SC_VIDEO_UNLOCK(scp->sc); } static void change_cursor_shape(scr_stat *scp, int flags, int base, int height) { if ((scp == scp->sc->cur_scp) && !ISGRAPHSC(scp)) sc_remove_cursor_image(scp); if (base >= 0) scp->curr_curs_attr.base = base; if (height >= 0) scp->curr_curs_attr.height = height; if (flags & CONS_RESET_CURSOR) scp->curr_curs_attr = scp->dflt_curs_attr; else scp->curr_curs_attr.flags = flags & CONS_CURSOR_ATTRS; if ((scp == scp->sc->cur_scp) && !ISGRAPHSC(scp)) { sc_set_cursor_image(scp); sc_draw_cursor_image(scp); } } void sc_change_cursor_shape(scr_stat *scp, int flags, int base, int height) { sc_softc_t *sc; struct tty *tp; int s; int i; s = spltty(); if ((flags != -1) && (flags & CONS_LOCAL_CURSOR)) { /* local (per vty) change */ change_cursor_shape(scp, flags, base, height); splx(s); return; } /* global change */ sc = scp->sc; if (base >= 0) sc->curs_attr.base = base; if (height >= 0) sc->curs_attr.height = height; if (flags != -1) { if (flags & CONS_RESET_CURSOR) sc->curs_attr = sc->dflt_curs_attr; else sc->curs_attr.flags = flags & CONS_CURSOR_ATTRS; } for (i = sc->first_vty; i < sc->first_vty + sc->vtys; ++i) { if ((tp = SC_DEV(sc, i)) == NULL) continue; if ((scp = sc_get_stat(tp)) == NULL) continue; scp->dflt_curs_attr = sc->curs_attr; change_cursor_shape(scp, CONS_RESET_CURSOR, -1, -1); } splx(s); } static void scinit(int unit, int flags) { /* * When syscons is being initialized as the kernel console, malloc() * is not yet functional, because various kernel structures has not been * fully initialized yet. Therefore, we need to declare the following * static buffers for the console. This is less than ideal, * but is necessry evil for the time being. XXX */ #ifdef PC98 static u_short sc_buffer[ROW*COL*2];/* XXX */ #else static u_short sc_buffer[ROW*COL]; /* XXX */ #endif #ifndef SC_NO_FONT_LOADING static u_char font_8[256*8]; static u_char font_14[256*14]; static u_char font_16[256*16]; #endif sc_softc_t *sc; scr_stat *scp; video_adapter_t *adp; int col; int row; int i; /* one time initialization */ if (init_done == COLD) sc_get_bios_values(&bios_value); init_done = WARM; /* * Allocate resources. Even if we are being called for the second * time, we must allocate them again, because they might have * disappeared... */ sc = sc_get_softc(unit, flags & SC_KERNEL_CONSOLE); if ((sc->flags & SC_INIT_DONE) == 0) SC_VIDEO_LOCKINIT(sc); adp = NULL; if (sc->adapter >= 0) { vid_release(sc->adp, (void *)&sc->adapter); adp = sc->adp; sc->adp = NULL; } if (sc->keyboard >= 0) { DPRINTF(5, ("sc%d: releasing kbd%d\n", unit, sc->keyboard)); i = kbd_release(sc->kbd, (void *)&sc->keyboard); DPRINTF(5, ("sc%d: kbd_release returned %d\n", unit, i)); if (sc->kbd != NULL) { DPRINTF(5, ("sc%d: kbd != NULL!, index:%d, unit:%d, flags:0x%x\n", unit, sc->kbd->kb_index, sc->kbd->kb_unit, sc->kbd->kb_flags)); } sc->kbd = NULL; } sc->adapter = vid_allocate("*", unit, (void *)&sc->adapter); sc->adp = vid_get_adapter(sc->adapter); /* assert((sc->adapter >= 0) && (sc->adp != NULL)) */ sc->keyboard = sc_allocate_keyboard(sc, unit); DPRINTF(1, ("sc%d: keyboard %d\n", unit, sc->keyboard)); sc->kbd = kbd_get_keyboard(sc->keyboard); if (sc->kbd != NULL) { DPRINTF(1, ("sc%d: kbd index:%d, unit:%d, flags:0x%x\n", unit, sc->kbd->kb_index, sc->kbd->kb_unit, sc->kbd->kb_flags)); } if (!(sc->flags & SC_INIT_DONE) || (adp != sc->adp)) { sc->initial_mode = sc->adp->va_initial_mode; #ifndef SC_NO_FONT_LOADING if (flags & SC_KERNEL_CONSOLE) { sc->font_8 = font_8; sc->font_14 = font_14; sc->font_16 = font_16; } else if (sc->font_8 == NULL) { /* assert(sc_malloc) */ sc->font_8 = malloc(sizeof(font_8), M_DEVBUF, M_WAITOK); sc->font_14 = malloc(sizeof(font_14), M_DEVBUF, M_WAITOK); sc->font_16 = malloc(sizeof(font_16), M_DEVBUF, M_WAITOK); } #endif /* extract the hardware cursor location and hide the cursor for now */ vidd_read_hw_cursor(sc->adp, &col, &row); vidd_set_hw_cursor(sc->adp, -1, -1); /* set up the first console */ sc->first_vty = unit*MAXCONS; sc->vtys = MAXCONS; /* XXX: should be configurable */ if (flags & SC_KERNEL_CONSOLE) { /* * Set up devs structure but don't use it yet, calling make_dev() * might panic kernel. Wait for sc_attach_unit() to actually * create the devices. */ sc->dev = main_devs; scp = &main_console; init_scp(sc, sc->first_vty, scp); sc_vtb_init(&scp->vtb, VTB_MEMORY, scp->xsize, scp->ysize, (void *)sc_buffer, FALSE); /* move cursors to the initial positions */ if (col >= scp->xsize) col = 0; if (row >= scp->ysize) row = scp->ysize - 1; scp->xpos = col; scp->ypos = row; scp->cursor_pos = scp->cursor_oldpos = row*scp->xsize + col; if (sc_init_emulator(scp, SC_DFLT_TERM)) sc_init_emulator(scp, "*"); (*scp->tsw->te_default_attr)(scp, user_default.std_color, user_default.rev_color); } else { /* assert(sc_malloc) */ sc->dev = malloc(sizeof(struct tty *)*sc->vtys, M_DEVBUF, M_WAITOK|M_ZERO); sc->dev[0] = sc_alloc_tty(0, unit * MAXCONS); scp = alloc_scp(sc, sc->first_vty); SC_STAT(sc->dev[0]) = scp; } sc->cur_scp = scp; #ifndef __sparc64__ /* copy screen to temporary buffer */ sc_vtb_init(&scp->scr, VTB_FRAMEBUFFER, scp->xsize, scp->ysize, (void *)scp->sc->adp->va_window, FALSE); if (ISTEXTSC(scp)) sc_vtb_copy(&scp->scr, 0, &scp->vtb, 0, scp->xsize*scp->ysize); #endif if (bios_value.cursor_end < scp->font_size) sc->dflt_curs_attr.base = scp->font_size - bios_value.cursor_end - 1; else sc->dflt_curs_attr.base = 0; i = bios_value.cursor_end - bios_value.cursor_start + 1; sc->dflt_curs_attr.height = imin(i, scp->font_size); sc->dflt_curs_attr.flags = 0; sc->curs_attr = sc->dflt_curs_attr; scp->curr_curs_attr = scp->dflt_curs_attr = sc->curs_attr; #ifndef SC_NO_SYSMOUSE sc_mouse_move(scp, scp->xpixel/2, scp->ypixel/2); #endif if (!ISGRAPHSC(scp)) { sc_set_cursor_image(scp); sc_draw_cursor_image(scp); } /* save font and palette */ #ifndef SC_NO_FONT_LOADING sc->fonts_loaded = 0; if (ISFONTAVAIL(sc->adp->va_flags)) { #ifdef SC_DFLT_FONT bcopy(dflt_font_8, sc->font_8, sizeof(dflt_font_8)); bcopy(dflt_font_14, sc->font_14, sizeof(dflt_font_14)); bcopy(dflt_font_16, sc->font_16, sizeof(dflt_font_16)); sc->fonts_loaded = FONT_16 | FONT_14 | FONT_8; if (scp->font_size < 14) { sc_load_font(scp, 0, 8, 8, sc->font_8, 0, 256); } else if (scp->font_size >= 16) { sc_load_font(scp, 0, 16, 8, sc->font_16, 0, 256); } else { sc_load_font(scp, 0, 14, 8, sc->font_14, 0, 256); } #else /* !SC_DFLT_FONT */ if (scp->font_size < 14) { sc_save_font(scp, 0, 8, 8, sc->font_8, 0, 256); sc->fonts_loaded = FONT_8; } else if (scp->font_size >= 16) { sc_save_font(scp, 0, 16, 8, sc->font_16, 0, 256); sc->fonts_loaded = FONT_16; } else { sc_save_font(scp, 0, 14, 8, sc->font_14, 0, 256); sc->fonts_loaded = FONT_14; } #endif /* SC_DFLT_FONT */ /* FONT KLUDGE: always use the font page #0. XXX */ sc_show_font(scp, 0); } #endif /* !SC_NO_FONT_LOADING */ #ifndef SC_NO_PALETTE_LOADING vidd_save_palette(sc->adp, sc->palette); #ifdef SC_PIXEL_MODE for (i = 0; i < sizeof(sc->palette2); i++) sc->palette2[i] = i / 3; #endif #endif #ifdef DEV_SPLASH if (!(sc->flags & SC_SPLASH_SCRN)) { /* we are ready to put up the splash image! */ splash_init(sc->adp, scsplash_callback, sc); sc->flags |= SC_SPLASH_SCRN; } #endif } /* the rest is not necessary, if we have done it once */ if (sc->flags & SC_INIT_DONE) return; /* initialize mapscrn arrays to a one to one map */ for (i = 0; i < sizeof(sc->scr_map); i++) sc->scr_map[i] = sc->scr_rmap[i] = i; #ifdef PC98 sc->scr_map[0x5c] = (u_char)0xfc; /* for backslash */ #endif sc->flags |= SC_INIT_DONE; } static void scterm(int unit, int flags) { sc_softc_t *sc; scr_stat *scp; sc = sc_get_softc(unit, flags & SC_KERNEL_CONSOLE); if (sc == NULL) return; /* shouldn't happen */ #ifdef DEV_SPLASH /* this console is no longer available for the splash screen */ if (sc->flags & SC_SPLASH_SCRN) { splash_term(sc->adp); sc->flags &= ~SC_SPLASH_SCRN; } #endif #if 0 /* XXX */ /* move the hardware cursor to the upper-left corner */ vidd_set_hw_cursor(sc->adp, 0, 0); #endif /* release the keyboard and the video card */ if (sc->keyboard >= 0) kbd_release(sc->kbd, &sc->keyboard); if (sc->adapter >= 0) vid_release(sc->adp, &sc->adapter); /* stop the terminal emulator, if any */ scp = sc_get_stat(sc->dev[0]); if (scp->tsw) (*scp->tsw->te_term)(scp, &scp->ts); if (scp->ts != NULL) free(scp->ts, M_DEVBUF); mtx_destroy(&sc->video_mtx); /* clear the structure */ if (!(flags & SC_KERNEL_CONSOLE)) { /* XXX: We need delete_dev() for this */ free(sc->dev, M_DEVBUF); #if 0 /* XXX: We need a ttyunregister for this */ free(sc->tty, M_DEVBUF); #endif #ifndef SC_NO_FONT_LOADING free(sc->font_8, M_DEVBUF); free(sc->font_14, M_DEVBUF); free(sc->font_16, M_DEVBUF); #endif /* XXX vtb, history */ } bzero(sc, sizeof(*sc)); sc->keyboard = -1; sc->adapter = -1; } static void scshutdown(__unused void *arg, __unused int howto) { KASSERT(sc_console != NULL, ("sc_console != NULL")); KASSERT(sc_console->sc != NULL, ("sc_console->sc != NULL")); KASSERT(sc_console->sc->cur_scp != NULL, ("sc_console->sc->cur_scp != NULL")); sc_touch_scrn_saver(); if (!cold && sc_console->sc->cur_scp->index != sc_console->index && sc_console->sc->cur_scp->smode.mode == VT_AUTO && sc_console->smode.mode == VT_AUTO) sc_switch_scr(sc_console->sc, sc_console->index); shutdown_in_progress = TRUE; } static void scsuspend(__unused void *arg) { int retry; KASSERT(sc_console != NULL, ("sc_console != NULL")); KASSERT(sc_console->sc != NULL, ("sc_console->sc != NULL")); KASSERT(sc_console->sc->cur_scp != NULL, ("sc_console->sc->cur_scp != NULL")); sc_susp_scr = sc_console->sc->cur_scp->index; if (sc_no_suspend_vtswitch || sc_susp_scr == sc_console->index) { sc_touch_scrn_saver(); sc_susp_scr = -1; return; } for (retry = 0; retry < 10; retry++) { sc_switch_scr(sc_console->sc, sc_console->index); if (!sc_console->sc->switch_in_progress) break; pause("scsuspend", hz); } suspend_in_progress = TRUE; } static void scresume(__unused void *arg) { KASSERT(sc_console != NULL, ("sc_console != NULL")); KASSERT(sc_console->sc != NULL, ("sc_console->sc != NULL")); KASSERT(sc_console->sc->cur_scp != NULL, ("sc_console->sc->cur_scp != NULL")); suspend_in_progress = FALSE; if (sc_susp_scr < 0) { update_font(sc_console->sc->cur_scp); return; } sc_switch_scr(sc_console->sc, sc_susp_scr); } int sc_clean_up(scr_stat *scp) { #ifdef DEV_SPLASH int error; #endif if (scp->sc->flags & SC_SCRN_BLANKED) { sc_touch_scrn_saver(); #ifdef DEV_SPLASH if ((error = wait_scrn_saver_stop(scp->sc))) return error; #endif } scp->status |= MOUSE_HIDDEN; sc_remove_mouse_image(scp); sc_remove_cutmarking(scp); return 0; } void sc_alloc_scr_buffer(scr_stat *scp, int wait, int discard) { sc_vtb_t new; sc_vtb_t old; old = scp->vtb; sc_vtb_init(&new, VTB_MEMORY, scp->xsize, scp->ysize, NULL, wait); if (!discard && (old.vtb_flags & VTB_VALID)) { /* retain the current cursor position and buffer contants */ scp->cursor_oldpos = scp->cursor_pos; /* * This works only if the old buffer has the same size as or larger * than the new one. XXX */ sc_vtb_copy(&old, 0, &new, 0, scp->xsize*scp->ysize); scp->vtb = new; } else { scp->vtb = new; sc_vtb_destroy(&old); } #ifndef SC_NO_SYSMOUSE /* move the mouse cursor at the center of the screen */ sc_mouse_move(scp, scp->xpixel / 2, scp->ypixel / 2); #endif } static scr_stat *alloc_scp(sc_softc_t *sc, int vty) { scr_stat *scp; /* assert(sc_malloc) */ scp = (scr_stat *)malloc(sizeof(scr_stat), M_DEVBUF, M_WAITOK); init_scp(sc, vty, scp); sc_alloc_scr_buffer(scp, TRUE, TRUE); if (sc_init_emulator(scp, SC_DFLT_TERM)) sc_init_emulator(scp, "*"); #ifndef SC_NO_CUTPASTE sc_alloc_cut_buffer(scp, TRUE); #endif #ifndef SC_NO_HISTORY sc_alloc_history_buffer(scp, 0, 0, TRUE); #endif return scp; } static void init_scp(sc_softc_t *sc, int vty, scr_stat *scp) { video_info_t info; bzero(scp, sizeof(*scp)); scp->index = vty; scp->sc = sc; scp->status = 0; scp->mode = sc->initial_mode; vidd_get_info(sc->adp, scp->mode, &info); if (info.vi_flags & V_INFO_GRAPHICS) { scp->status |= GRAPHICS_MODE; scp->xpixel = info.vi_width; scp->ypixel = info.vi_height; scp->xsize = info.vi_width/info.vi_cwidth; scp->ysize = info.vi_height/info.vi_cheight; scp->font_size = 0; scp->font = NULL; } else { scp->xsize = info.vi_width; scp->ysize = info.vi_height; scp->xpixel = scp->xsize*info.vi_cwidth; scp->ypixel = scp->ysize*info.vi_cheight; } scp->font_size = info.vi_cheight; scp->font_width = info.vi_cwidth; #ifndef SC_NO_FONT_LOADING if (info.vi_cheight < 14) scp->font = sc->font_8; else if (info.vi_cheight >= 16) scp->font = sc->font_16; else scp->font = sc->font_14; #else scp->font = NULL; #endif sc_vtb_init(&scp->vtb, VTB_MEMORY, 0, 0, NULL, FALSE); #ifndef __sparc64__ sc_vtb_init(&scp->scr, VTB_FRAMEBUFFER, 0, 0, NULL, FALSE); #endif scp->xoff = scp->yoff = 0; scp->xpos = scp->ypos = 0; scp->start = scp->xsize * scp->ysize - 1; scp->end = 0; scp->tsw = NULL; scp->ts = NULL; scp->rndr = NULL; scp->border = (SC_NORM_ATTR >> 4) & 0x0f; scp->curr_curs_attr = scp->dflt_curs_attr = sc->curs_attr; scp->mouse_cut_start = scp->xsize*scp->ysize; scp->mouse_cut_end = -1; scp->mouse_signal = 0; scp->mouse_pid = 0; scp->mouse_proc = NULL; scp->kbd_mode = K_XLATE; scp->bell_pitch = bios_value.bell_pitch; scp->bell_duration = BELL_DURATION; scp->status |= (bios_value.shift_state & NLKED); scp->status |= CURSOR_ENABLED | MOUSE_HIDDEN; scp->pid = 0; scp->proc = NULL; scp->smode.mode = VT_AUTO; scp->history = NULL; scp->history_pos = 0; scp->history_size = 0; } int sc_init_emulator(scr_stat *scp, char *name) { sc_term_sw_t *sw; sc_rndr_sw_t *rndr; void *p; int error; if (name == NULL) /* if no name is given, use the current emulator */ sw = scp->tsw; else /* ...otherwise find the named emulator */ sw = sc_term_match(name); if (sw == NULL) return EINVAL; rndr = NULL; if (strcmp(sw->te_renderer, "*") != 0) { rndr = sc_render_match(scp, sw->te_renderer, scp->status & (GRAPHICS_MODE | PIXEL_MODE)); } if (rndr == NULL) { rndr = sc_render_match(scp, scp->sc->adp->va_name, scp->status & (GRAPHICS_MODE | PIXEL_MODE)); if (rndr == NULL) return ENODEV; } if (sw == scp->tsw) { error = (*sw->te_init)(scp, &scp->ts, SC_TE_WARM_INIT); scp->rndr = rndr; scp->rndr->init(scp); sc_clear_screen(scp); /* assert(error == 0); */ return error; } if (sc_malloc && (sw->te_size > 0)) p = malloc(sw->te_size, M_DEVBUF, M_NOWAIT); else p = NULL; error = (*sw->te_init)(scp, &p, SC_TE_COLD_INIT); if (error) return error; if (scp->tsw) (*scp->tsw->te_term)(scp, &scp->ts); if (scp->ts != NULL) free(scp->ts, M_DEVBUF); scp->tsw = sw; scp->ts = p; scp->rndr = rndr; scp->rndr->init(scp); /* XXX */ (*sw->te_default_attr)(scp, user_default.std_color, user_default.rev_color); sc_clear_screen(scp); return 0; } /* * scgetc(flags) - get character from keyboard. * If flags & SCGETC_CN, then avoid harmful side effects. * If flags & SCGETC_NONBLOCK, then wait until a key is pressed, else * return NOKEY if there is nothing there. */ static u_int scgetc(sc_softc_t *sc, u_int flags, struct sc_cnstate *sp) { scr_stat *scp; #ifndef SC_NO_HISTORY struct tty *tp; #endif u_int c; int this_scr; int f; int i; if (sc->kbd == NULL) return NOKEY; next_code: #if 1 /* I don't like this, but... XXX */ if (flags & SCGETC_CN) sccnupdate(sc->cur_scp); #endif scp = sc->cur_scp; /* first see if there is something in the keyboard port */ for (;;) { c = kbdd_read_char(sc->kbd, !(flags & SCGETC_NONBLOCK)); if (c == ERRKEY) { if (!(flags & SCGETC_CN)) sc_bell(scp, bios_value.bell_pitch, BELL_DURATION); } else if (c == NOKEY) return c; else break; } /* make screensaver happy */ if (!(c & RELKEY)) sc_touch_scrn_saver(); if (!(flags & SCGETC_CN)) random_harvest_queue(&c, sizeof(c), 1, RANDOM_KEYBOARD); if (sc->kbd_open_level == 0 && scp->kbd_mode != K_XLATE) return KEYCHAR(c); /* if scroll-lock pressed allow history browsing */ if (!ISGRAPHSC(scp) && scp->history && scp->status & SLKED) { scp->status &= ~CURSOR_ENABLED; sc_remove_cursor_image(scp); #ifndef SC_NO_HISTORY if (!(scp->status & BUFFER_SAVED)) { scp->status |= BUFFER_SAVED; sc_hist_save(scp); } switch (c) { /* FIXME: key codes */ case SPCLKEY | FKEY | F(49): /* home key */ sc_remove_cutmarking(scp); sc_hist_home(scp); goto next_code; case SPCLKEY | FKEY | F(57): /* end key */ sc_remove_cutmarking(scp); sc_hist_end(scp); goto next_code; case SPCLKEY | FKEY | F(50): /* up arrow key */ sc_remove_cutmarking(scp); if (sc_hist_up_line(scp)) if (!(flags & SCGETC_CN)) sc_bell(scp, bios_value.bell_pitch, BELL_DURATION); goto next_code; case SPCLKEY | FKEY | F(58): /* down arrow key */ sc_remove_cutmarking(scp); if (sc_hist_down_line(scp)) if (!(flags & SCGETC_CN)) sc_bell(scp, bios_value.bell_pitch, BELL_DURATION); goto next_code; case SPCLKEY | FKEY | F(51): /* page up key */ sc_remove_cutmarking(scp); for (i=0; iysize; i++) if (sc_hist_up_line(scp)) { if (!(flags & SCGETC_CN)) sc_bell(scp, bios_value.bell_pitch, BELL_DURATION); break; } goto next_code; case SPCLKEY | FKEY | F(59): /* page down key */ sc_remove_cutmarking(scp); for (i=0; iysize; i++) if (sc_hist_down_line(scp)) { if (!(flags & SCGETC_CN)) sc_bell(scp, bios_value.bell_pitch, BELL_DURATION); break; } goto next_code; } #endif /* SC_NO_HISTORY */ } /* * Process and consume special keys here. Return a plain char code * or a char code with the META flag or a function key code. */ if (c & RELKEY) { /* key released */ /* goto next_code */ } else { /* key pressed */ if (c & SPCLKEY) { c &= ~SPCLKEY; switch (KEYCHAR(c)) { /* LOCKING KEYS */ case NLK: case CLK: case ALK: break; case SLK: (void)kbdd_ioctl(sc->kbd, KDGKBSTATE, (caddr_t)&f); if (f & SLKED) { scp->status |= SLKED; } else { if (scp->status & SLKED) { scp->status &= ~SLKED; #ifndef SC_NO_HISTORY if (scp->status & BUFFER_SAVED) { if (!sc_hist_restore(scp)) sc_remove_cutmarking(scp); scp->status &= ~BUFFER_SAVED; scp->status |= CURSOR_ENABLED; sc_draw_cursor_image(scp); } /* Only safe in Giant-locked context. */ tp = SC_DEV(sc, scp->index); if (!(flags & SCGETC_CN) && tty_opened_ns(tp)) sctty_outwakeup(tp); #endif } } break; case PASTE: #ifndef SC_NO_CUTPASTE sc_mouse_paste(scp); #endif break; /* NON-LOCKING KEYS */ case NOP: case LSH: case RSH: case LCTR: case RCTR: case LALT: case RALT: case ASH: case META: break; case BTAB: if (!(sc->flags & SC_SCRN_BLANKED)) return c; break; case SPSC: #ifdef DEV_SPLASH /* force activatation/deactivation of the screen saver */ if (!(sc->flags & SC_SCRN_BLANKED)) { run_scrn_saver = TRUE; sc->scrn_time_stamp -= scrn_blank_time; } if (cold) { /* * While devices are being probed, the screen saver need * to be invoked explicitly. XXX */ if (sc->flags & SC_SCRN_BLANKED) { scsplash_stick(FALSE); stop_scrn_saver(sc, current_saver); } else { if (!ISGRAPHSC(scp)) { scsplash_stick(TRUE); (*current_saver)(sc, TRUE); } } } #endif /* DEV_SPLASH */ break; case RBT: #ifndef SC_DISABLE_REBOOT if (enable_reboot && !(flags & SCGETC_CN)) shutdown_nice(0); #endif break; case HALT: #ifndef SC_DISABLE_REBOOT if (enable_reboot && !(flags & SCGETC_CN)) shutdown_nice(RB_HALT); #endif break; case PDWN: #ifndef SC_DISABLE_REBOOT if (enable_reboot && !(flags & SCGETC_CN)) shutdown_nice(RB_HALT|RB_POWEROFF); #endif break; case SUSP: power_pm_suspend(POWER_SLEEP_STATE_SUSPEND); break; case STBY: power_pm_suspend(POWER_SLEEP_STATE_STANDBY); break; case DBG: #ifndef SC_DISABLE_KDBKEY if (enable_kdbkey) kdb_break(); #endif break; case PNC: if (enable_panic_key) panic("Forced by the panic key"); break; case NEXT: this_scr = scp->index; for (i = (this_scr - sc->first_vty + 1)%sc->vtys; sc->first_vty + i != this_scr; i = (i + 1)%sc->vtys) { struct tty *tp = SC_DEV(sc, sc->first_vty + i); if (tty_opened_ns(tp)) { sc_switch_scr(scp->sc, sc->first_vty + i); break; } } break; case PREV: this_scr = scp->index; for (i = (this_scr - sc->first_vty + sc->vtys - 1)%sc->vtys; sc->first_vty + i != this_scr; i = (i + sc->vtys - 1)%sc->vtys) { struct tty *tp = SC_DEV(sc, sc->first_vty + i); if (tty_opened_ns(tp)) { sc_switch_scr(scp->sc, sc->first_vty + i); break; } } break; default: if (KEYCHAR(c) >= F_SCR && KEYCHAR(c) <= L_SCR) { sc_switch_scr(scp->sc, sc->first_vty + KEYCHAR(c) - F_SCR); break; } /* assert(c & FKEY) */ if (!(sc->flags & SC_SCRN_BLANKED)) return c; break; } /* goto next_code */ } else { /* regular keys (maybe MKEY is set) */ #if !defined(SC_DISABLE_KDBKEY) && defined(KDB) if (enable_kdbkey) kdb_alt_break(c, &sc->sc_altbrk); #endif if (!(sc->flags & SC_SCRN_BLANKED)) return c; } } goto next_code; } static int sctty_mmap(struct tty *tp, vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr) { scr_stat *scp; scp = sc_get_stat(tp); if (scp != scp->sc->cur_scp) return -1; return vidd_mmap(scp->sc->adp, offset, paddr, nprot, memattr); } static void update_font(scr_stat *scp) { #ifndef SC_NO_FONT_LOADING /* load appropriate font */ if (!(scp->status & GRAPHICS_MODE)) { if (!(scp->status & PIXEL_MODE) && ISFONTAVAIL(scp->sc->adp->va_flags)) { if (scp->font_size < 14) { if (scp->sc->fonts_loaded & FONT_8) sc_load_font(scp, 0, 8, 8, scp->sc->font_8, 0, 256); } else if (scp->font_size >= 16) { if (scp->sc->fonts_loaded & FONT_16) sc_load_font(scp, 0, 16, 8, scp->sc->font_16, 0, 256); } else { if (scp->sc->fonts_loaded & FONT_14) sc_load_font(scp, 0, 14, 8, scp->sc->font_14, 0, 256); } /* * FONT KLUDGE: * This is an interim kludge to display correct font. * Always use the font page #0 on the video plane 2. * Somehow we cannot show the font in other font pages on * some video cards... XXX */ sc_show_font(scp, 0); } mark_all(scp); } #endif /* !SC_NO_FONT_LOADING */ } static int save_kbd_state(scr_stat *scp) { int state; int error; error = kbdd_ioctl(scp->sc->kbd, KDGKBSTATE, (caddr_t)&state); if (error == ENOIOCTL) error = ENODEV; if (error == 0) { scp->status &= ~LOCK_MASK; scp->status |= state; } return error; } static int update_kbd_state(scr_stat *scp, int new_bits, int mask) { int state; int error; if (mask != LOCK_MASK) { error = kbdd_ioctl(scp->sc->kbd, KDGKBSTATE, (caddr_t)&state); if (error == ENOIOCTL) error = ENODEV; if (error) return error; state &= ~mask; state |= new_bits & mask; } else { state = new_bits & LOCK_MASK; } error = kbdd_ioctl(scp->sc->kbd, KDSKBSTATE, (caddr_t)&state); if (error == ENOIOCTL) error = ENODEV; return error; } static int update_kbd_leds(scr_stat *scp, int which) { int error; which &= LOCK_MASK; error = kbdd_ioctl(scp->sc->kbd, KDSETLED, (caddr_t)&which); if (error == ENOIOCTL) error = ENODEV; return error; } int set_mode(scr_stat *scp) { video_info_t info; /* reject unsupported mode */ if (vidd_get_info(scp->sc->adp, scp->mode, &info)) return 1; /* if this vty is not currently showing, do nothing */ if (scp != scp->sc->cur_scp) return 0; /* setup video hardware for the given mode */ vidd_set_mode(scp->sc->adp, scp->mode); scp->rndr->init(scp); #ifndef __sparc64__ sc_vtb_init(&scp->scr, VTB_FRAMEBUFFER, scp->xsize, scp->ysize, (void *)scp->sc->adp->va_window, FALSE); #endif update_font(scp); sc_set_border(scp, scp->border); sc_set_cursor_image(scp); return 0; } void sc_set_border(scr_stat *scp, int color) { SC_VIDEO_LOCK(scp->sc); (*scp->rndr->draw_border)(scp, color); SC_VIDEO_UNLOCK(scp->sc); } #ifndef SC_NO_FONT_LOADING void sc_load_font(scr_stat *scp, int page, int size, int width, u_char *buf, int base, int count) { sc_softc_t *sc; sc = scp->sc; sc->font_loading_in_progress = TRUE; vidd_load_font(sc->adp, page, size, width, buf, base, count); sc->font_loading_in_progress = FALSE; } void sc_save_font(scr_stat *scp, int page, int size, int width, u_char *buf, int base, int count) { sc_softc_t *sc; sc = scp->sc; sc->font_loading_in_progress = TRUE; vidd_save_font(sc->adp, page, size, width, buf, base, count); sc->font_loading_in_progress = FALSE; } void sc_show_font(scr_stat *scp, int page) { vidd_show_font(scp->sc->adp, page); } #endif /* !SC_NO_FONT_LOADING */ void sc_paste(scr_stat *scp, const u_char *p, int count) { struct tty *tp; u_char *rmap; tp = SC_DEV(scp->sc, scp->sc->cur_scp->index); if (!tty_opened_ns(tp)) return; rmap = scp->sc->scr_rmap; for (; count > 0; --count) ttydisc_rint(tp, rmap[*p++], 0); ttydisc_rint_done(tp); } void sc_respond(scr_stat *scp, const u_char *p, int count, int wakeup) { struct tty *tp; tp = SC_DEV(scp->sc, scp->sc->cur_scp->index); if (!tty_opened_ns(tp)) return; ttydisc_rint_simple(tp, p, count); if (wakeup) { /* XXX: we can't always call ttydisc_rint_done() here! */ ttydisc_rint_done(tp); } } void sc_bell(scr_stat *scp, int pitch, int duration) { if (cold || kdb_active || shutdown_in_progress || !enable_bell) return; if (scp != scp->sc->cur_scp && (scp->sc->flags & SC_QUIET_BELL)) return; if (scp->sc->flags & SC_VISUAL_BELL) { if (scp->sc->blink_in_progress) return; scp->sc->blink_in_progress = 3; if (scp != scp->sc->cur_scp) scp->sc->blink_in_progress += 2; blink_screen(scp->sc->cur_scp); } else if (duration != 0 && pitch != 0) { if (scp != scp->sc->cur_scp) pitch *= 2; sysbeep(1193182 / pitch, duration); } } static void blink_screen(void *arg) { scr_stat *scp = arg; struct tty *tp; if (ISGRAPHSC(scp) || (scp->sc->blink_in_progress <= 1)) { scp->sc->blink_in_progress = 0; mark_all(scp); tp = SC_DEV(scp->sc, scp->index); if (tty_opened_ns(tp)) sctty_outwakeup(tp); if (scp->sc->delayed_next_scr) sc_switch_scr(scp->sc, scp->sc->delayed_next_scr - 1); } else { (*scp->rndr->draw)(scp, 0, scp->xsize*scp->ysize, scp->sc->blink_in_progress & 1); scp->sc->blink_in_progress--; callout_reset_sbt(&scp->sc->cblink, SBT_1S / 15, 0, blink_screen, scp, C_PREL(0)); } } /* * Until sc_attach_unit() gets called no dev structures will be available * to store the per-screen current status. This is the case when the * kernel is initially booting and needs access to its console. During * this early phase of booting the console's current status is kept in * one statically defined scr_stat structure, and any pointers to the * dev structures will be NULL. */ static scr_stat * sc_get_stat(struct tty *tp) { if (tp == NULL) return (&main_console); return (SC_STAT(tp)); } /* * Allocate active keyboard. Try to allocate "kbdmux" keyboard first, and, * if found, add all non-busy keyboards to "kbdmux". Otherwise look for * any keyboard. */ static int sc_allocate_keyboard(sc_softc_t *sc, int unit) { int idx0, idx; keyboard_t *k0, *k; keyboard_info_t ki; idx0 = kbd_allocate("kbdmux", -1, (void *)&sc->keyboard, sckbdevent, sc); if (idx0 != -1) { k0 = kbd_get_keyboard(idx0); for (idx = kbd_find_keyboard2("*", -1, 0); idx != -1; idx = kbd_find_keyboard2("*", -1, idx + 1)) { k = kbd_get_keyboard(idx); if (idx == idx0 || KBD_IS_BUSY(k)) continue; bzero(&ki, sizeof(ki)); strcpy(ki.kb_name, k->kb_name); ki.kb_unit = k->kb_unit; (void)kbdd_ioctl(k0, KBADDKBD, (caddr_t) &ki); } } else idx0 = kbd_allocate("*", unit, (void *)&sc->keyboard, sckbdevent, sc); return (idx0); } Index: projects/clang390-import/sys/dev/tsec/if_tsec.h =================================================================== --- projects/clang390-import/sys/dev/tsec/if_tsec.h (revision 305080) +++ projects/clang390-import/sys/dev/tsec/if_tsec.h (revision 305081) @@ -1,384 +1,387 @@ /*- * Copyright (C) 2006-2007 Semihalf, Piotr Kruszynski * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _IF_TSEC_H #define _IF_TSEC_H #include #define TSEC_RX_NUM_DESC 256 #define TSEC_TX_NUM_DESC 256 /* Interrupt Coalescing types */ #define TSEC_IC_RX 0 #define TSEC_IC_TX 1 /* eTSEC ID */ #define TSEC_ETSEC_ID 0x0124 /* Frame sizes */ #define TSEC_MIN_FRAME_SIZE 64 #define TSEC_MAX_FRAME_SIZE 9600 struct tsec_softc { /* XXX MII bus requires that struct ifnet is first!!! */ struct ifnet *tsec_ifp; struct mtx transmit_lock; /* transmitter lock */ struct mtx receive_lock; /* receiver lock */ phandle_t node; device_t dev; device_t tsec_miibus; struct mii_data *tsec_mii; /* MII media control */ int tsec_link; bus_dma_tag_t tsec_tx_dtag; /* TX descriptors tag */ bus_dmamap_t tsec_tx_dmap; /* TX descriptors map */ struct tsec_desc *tsec_tx_vaddr;/* vadress of TX descriptors */ uint32_t tsec_tx_raddr; /* real address of TX descriptors */ bus_dma_tag_t tsec_rx_dtag; /* RX descriptors tag */ bus_dmamap_t tsec_rx_dmap; /* RX descriptors map */ struct tsec_desc *tsec_rx_vaddr; /* vadress of RX descriptors */ uint32_t tsec_rx_raddr; /* real address of RX descriptors */ bus_dma_tag_t tsec_tx_mtag; /* TX mbufs tag */ bus_dma_tag_t tsec_rx_mtag; /* TX mbufs tag */ struct rx_data_type { bus_dmamap_t map; /* mbuf map */ struct mbuf *mbuf; uint32_t paddr; /* DMA address of buffer */ } rx_data[TSEC_RX_NUM_DESC]; uint32_t tx_cur_desc_cnt; uint32_t tx_dirty_desc_cnt; uint32_t rx_cur_desc_cnt; struct resource *sc_rres; /* register resource */ int sc_rrid; /* register rid */ struct { bus_space_tag_t bst; bus_space_handle_t bsh; } sc_bas; struct resource *sc_transmit_ires; void *sc_transmit_ihand; int sc_transmit_irid; struct resource *sc_receive_ires; void *sc_receive_ihand; int sc_receive_irid; struct resource *sc_error_ires; void *sc_error_ihand; int sc_error_irid; int tsec_if_flags; int is_etsec; /* Watchdog and MII tick related */ struct callout tsec_callout; int tsec_watchdog; /* TX maps */ bus_dmamap_t tx_map_data[TSEC_TX_NUM_DESC]; /* unused TX maps data */ uint32_t tx_map_unused_get_cnt; uint32_t tx_map_unused_put_cnt; bus_dmamap_t *tx_map_unused_data[TSEC_TX_NUM_DESC]; /* used TX maps data */ uint32_t tx_map_used_get_cnt; uint32_t tx_map_used_put_cnt; bus_dmamap_t *tx_map_used_data[TSEC_TX_NUM_DESC]; /* mbufs in TX queue */ uint32_t tx_mbuf_used_get_cnt; uint32_t tx_mbuf_used_put_cnt; struct mbuf *tx_mbuf_used_data[TSEC_TX_NUM_DESC]; /* interrupt coalescing */ struct mtx ic_lock; uint32_t rx_ic_time; /* RW, valid values 0..65535 */ uint32_t rx_ic_count; /* RW, valid values 0..255 */ uint32_t tx_ic_time; uint32_t tx_ic_count; /* currently received frame */ struct mbuf *frame; int phyaddr; bus_space_tag_t phy_bst; bus_space_handle_t phy_bsh; + int phy_regoff; }; /* interface to get/put generic objects */ #define TSEC_CNT_INIT(cnt, wrap) ((cnt) = ((wrap) - 1)) #define TSEC_INC(count, wrap) (count = ((count) + 1) & ((wrap) - 1)) #define TSEC_GET_GENERIC(hand, tab, count, wrap) \ ((hand)->tab[TSEC_INC((hand)->count, wrap)]) #define TSEC_PUT_GENERIC(hand, tab, count, wrap, val) \ ((hand)->tab[TSEC_INC((hand)->count, wrap)] = val) #define TSEC_BACK_GENERIC(sc, count, wrap) do { \ if ((sc)->count > 0) \ (sc)->count--; \ else \ (sc)->count = (wrap) - 1; \ } while (0) /* TX maps interface */ #define TSEC_TX_MAP_CNT_INIT(sc) do { \ TSEC_CNT_INIT((sc)->tx_map_unused_get_cnt, TSEC_TX_NUM_DESC); \ TSEC_CNT_INIT((sc)->tx_map_unused_put_cnt, TSEC_TX_NUM_DESC); \ TSEC_CNT_INIT((sc)->tx_map_used_get_cnt, TSEC_TX_NUM_DESC); \ TSEC_CNT_INIT((sc)->tx_map_used_put_cnt, TSEC_TX_NUM_DESC); \ } while (0) /* interface to get/put unused TX maps */ #define TSEC_ALLOC_TX_MAP(sc) \ TSEC_GET_GENERIC(sc, tx_map_unused_data, tx_map_unused_get_cnt, \ TSEC_TX_NUM_DESC) #define TSEC_FREE_TX_MAP(sc, val) \ TSEC_PUT_GENERIC(sc, tx_map_unused_data, tx_map_unused_put_cnt, \ TSEC_TX_NUM_DESC, val) /* interface to get/put used TX maps */ #define TSEC_GET_TX_MAP(sc) \ TSEC_GET_GENERIC(sc, tx_map_used_data, tx_map_used_get_cnt, \ TSEC_TX_NUM_DESC) #define TSEC_PUT_TX_MAP(sc, val) \ TSEC_PUT_GENERIC(sc, tx_map_used_data, tx_map_used_put_cnt, \ TSEC_TX_NUM_DESC, val) /* interface to get/put TX mbufs in send queue */ #define TSEC_TX_MBUF_CNT_INIT(sc) do { \ TSEC_CNT_INIT((sc)->tx_mbuf_used_get_cnt, TSEC_TX_NUM_DESC); \ TSEC_CNT_INIT((sc)->tx_mbuf_used_put_cnt, TSEC_TX_NUM_DESC); \ } while (0) #define TSEC_GET_TX_MBUF(sc) \ TSEC_GET_GENERIC(sc, tx_mbuf_used_data, tx_mbuf_used_get_cnt, \ TSEC_TX_NUM_DESC) #define TSEC_PUT_TX_MBUF(sc, val) \ TSEC_PUT_GENERIC(sc, tx_mbuf_used_data, tx_mbuf_used_put_cnt, \ TSEC_TX_NUM_DESC, val) #define TSEC_EMPTYQ_TX_MBUF(sc) \ ((sc)->tx_mbuf_used_get_cnt == (sc)->tx_mbuf_used_put_cnt) /* interface for manage tx tsec_desc */ #define TSEC_TX_DESC_CNT_INIT(sc) do { \ TSEC_CNT_INIT((sc)->tx_cur_desc_cnt, TSEC_TX_NUM_DESC); \ TSEC_CNT_INIT((sc)->tx_dirty_desc_cnt, TSEC_TX_NUM_DESC); \ } while (0) #define TSEC_GET_CUR_TX_DESC(sc) \ &TSEC_GET_GENERIC(sc, tsec_tx_vaddr, tx_cur_desc_cnt, \ TSEC_TX_NUM_DESC) #define TSEC_GET_DIRTY_TX_DESC(sc) \ &TSEC_GET_GENERIC(sc, tsec_tx_vaddr, tx_dirty_desc_cnt, \ TSEC_TX_NUM_DESC) #define TSEC_BACK_DIRTY_TX_DESC(sc) \ TSEC_BACK_GENERIC(sc, tx_dirty_desc_cnt, TSEC_TX_NUM_DESC) #define TSEC_CUR_DIFF_DIRTY_TX_DESC(sc) \ ((sc)->tx_cur_desc_cnt != (sc)->tx_dirty_desc_cnt) #define TSEC_FREE_TX_DESC(sc) \ (((sc)->tx_cur_desc_cnt < (sc)->tx_dirty_desc_cnt) ? \ ((sc)->tx_dirty_desc_cnt - (sc)->tx_cur_desc_cnt - 1) \ : \ (TSEC_TX_NUM_DESC - (sc)->tx_cur_desc_cnt \ + (sc)->tx_dirty_desc_cnt - 1)) /* interface for manage rx tsec_desc */ #define TSEC_RX_DESC_CNT_INIT(sc) do { \ TSEC_CNT_INIT((sc)->rx_cur_desc_cnt, TSEC_RX_NUM_DESC); \ } while (0) #define TSEC_GET_CUR_RX_DESC(sc) \ &TSEC_GET_GENERIC(sc, tsec_rx_vaddr, rx_cur_desc_cnt, \ TSEC_RX_NUM_DESC) #define TSEC_BACK_CUR_RX_DESC(sc) \ TSEC_BACK_GENERIC(sc, rx_cur_desc_cnt, TSEC_RX_NUM_DESC) #define TSEC_GET_CUR_RX_DESC_CNT(sc) \ ((sc)->rx_cur_desc_cnt) /* init all counters (for init only!) */ #define TSEC_TX_RX_COUNTERS_INIT(sc) do { \ TSEC_TX_MAP_CNT_INIT(sc); \ TSEC_TX_MBUF_CNT_INIT(sc); \ TSEC_TX_DESC_CNT_INIT(sc); \ TSEC_RX_DESC_CNT_INIT(sc); \ } while (0) /* read/write bus functions */ #define TSEC_READ(sc, reg) \ bus_space_read_4((sc)->sc_bas.bst, (sc)->sc_bas.bsh, (reg)) #define TSEC_WRITE(sc, reg, val) \ bus_space_write_4((sc)->sc_bas.bst, (sc)->sc_bas.bsh, (reg), (val)) extern struct mtx tsec_phy_mtx; #define TSEC_PHY_LOCK(sc) mtx_lock(&tsec_phy_mtx) #define TSEC_PHY_UNLOCK(sc) mtx_unlock(&tsec_phy_mtx) #define TSEC_PHY_READ(sc, reg) \ - bus_space_read_4((sc)->phy_bst, (sc)->phy_bsh, (reg)) + bus_space_read_4((sc)->phy_bst, (sc)->phy_bsh, \ + (reg) + (sc)->phy_regoff) #define TSEC_PHY_WRITE(sc, reg, val) \ - bus_space_write_4((sc)->phy_bst, (sc)->phy_bsh, (reg), (val)) + bus_space_write_4((sc)->phy_bst, (sc)->phy_bsh, \ + (reg) + (sc)->phy_regoff, (val)) /* Lock for transmitter */ #define TSEC_TRANSMIT_LOCK(sc) do { \ mtx_assert(&(sc)->receive_lock, MA_NOTOWNED); \ mtx_lock(&(sc)->transmit_lock); \ } while (0) #define TSEC_TRANSMIT_UNLOCK(sc) mtx_unlock(&(sc)->transmit_lock) #define TSEC_TRANSMIT_LOCK_ASSERT(sc) mtx_assert(&(sc)->transmit_lock, MA_OWNED) /* Lock for receiver */ #define TSEC_RECEIVE_LOCK(sc) do { \ mtx_assert(&(sc)->transmit_lock, MA_NOTOWNED); \ mtx_lock(&(sc)->receive_lock); \ } while (0) #define TSEC_RECEIVE_UNLOCK(sc) mtx_unlock(&(sc)->receive_lock) #define TSEC_RECEIVE_LOCK_ASSERT(sc) mtx_assert(&(sc)->receive_lock, MA_OWNED) /* Lock for interrupts coalescing */ #define TSEC_IC_LOCK(sc) do { \ mtx_assert(&(sc)->ic_lock, MA_NOTOWNED); \ mtx_lock(&(sc)->ic_lock); \ } while (0) #define TSEC_IC_UNLOCK(sc) mtx_unlock(&(sc)->ic_lock) #define TSEC_IC_LOCK_ASSERT(sc) mtx_assert(&(sc)->ic_lock, MA_OWNED) /* Global tsec lock (with all locks) */ #define TSEC_GLOBAL_LOCK(sc) do { \ if ((mtx_owned(&(sc)->transmit_lock) ? 1 : 0) != \ (mtx_owned(&(sc)->receive_lock) ? 1 : 0)) { \ panic("tsec deadlock possibility detection!"); \ } \ mtx_lock(&(sc)->transmit_lock); \ mtx_lock(&(sc)->receive_lock); \ } while (0) #define TSEC_GLOBAL_UNLOCK(sc) do { \ TSEC_RECEIVE_UNLOCK(sc); \ TSEC_TRANSMIT_UNLOCK(sc); \ } while (0) #define TSEC_GLOBAL_LOCK_ASSERT(sc) do { \ TSEC_TRANSMIT_LOCK_ASSERT(sc); \ TSEC_RECEIVE_LOCK_ASSERT(sc); \ } while (0) /* From global to {transmit,receive} */ #define TSEC_GLOBAL_TO_TRANSMIT_LOCK(sc) do { \ mtx_unlock(&(sc)->receive_lock);\ } while (0) #define TSEC_GLOBAL_TO_RECEIVE_LOCK(sc) do { \ mtx_unlock(&(sc)->transmit_lock);\ } while (0) struct tsec_desc { volatile uint16_t flags; /* descriptor flags */ volatile uint16_t length; /* buffer length */ volatile uint32_t bufptr; /* buffer pointer */ }; #define TSEC_READ_RETRY 10000 #define TSEC_READ_DELAY 100 /* Structures and defines for TCP/IP Off-load */ struct tsec_tx_fcb { volatile uint16_t flags; volatile uint8_t l4_offset; volatile uint8_t l3_offset; volatile uint16_t ph_chsum; volatile uint16_t vlan; }; struct tsec_rx_fcb { volatile uint16_t flags; volatile uint8_t rq_index; volatile uint8_t protocol; volatile uint16_t unused; volatile uint16_t vlan; }; #define TSEC_CHECKSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP) #define TSEC_TX_FCB_IP4 TSEC_TX_FCB_L3_IS_IP #define TSEC_TX_FCB_IP6 (TSEC_TX_FCB_L3_IS_IP | TSEC_TX_FCB_L3_IS_IP6) #define TSEC_TX_FCB_TCP TSEC_TX_FCB_L4_IS_TCP_UDP #define TSEC_TX_FCB_UDP (TSEC_TX_FCB_L4_IS_TCP_UDP | TSEC_TX_FCB_L4_IS_UDP) #define TSEC_RX_FCB_IP_CSUM_CHECKED(flags) \ ((flags & (TSEC_RX_FCB_IP_FOUND | TSEC_RX_FCB_IP6_FOUND | \ TSEC_RX_FCB_IP_CSUM | TSEC_RX_FCB_PARSE_ERROR)) \ == (TSEC_RX_FCB_IP_FOUND | TSEC_RX_FCB_IP_CSUM)) #define TSEC_RX_FCB_TCP_UDP_CSUM_CHECKED(flags) \ ((flags & (TSEC_RX_FCB_TCP_UDP_FOUND | TSEC_RX_FCB_TCP_UDP_CSUM \ | TSEC_RX_FCB_PARSE_ERROR)) \ == (TSEC_RX_FCB_TCP_UDP_FOUND | TSEC_RX_FCB_TCP_UDP_CSUM)) /* Prototypes */ extern devclass_t tsec_devclass; int tsec_attach(struct tsec_softc *sc); int tsec_detach(struct tsec_softc *sc); void tsec_error_intr(void *arg); void tsec_receive_intr(void *arg); void tsec_transmit_intr(void *arg); int tsec_miibus_readreg(device_t dev, int phy, int reg); int tsec_miibus_writereg(device_t dev, int phy, int reg, int value); void tsec_miibus_statchg(device_t dev); int tsec_resume(device_t dev); /* XXX */ int tsec_shutdown(device_t dev); int tsec_suspend(device_t dev); /* XXX */ void tsec_get_hwaddr(struct tsec_softc *sc, uint8_t *addr); #endif /* _IF_TSEC_H */ Index: projects/clang390-import/sys/dev/tsec/if_tsec_fdt.c =================================================================== --- projects/clang390-import/sys/dev/tsec/if_tsec_fdt.c (revision 305080) +++ projects/clang390-import/sys/dev/tsec/if_tsec_fdt.c (revision 305081) @@ -1,354 +1,388 @@ /*- * Copyright (C) 2007-2008 Semihalf, Rafal Jaworowski * Copyright (C) 2006-2007 Semihalf, Piotr Kruszynski * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * From: FreeBSD: head/sys/dev/tsec/if_tsec_ocp.c 188712 2009-02-17 14:59:47Z raj */ /* * FDT 'simple-bus' attachment for Freescale TSEC controller. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "miibus_if.h" #define TSEC_RID_TXIRQ 0 #define TSEC_RID_RXIRQ 1 #define TSEC_RID_ERRIRQ 2 static int tsec_fdt_probe(device_t dev); static int tsec_fdt_attach(device_t dev); static int tsec_fdt_detach(device_t dev); static int tsec_setup_intr(struct tsec_softc *sc, struct resource **ires, void **ihand, int *irid, driver_intr_t handler, const char *iname); static void tsec_release_intr(struct tsec_softc *sc, struct resource *ires, void *ihand, int irid, const char *iname); static device_method_t tsec_methods[] = { /* Device interface */ DEVMETHOD(device_probe, tsec_fdt_probe), DEVMETHOD(device_attach, tsec_fdt_attach), DEVMETHOD(device_detach, tsec_fdt_detach), DEVMETHOD(device_shutdown, tsec_shutdown), DEVMETHOD(device_suspend, tsec_suspend), DEVMETHOD(device_resume, tsec_resume), /* MII interface */ DEVMETHOD(miibus_readreg, tsec_miibus_readreg), DEVMETHOD(miibus_writereg, tsec_miibus_writereg), DEVMETHOD(miibus_statchg, tsec_miibus_statchg), DEVMETHOD_END }; static driver_t tsec_fdt_driver = { "tsec", tsec_methods, sizeof(struct tsec_softc), }; DRIVER_MODULE(tsec, simplebus, tsec_fdt_driver, tsec_devclass, 0, 0); static int tsec_fdt_probe(device_t dev) { struct tsec_softc *sc; uint32_t id; if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_get_type(dev) == NULL || strcmp(ofw_bus_get_type(dev), "network") != 0) return (ENXIO); if (!ofw_bus_is_compatible(dev, "gianfar") && !ofw_bus_is_compatible(dev, "fsl,etsec2")) return (ENXIO); sc = device_get_softc(dev); - sc->sc_rrid = 0; - sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, - RF_ACTIVE); - if (sc->sc_rres == NULL) - return (ENXIO); + /* + * Device trees with "fsl,etsec2" compatible nodes don't have a reg + * property, as it's been relegated to the queue-group children. + */ + if (ofw_bus_is_compatible(dev, "fsl,etsec2")) + sc->is_etsec = 1; + else { + sc->sc_rrid = 0; + sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, + RF_ACTIVE); + if (sc->sc_rres == NULL) + return (ENXIO); - sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); - sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); + sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); + sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); - /* Check if we are eTSEC (enhanced TSEC) */ - id = TSEC_READ(sc, TSEC_REG_ID); - sc->is_etsec = ((id >> 16) == TSEC_ETSEC_ID) ? 1 : 0; - id |= TSEC_READ(sc, TSEC_REG_ID2); + /* Check if we are eTSEC (enhanced TSEC) */ + id = TSEC_READ(sc, TSEC_REG_ID); + sc->is_etsec = ((id >> 16) == TSEC_ETSEC_ID) ? 1 : 0; + id |= TSEC_READ(sc, TSEC_REG_ID2); - bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); + bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); - if (id == 0) { - device_printf(dev, "could not identify TSEC type\n"); - return (ENXIO); + if (id == 0) { + device_printf(dev, "could not identify TSEC type\n"); + return (ENXIO); + } } if (sc->is_etsec) device_set_desc(dev, "Enhanced Three-Speed Ethernet Controller"); else device_set_desc(dev, "Three-Speed Ethernet Controller"); return (BUS_PROBE_DEFAULT); } static int tsec_fdt_attach(device_t dev) { struct tsec_softc *sc; - phandle_t phy; + struct resource_list *rl; + phandle_t child, mdio, phy; + int acells, scells; int error = 0; sc = device_get_softc(dev); sc->dev = dev; sc->node = ofw_bus_get_node(dev); + if (fdt_addrsize_cells(sc->node, &acells, &scells) != 0) { + acells = 1; + scells = 1; + } + if (ofw_bus_is_compatible(dev, "fsl,etsec2")) { + rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev); + + /* + * TODO: Add all children resources to the list. Will be + * required to support multigroup mode. + */ + child = OF_child(sc->node); + ofw_bus_reg_to_rl(dev, child, acells, scells, rl); + ofw_bus_intr_to_rl(dev, child, rl, NULL); + } + /* Get phy address from fdt */ if (OF_getencprop(sc->node, "phy-handle", &phy, sizeof(phy)) <= 0) { device_printf(dev, "PHY not found in device tree"); return (ENXIO); } phy = OF_node_from_xref(phy); - OF_decode_addr(OF_parent(phy), 0, &sc->phy_bst, &sc->phy_bsh, NULL); + mdio = OF_parent(phy); + OF_decode_addr(mdio, 0, &sc->phy_bst, &sc->phy_bsh, NULL); OF_getencprop(phy, "reg", &sc->phyaddr, sizeof(sc->phyaddr)); + + /* + * etsec2 MDIO nodes are given the MDIO module base address, so we need + * to add the MII offset to get the PHY registers. + */ + if (ofw_bus_node_is_compatible(mdio, "fsl,etsec2-mdio")) + sc->phy_regoff = TSEC_REG_MIIBASE; /* Init timer */ callout_init(&sc->tsec_callout, 1); /* Init locks */ mtx_init(&sc->transmit_lock, device_get_nameunit(dev), "TSEC TX lock", MTX_DEF); mtx_init(&sc->receive_lock, device_get_nameunit(dev), "TSEC RX lock", MTX_DEF); mtx_init(&sc->ic_lock, device_get_nameunit(dev), "TSEC IC lock", MTX_DEF); /* Allocate IO memory for TSEC registers */ sc->sc_rrid = 0; sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, RF_ACTIVE); if (sc->sc_rres == NULL) { device_printf(dev, "could not allocate IO memory range!\n"); goto fail1; } sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); /* TSEC attach */ if (tsec_attach(sc) != 0) { device_printf(dev, "could not be configured\n"); goto fail2; } /* Set up interrupts (TX/RX/ERR) */ sc->sc_transmit_irid = TSEC_RID_TXIRQ; error = tsec_setup_intr(sc, &sc->sc_transmit_ires, &sc->sc_transmit_ihand, &sc->sc_transmit_irid, tsec_transmit_intr, "TX"); if (error) goto fail2; sc->sc_receive_irid = TSEC_RID_RXIRQ; error = tsec_setup_intr(sc, &sc->sc_receive_ires, &sc->sc_receive_ihand, &sc->sc_receive_irid, tsec_receive_intr, "RX"); if (error) goto fail3; sc->sc_error_irid = TSEC_RID_ERRIRQ; error = tsec_setup_intr(sc, &sc->sc_error_ires, &sc->sc_error_ihand, &sc->sc_error_irid, tsec_error_intr, "ERR"); if (error) goto fail4; return (0); fail4: tsec_release_intr(sc, sc->sc_receive_ires, sc->sc_receive_ihand, sc->sc_receive_irid, "RX"); fail3: tsec_release_intr(sc, sc->sc_transmit_ires, sc->sc_transmit_ihand, sc->sc_transmit_irid, "TX"); fail2: bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); fail1: mtx_destroy(&sc->receive_lock); mtx_destroy(&sc->transmit_lock); return (ENXIO); } static int tsec_setup_intr(struct tsec_softc *sc, struct resource **ires, void **ihand, int *irid, driver_intr_t handler, const char *iname) { int error; *ires = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, irid, RF_ACTIVE); if (*ires == NULL) { device_printf(sc->dev, "could not allocate %s IRQ\n", iname); return (ENXIO); } error = bus_setup_intr(sc->dev, *ires, INTR_TYPE_NET | INTR_MPSAFE, NULL, handler, sc, ihand); if (error) { device_printf(sc->dev, "failed to set up %s IRQ\n", iname); if (bus_release_resource(sc->dev, SYS_RES_IRQ, *irid, *ires)) device_printf(sc->dev, "could not release %s IRQ\n", iname); *ires = NULL; return (error); } return (0); } static void tsec_release_intr(struct tsec_softc *sc, struct resource *ires, void *ihand, int irid, const char *iname) { int error; if (ires == NULL) return; error = bus_teardown_intr(sc->dev, ires, ihand); if (error) device_printf(sc->dev, "bus_teardown_intr() failed for %s intr" ", error %d\n", iname, error); error = bus_release_resource(sc->dev, SYS_RES_IRQ, irid, ires); if (error) device_printf(sc->dev, "bus_release_resource() failed for %s " "intr, error %d\n", iname, error); } static int tsec_fdt_detach(device_t dev) { struct tsec_softc *sc; int error; sc = device_get_softc(dev); /* Wait for stopping watchdog */ callout_drain(&sc->tsec_callout); /* Stop and release all interrupts */ tsec_release_intr(sc, sc->sc_transmit_ires, sc->sc_transmit_ihand, sc->sc_transmit_irid, "TX"); tsec_release_intr(sc, sc->sc_receive_ires, sc->sc_receive_ihand, sc->sc_receive_irid, "RX"); tsec_release_intr(sc, sc->sc_error_ires, sc->sc_error_ihand, sc->sc_error_irid, "ERR"); /* TSEC detach */ tsec_detach(sc); /* Free IO memory handler */ if (sc->sc_rres) { error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); if (error) device_printf(dev, "bus_release_resource() failed for" " IO memory, error %d\n", error); } /* Destroy locks */ mtx_destroy(&sc->receive_lock); mtx_destroy(&sc->transmit_lock); mtx_destroy(&sc->ic_lock); return (0); } void tsec_get_hwaddr(struct tsec_softc *sc, uint8_t *addr) { union { uint32_t reg[2]; uint8_t addr[6]; } hw; int i; hw.reg[0] = hw.reg[1] = 0; /* Retrieve the hardware address from the device tree. */ i = OF_getprop(sc->node, "local-mac-address", (void *)hw.addr, 6); if (i == 6 && (hw.reg[0] != 0 || hw.reg[1] != 0)) { bcopy(hw.addr, addr, 6); return; } /* Also try the mac-address property, which is second-best */ i = OF_getprop(sc->node, "mac-address", (void *)hw.addr, 6); if (i == 6 && (hw.reg[0] != 0 || hw.reg[1] != 0)) { bcopy(hw.addr, addr, 6); return; } /* * Fall back -- use the currently programmed address in the hope that * it was set be firmware... */ hw.reg[0] = TSEC_READ(sc, TSEC_REG_MACSTNADDR1); hw.reg[1] = TSEC_READ(sc, TSEC_REG_MACSTNADDR2); for (i = 0; i < 6; i++) addr[5-i] = hw.addr[i]; } Index: projects/clang390-import/sys/dev/usb/net/if_urndis.c =================================================================== --- projects/clang390-import/sys/dev/usb/net/if_urndis.c (revision 305080) +++ projects/clang390-import/sys/dev/usb/net/if_urndis.c (revision 305081) @@ -1,1055 +1,1055 @@ /* $OpenBSD: if_urndis.c,v 1.46 2013/12/09 15:45:29 pirofti Exp $ */ /* * Copyright (c) 2010 Jonathan Armani * Copyright (c) 2010 Fabien Romano * Copyright (c) 2010 Michael Knudsen * Copyright (c) 2014 Hans Petter Selasky * All rights reserved. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "usbdevs.h" #define USB_DEBUG_VAR urndis_debug #include #include #include "usb_if.h" #include #include #include static device_probe_t urndis_probe; static device_attach_t urndis_attach; static device_detach_t urndis_detach; static device_suspend_t urndis_suspend; static device_resume_t urndis_resume; static usb_callback_t urndis_bulk_write_callback; static usb_callback_t urndis_bulk_read_callback; static usb_callback_t urndis_intr_read_callback; static uether_fn_t urndis_attach_post; static uether_fn_t urndis_init; static uether_fn_t urndis_stop; static uether_fn_t urndis_start; static uether_fn_t urndis_setmulti; static uether_fn_t urndis_setpromisc; static uint32_t urndis_ctrl_query(struct urndis_softc *sc, uint32_t oid, struct rndis_query_req *msg, uint16_t len, const void **rbuf, uint16_t *rbufsz); static uint32_t urndis_ctrl_set(struct urndis_softc *sc, uint32_t oid, struct rndis_set_req *msg, uint16_t len); static uint32_t urndis_ctrl_handle_init(struct urndis_softc *sc, const struct rndis_comp_hdr *hdr); static uint32_t urndis_ctrl_handle_query(struct urndis_softc *sc, const struct rndis_comp_hdr *hdr, const void **buf, uint16_t *bufsz); static uint32_t urndis_ctrl_handle_reset(struct urndis_softc *sc, const struct rndis_comp_hdr *hdr); static uint32_t urndis_ctrl_init(struct urndis_softc *sc); static uint32_t urndis_ctrl_halt(struct urndis_softc *sc); #ifdef USB_DEBUG static int urndis_debug = 0; static SYSCTL_NODE(_hw_usb, OID_AUTO, urndis, CTLFLAG_RW, 0, "USB RNDIS-Ethernet"); SYSCTL_INT(_hw_usb_urndis, OID_AUTO, debug, CTLFLAG_RWTUN, &urndis_debug, 0, "Debug level"); #endif static const struct usb_config urndis_config[URNDIS_N_TRANSFER] = { [URNDIS_BULK_RX] = { .type = UE_BULK, .endpoint = UE_ADDR_ANY, .direction = UE_DIR_RX, .if_index = 0, .frames = 1, .bufsize = RNDIS_RX_MAXLEN, .flags = {.short_xfer_ok = 1,}, .callback = urndis_bulk_read_callback, .timeout = 0, /* no timeout */ .usb_mode = USB_MODE_HOST, }, [URNDIS_BULK_TX] = { .type = UE_BULK, .endpoint = UE_ADDR_ANY, .direction = UE_DIR_TX, .if_index = 0, .frames = RNDIS_TX_FRAMES_MAX, .bufsize = (RNDIS_TX_FRAMES_MAX * RNDIS_TX_MAXLEN), .flags = { .force_short_xfer = 1, }, .callback = urndis_bulk_write_callback, .timeout = 10000, /* 10 seconds */ .usb_mode = USB_MODE_HOST, }, [URNDIS_INTR_RX] = { .type = UE_INTERRUPT, .endpoint = UE_ADDR_ANY, .direction = UE_DIR_RX, .if_index = 1, .bufsize = 0, /* use wMaxPacketSize */ .flags = {.short_xfer_ok = 1,.no_pipe_ok = 1,}, .callback = urndis_intr_read_callback, .timeout = 0, .usb_mode = USB_MODE_HOST, }, }; static device_method_t urndis_methods[] = { /* Device interface */ DEVMETHOD(device_probe, urndis_probe), DEVMETHOD(device_attach, urndis_attach), DEVMETHOD(device_detach, urndis_detach), DEVMETHOD(device_suspend, urndis_suspend), DEVMETHOD(device_resume, urndis_resume), DEVMETHOD_END }; static driver_t urndis_driver = { .name = "urndis", .methods = urndis_methods, .size = sizeof(struct urndis_softc), }; static devclass_t urndis_devclass; static const STRUCT_USB_HOST_ID urndis_host_devs[] = { /* Generic RNDIS class match */ {USB_IFACE_CLASS(UICLASS_CDC), USB_IFACE_SUBCLASS(UISUBCLASS_ABSTRACT_CONTROL_MODEL), USB_IFACE_PROTOCOL(0xff)}, {USB_IFACE_CLASS(UICLASS_WIRELESS), USB_IFACE_SUBCLASS(UISUBCLASS_RF), USB_IFACE_PROTOCOL(UIPROTO_RNDIS)}, {USB_IFACE_CLASS(UICLASS_IAD), USB_IFACE_SUBCLASS(UISUBCLASS_SYNC), USB_IFACE_PROTOCOL(UIPROTO_ACTIVESYNC)}, /* HP-WebOS */ {USB_VENDOR(USB_VENDOR_PALM), USB_IFACE_CLASS(UICLASS_CDC), USB_IFACE_SUBCLASS(UISUBCLASS_ABSTRACT_CONTROL_MODEL), USB_IFACE_PROTOCOL(0xff)}, }; DRIVER_MODULE(urndis, uhub, urndis_driver, urndis_devclass, NULL, NULL); MODULE_VERSION(urndis, 1); MODULE_DEPEND(urndis, uether, 1, 1, 1); MODULE_DEPEND(urndis, usb, 1, 1, 1); MODULE_DEPEND(urndis, ether, 1, 1, 1); USB_PNP_HOST_INFO(urndis_host_devs); static const struct usb_ether_methods urndis_ue_methods = { .ue_attach_post = urndis_attach_post, .ue_start = urndis_start, .ue_init = urndis_init, .ue_stop = urndis_stop, .ue_setmulti = urndis_setmulti, .ue_setpromisc = urndis_setpromisc, }; static int urndis_probe(device_t dev) { struct usb_attach_arg *uaa = device_get_ivars(dev); return (usbd_lookup_id_by_uaa(urndis_host_devs, sizeof(urndis_host_devs), uaa)); } static void urndis_attach_post(struct usb_ether *ue) { /* no-op */ } static int urndis_attach(device_t dev) { static struct { union { struct rndis_query_req query; struct rndis_set_req set; } hdr; union { uint8_t eaddr[ETHER_ADDR_LEN]; uint32_t filter; } ibuf; } msg; struct urndis_softc *sc = device_get_softc(dev); struct usb_ether *ue = &sc->sc_ue; struct usb_attach_arg *uaa = device_get_ivars(dev); struct usb_cdc_cm_descriptor *cmd; const void *buf; uint16_t bufsz; uint8_t iface_index[2] = { uaa->info.bIfaceIndex + 1, uaa->info.bIfaceIndex }; int error; uint8_t i; sc->sc_ue.ue_udev = uaa->device; sc->sc_ifaceno_ctl = uaa->info.bIfaceNum; cmd = usbd_find_descriptor(uaa->device, NULL, uaa->info.bIfaceIndex, UDESC_CS_INTERFACE, 0xFF, UDESCSUB_CDC_CM, 0xFF); if (cmd != NULL) { DPRINTF("Call Mode Descriptor found, dataif=%d\n", cmd->bDataInterface); iface_index[0] = cmd->bDataInterface; } device_set_usb_desc(dev); mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF); /* scan the alternate settings looking for a valid one */ for (i = 0; i != 32; i++) { error = usbd_set_alt_interface_index(uaa->device, iface_index[0], i); if (error != 0) break; error = usbd_transfer_setup(uaa->device, iface_index, sc->sc_xfer, urndis_config, URNDIS_N_TRANSFER, sc, &sc->sc_mtx); if (error == 0) break; } if ((error != 0) || (i == 32)) { device_printf(dev, "No valid alternate setting found\n"); goto detach; } /* Initialize device - must be done before even querying it */ URNDIS_LOCK(sc); error = urndis_ctrl_init(sc); URNDIS_UNLOCK(sc); if (error != (int)RNDIS_STATUS_SUCCESS) { device_printf(dev, "Unable to initialize hardware\n"); goto detach; } /* Determine MAC address */ memset(msg.ibuf.eaddr, 0, sizeof(msg.ibuf.eaddr)); URNDIS_LOCK(sc); error = urndis_ctrl_query(sc, OID_802_3_PERMANENT_ADDRESS, &msg.hdr.query, sizeof(msg.hdr.query) + sizeof(msg.ibuf.eaddr), &buf, &bufsz); URNDIS_UNLOCK(sc); if (error != (int)RNDIS_STATUS_SUCCESS) { device_printf(dev, "Unable to get hardware address\n"); goto detach; } if (bufsz != ETHER_ADDR_LEN) { device_printf(dev, "Invalid address length: %d bytes\n", bufsz); goto detach; } memcpy(&sc->sc_ue.ue_eaddr, buf, ETHER_ADDR_LEN); /* Initialize packet filter */ - sc->sc_filter = RNDIS_PACKET_TYPE_BROADCAST | - RNDIS_PACKET_TYPE_ALL_MULTICAST; + sc->sc_filter = NDIS_PACKET_TYPE_BROADCAST | + NDIS_PACKET_TYPE_ALL_MULTICAST; msg.ibuf.filter = htole32(sc->sc_filter); URNDIS_LOCK(sc); error = urndis_ctrl_set(sc, OID_GEN_CURRENT_PACKET_FILTER, &msg.hdr.set, sizeof(msg.hdr.set) + sizeof(msg.ibuf.filter)); URNDIS_UNLOCK(sc); if (error != (int)RNDIS_STATUS_SUCCESS) { device_printf(dev, "Unable to set data filters\n"); goto detach; } ue->ue_sc = sc; ue->ue_dev = dev; ue->ue_udev = uaa->device; ue->ue_mtx = &sc->sc_mtx; ue->ue_methods = &urndis_ue_methods; error = uether_ifattach(ue); if (error) { device_printf(dev, "Could not attach interface\n"); goto detach; } URNDIS_LOCK(sc); /* start interrupt endpoint, if any */ usbd_transfer_start(sc->sc_xfer[URNDIS_INTR_RX]); URNDIS_UNLOCK(sc); return (0); /* success */ detach: (void)urndis_detach(dev); return (ENXIO); /* failure */ } static int urndis_detach(device_t dev) { struct urndis_softc *sc = device_get_softc(dev); struct usb_ether *ue = &sc->sc_ue; /* stop all USB transfers first */ usbd_transfer_unsetup(sc->sc_xfer, URNDIS_N_TRANSFER); uether_ifdetach(ue); URNDIS_LOCK(sc); (void)urndis_ctrl_halt(sc); URNDIS_UNLOCK(sc); mtx_destroy(&sc->sc_mtx); return (0); } static void urndis_start(struct usb_ether *ue) { struct urndis_softc *sc = uether_getsc(ue); /* * Start the USB transfers, if not already started: */ usbd_transfer_start(sc->sc_xfer[URNDIS_BULK_TX]); usbd_transfer_start(sc->sc_xfer[URNDIS_BULK_RX]); } static void urndis_init(struct usb_ether *ue) { struct urndis_softc *sc = uether_getsc(ue); struct ifnet *ifp = uether_getifp(ue); URNDIS_LOCK_ASSERT(sc, MA_OWNED); ifp->if_drv_flags |= IFF_DRV_RUNNING; /* stall data write direction, which depends on USB mode */ usbd_xfer_set_stall(sc->sc_xfer[URNDIS_BULK_TX]); /* start data transfers */ urndis_start(ue); } static void urndis_stop(struct usb_ether *ue) { struct urndis_softc *sc = uether_getsc(ue); struct ifnet *ifp = uether_getifp(ue); URNDIS_LOCK_ASSERT(sc, MA_OWNED); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; /* * stop all the transfers, if not already stopped: */ usbd_transfer_stop(sc->sc_xfer[URNDIS_BULK_RX]); usbd_transfer_stop(sc->sc_xfer[URNDIS_BULK_TX]); } static void urndis_setmulti(struct usb_ether *ue) { /* no-op */ } static void urndis_setpromisc(struct usb_ether *ue) { /* no-op */ } static int urndis_suspend(device_t dev) { device_printf(dev, "Suspending\n"); return (0); } static int urndis_resume(device_t dev) { device_printf(dev, "Resuming\n"); return (0); } static usb_error_t urndis_ctrl_msg(struct urndis_softc *sc, uint8_t rt, uint8_t r, uint16_t index, uint16_t value, void *buf, uint16_t buflen) { usb_device_request_t req; req.bmRequestType = rt; req.bRequest = r; USETW(req.wValue, value); USETW(req.wIndex, index); USETW(req.wLength, buflen); return (usbd_do_request_flags(sc->sc_ue.ue_udev, &sc->sc_mtx, &req, buf, (rt & UT_READ) ? USB_SHORT_XFER_OK : 0, NULL, 2000 /* ms */ )); } static usb_error_t urndis_ctrl_send(struct urndis_softc *sc, void *buf, uint16_t len) { usb_error_t err; err = urndis_ctrl_msg(sc, UT_WRITE_CLASS_INTERFACE, UCDC_SEND_ENCAPSULATED_COMMAND, sc->sc_ifaceno_ctl, 0, buf, len); DPRINTF("%s\n", usbd_errstr(err)); return (err); } static struct rndis_comp_hdr * urndis_ctrl_recv(struct urndis_softc *sc) { struct rndis_comp_hdr *hdr; usb_error_t err; err = urndis_ctrl_msg(sc, UT_READ_CLASS_INTERFACE, UCDC_GET_ENCAPSULATED_RESPONSE, sc->sc_ifaceno_ctl, 0, sc->sc_response_buf, RNDIS_RESPONSE_LEN); if (err != USB_ERR_NORMAL_COMPLETION) return (NULL); hdr = (struct rndis_comp_hdr *)sc->sc_response_buf; DPRINTF("type 0x%x len %u\n", le32toh(hdr->rm_type), le32toh(hdr->rm_len)); if (le32toh(hdr->rm_len) > RNDIS_RESPONSE_LEN) { DPRINTF("ctrl message error: wrong size %u > %u\n", le32toh(hdr->rm_len), RNDIS_RESPONSE_LEN); return (NULL); } return (hdr); } static uint32_t urndis_ctrl_handle(struct urndis_softc *sc, struct rndis_comp_hdr *hdr, const void **buf, uint16_t *bufsz) { uint32_t rval; DPRINTF("\n"); if (buf != NULL && bufsz != NULL) { *buf = NULL; *bufsz = 0; } switch (le32toh(hdr->rm_type)) { case REMOTE_NDIS_INITIALIZE_CMPLT: rval = urndis_ctrl_handle_init(sc, hdr); break; case REMOTE_NDIS_QUERY_CMPLT: rval = urndis_ctrl_handle_query(sc, hdr, buf, bufsz); break; case REMOTE_NDIS_RESET_CMPLT: rval = urndis_ctrl_handle_reset(sc, hdr); break; case REMOTE_NDIS_KEEPALIVE_CMPLT: case REMOTE_NDIS_SET_CMPLT: rval = le32toh(hdr->rm_status); break; default: device_printf(sc->sc_ue.ue_dev, "ctrl message error: unknown event 0x%x\n", le32toh(hdr->rm_type)); rval = RNDIS_STATUS_FAILURE; break; } return (rval); } static uint32_t urndis_ctrl_handle_init(struct urndis_softc *sc, const struct rndis_comp_hdr *hdr) { const struct rndis_init_comp *msg; msg = (const struct rndis_init_comp *)hdr; DPRINTF("len %u rid %u status 0x%x " "ver_major %u ver_minor %u devflags 0x%x medium 0x%x pktmaxcnt %u " "pktmaxsz %u align %u aflistoffset %u aflistsz %u\n", le32toh(msg->rm_len), le32toh(msg->rm_rid), le32toh(msg->rm_status), le32toh(msg->rm_ver_major), le32toh(msg->rm_ver_minor), le32toh(msg->rm_devflags), le32toh(msg->rm_medium), le32toh(msg->rm_pktmaxcnt), le32toh(msg->rm_pktmaxsz), le32toh(msg->rm_align), le32toh(msg->rm_aflistoffset), le32toh(msg->rm_aflistsz)); if (le32toh(msg->rm_status) != RNDIS_STATUS_SUCCESS) { DPRINTF("init failed 0x%x\n", le32toh(msg->rm_status)); return (le32toh(msg->rm_status)); } if (le32toh(msg->rm_devflags) != RNDIS_DF_CONNECTIONLESS) { DPRINTF("wrong device type (current type: 0x%x)\n", le32toh(msg->rm_devflags)); return (RNDIS_STATUS_FAILURE); } if (le32toh(msg->rm_medium) != RNDIS_MEDIUM_802_3) { DPRINTF("medium not 802.3 (current medium: 0x%x)\n", le32toh(msg->rm_medium)); return (RNDIS_STATUS_FAILURE); } sc->sc_lim_pktsz = le32toh(msg->rm_pktmaxsz); return (le32toh(msg->rm_status)); } static uint32_t urndis_ctrl_handle_query(struct urndis_softc *sc, const struct rndis_comp_hdr *hdr, const void **buf, uint16_t *bufsz) { const struct rndis_query_comp *msg; uint64_t limit; msg = (const struct rndis_query_comp *)hdr; DPRINTF("len %u rid %u status 0x%x " "buflen %u bufoff %u\n", le32toh(msg->rm_len), le32toh(msg->rm_rid), le32toh(msg->rm_status), le32toh(msg->rm_infobuflen), le32toh(msg->rm_infobufoffset)); *buf = NULL; *bufsz = 0; if (le32toh(msg->rm_status) != RNDIS_STATUS_SUCCESS) { DPRINTF("query failed 0x%x\n", le32toh(msg->rm_status)); return (le32toh(msg->rm_status)); } limit = le32toh(msg->rm_infobuflen); limit += le32toh(msg->rm_infobufoffset); limit += RNDIS_HEADER_OFFSET; if (limit > (uint64_t)le32toh(msg->rm_len)) { DPRINTF("ctrl message error: invalid query info " "len/offset/end_position(%u/%u/%u) -> " "go out of buffer limit %u\n", le32toh(msg->rm_infobuflen), le32toh(msg->rm_infobufoffset), le32toh(msg->rm_infobuflen) + le32toh(msg->rm_infobufoffset) + RNDIS_HEADER_OFFSET, le32toh(msg->rm_len)); return (RNDIS_STATUS_FAILURE); } *buf = ((const uint8_t *)msg) + RNDIS_HEADER_OFFSET + le32toh(msg->rm_infobufoffset); *bufsz = le32toh(msg->rm_infobuflen); return (le32toh(msg->rm_status)); } static uint32_t urndis_ctrl_handle_reset(struct urndis_softc *sc, const struct rndis_comp_hdr *hdr) { const struct rndis_reset_comp *msg; uint32_t rval; msg = (const struct rndis_reset_comp *)hdr; rval = le32toh(msg->rm_status); DPRINTF("len %u status 0x%x " "adrreset %u\n", le32toh(msg->rm_len), rval, le32toh(msg->rm_adrreset)); if (rval != RNDIS_STATUS_SUCCESS) { DPRINTF("reset failed 0x%x\n", rval); return (rval); } if (msg->rm_adrreset != 0) { struct { struct rndis_set_req hdr; uint32_t filter; } msg_filter; msg_filter.filter = htole32(sc->sc_filter); rval = urndis_ctrl_set(sc, OID_GEN_CURRENT_PACKET_FILTER, &msg_filter.hdr, sizeof(msg_filter)); if (rval != RNDIS_STATUS_SUCCESS) { DPRINTF("unable to reset data filters\n"); return (rval); } } return (rval); } static uint32_t urndis_ctrl_init(struct urndis_softc *sc) { struct rndis_init_req msg; struct rndis_comp_hdr *hdr; uint32_t rval; msg.rm_type = htole32(REMOTE_NDIS_INITIALIZE_MSG); msg.rm_len = htole32(sizeof(msg)); msg.rm_rid = 0; msg.rm_ver_major = htole32(RNDIS_VERSION_MAJOR); msg.rm_ver_minor = htole32(1); msg.rm_max_xfersz = htole32(RNDIS_RX_MAXLEN); DPRINTF("type %u len %u rid %u ver_major %u " "ver_minor %u max_xfersz %u\n", le32toh(msg.rm_type), le32toh(msg.rm_len), le32toh(msg.rm_rid), le32toh(msg.rm_ver_major), le32toh(msg.rm_ver_minor), le32toh(msg.rm_max_xfersz)); rval = urndis_ctrl_send(sc, &msg, sizeof(msg)); if (rval != RNDIS_STATUS_SUCCESS) { DPRINTF("init failed\n"); return (rval); } if ((hdr = urndis_ctrl_recv(sc)) == NULL) { DPRINTF("unable to get init response\n"); return (RNDIS_STATUS_FAILURE); } rval = urndis_ctrl_handle(sc, hdr, NULL, NULL); return (rval); } static uint32_t urndis_ctrl_halt(struct urndis_softc *sc) { struct rndis_halt_req msg; uint32_t rval; msg.rm_type = htole32(REMOTE_NDIS_HALT_MSG); msg.rm_len = htole32(sizeof(msg)); msg.rm_rid = 0; DPRINTF("type %u len %u rid %u\n", le32toh(msg.rm_type), le32toh(msg.rm_len), le32toh(msg.rm_rid)); rval = urndis_ctrl_send(sc, &msg, sizeof(msg)); if (rval != RNDIS_STATUS_SUCCESS) DPRINTF("halt failed\n"); return (rval); } /* * NB: Querying a device has the requirement of using an input buffer the size * of the expected reply or larger, except for variably sized replies. */ static uint32_t urndis_ctrl_query(struct urndis_softc *sc, uint32_t oid, struct rndis_query_req *msg, uint16_t len, const void **rbuf, uint16_t *rbufsz) { struct rndis_comp_hdr *hdr; uint32_t datalen, rval; msg->rm_type = htole32(REMOTE_NDIS_QUERY_MSG); msg->rm_len = htole32(len); msg->rm_rid = 0; /* XXX */ msg->rm_oid = htole32(oid); datalen = len - sizeof(*msg); msg->rm_infobuflen = htole32(datalen); if (datalen != 0) { msg->rm_infobufoffset = htole32(sizeof(*msg) - RNDIS_HEADER_OFFSET); } else { msg->rm_infobufoffset = 0; } msg->rm_devicevchdl = 0; DPRINTF("type %u len %u rid %u oid 0x%x " "infobuflen %u infobufoffset %u devicevchdl %u\n", le32toh(msg->rm_type), le32toh(msg->rm_len), le32toh(msg->rm_rid), le32toh(msg->rm_oid), le32toh(msg->rm_infobuflen), le32toh(msg->rm_infobufoffset), le32toh(msg->rm_devicevchdl)); rval = urndis_ctrl_send(sc, msg, len); if (rval != RNDIS_STATUS_SUCCESS) { DPRINTF("query failed\n"); return (rval); } if ((hdr = urndis_ctrl_recv(sc)) == NULL) { DPRINTF("unable to get query response\n"); return (RNDIS_STATUS_FAILURE); } rval = urndis_ctrl_handle(sc, hdr, rbuf, rbufsz); return (rval); } static uint32_t urndis_ctrl_set(struct urndis_softc *sc, uint32_t oid, struct rndis_set_req *msg, uint16_t len) { struct rndis_comp_hdr *hdr; uint32_t datalen, rval; msg->rm_type = htole32(REMOTE_NDIS_SET_MSG); msg->rm_len = htole32(len); msg->rm_rid = 0; /* XXX */ msg->rm_oid = htole32(oid); datalen = len - sizeof(*msg); msg->rm_infobuflen = htole32(datalen); if (datalen != 0) { msg->rm_infobufoffset = htole32(sizeof(*msg) - RNDIS_HEADER_OFFSET); } else { msg->rm_infobufoffset = 0; } msg->rm_devicevchdl = 0; DPRINTF("type %u len %u rid %u oid 0x%x " "infobuflen %u infobufoffset %u devicevchdl %u\n", le32toh(msg->rm_type), le32toh(msg->rm_len), le32toh(msg->rm_rid), le32toh(msg->rm_oid), le32toh(msg->rm_infobuflen), le32toh(msg->rm_infobufoffset), le32toh(msg->rm_devicevchdl)); rval = urndis_ctrl_send(sc, msg, len); if (rval != RNDIS_STATUS_SUCCESS) { DPRINTF("set failed\n"); return (rval); } if ((hdr = urndis_ctrl_recv(sc)) == NULL) { DPRINTF("unable to get set response\n"); return (RNDIS_STATUS_FAILURE); } rval = urndis_ctrl_handle(sc, hdr, NULL, NULL); if (rval != RNDIS_STATUS_SUCCESS) DPRINTF("set failed 0x%x\n", rval); return (rval); } static void urndis_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error) { struct urndis_softc *sc = usbd_xfer_softc(xfer); struct usb_page_cache *pc = usbd_xfer_get_frame(xfer, 0); struct ifnet *ifp = uether_getifp(&sc->sc_ue); struct rndis_packet_msg msg; struct mbuf *m; int actlen; int aframes; int offset; switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: usbd_xfer_status(xfer, &actlen, NULL, &aframes, NULL); DPRINTFN(1, "received %u bytes in %u frames\n", actlen, aframes); for (offset = 0; actlen >= (uint32_t)sizeof(msg);) { /* copy out header */ usbd_copy_out(pc, offset, &msg, sizeof(msg)); if (le32toh(0x1234567U) != 0x1234567U) { /* swap endianness */ msg.rm_type = le32toh(msg.rm_type); msg.rm_len = le32toh(msg.rm_len); msg.rm_dataoffset = le32toh(msg.rm_dataoffset); msg.rm_datalen = le32toh(msg.rm_datalen); msg.rm_oobdataoffset = le32toh(msg.rm_oobdataoffset); msg.rm_oobdatalen = le32toh(msg.rm_oobdatalen); msg.rm_oobdataelements = le32toh(msg.rm_oobdataelements); msg.rm_pktinfooffset = le32toh(msg.rm_pktinfooffset); msg.rm_pktinfolen = le32toh(msg.rm_pktinfolen); msg.rm_vchandle = le32toh(msg.rm_vchandle); msg.rm_reserved = le32toh(msg.rm_reserved); } DPRINTF("len %u data(off:%u len:%u) " "oobdata(off:%u len:%u nb:%u) perpacket(off:%u len:%u)\n", msg.rm_len, msg.rm_dataoffset, msg.rm_datalen, msg.rm_oobdataoffset, msg.rm_oobdatalen, msg.rm_oobdataelements, msg.rm_pktinfooffset, msg.rm_pktinfooffset); /* sanity check the RNDIS header */ if (msg.rm_type != REMOTE_NDIS_PACKET_MSG) { DPRINTF("invalid type 0x%x != 0x%x\n", msg.rm_type, REMOTE_NDIS_PACKET_MSG); goto tr_setup; } else if (msg.rm_len < (uint32_t)sizeof(msg)) { DPRINTF("invalid msg len %u < %u\n", msg.rm_len, (unsigned)sizeof(msg)); goto tr_setup; } else if (msg.rm_len > (uint32_t)actlen) { DPRINTF("invalid msg len %u > buffer " "len %u\n", msg.rm_len, actlen); goto tr_setup; } else if (msg.rm_dataoffset >= (uint32_t)actlen) { DPRINTF("invalid msg dataoffset %u > buffer " "dataoffset %u\n", msg.rm_dataoffset, actlen); goto tr_setup; } else if (msg.rm_datalen > (uint32_t)actlen) { DPRINTF("invalid msg datalen %u > buffer " "datalen %u\n", msg.rm_datalen, actlen); goto tr_setup; } else if ((msg.rm_dataoffset + msg.rm_datalen + (uint32_t)__offsetof(struct rndis_packet_msg, rm_dataoffset)) > (uint32_t)actlen) { DPRINTF("invalid dataoffset %u larger than %u\n", msg.rm_dataoffset + msg.rm_datalen + (uint32_t)__offsetof(struct rndis_packet_msg, rm_dataoffset), actlen); goto tr_setup; } else if (msg.rm_datalen < (uint32_t)sizeof(struct ether_header)) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); DPRINTF("invalid ethernet size " "%u < %u\n", msg.rm_datalen, (unsigned)sizeof(struct ether_header)); goto tr_setup; } else if (msg.rm_datalen > (uint32_t)(MCLBYTES - ETHER_ALIGN)) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); DPRINTF("invalid ethernet size " "%u > %u\n", msg.rm_datalen, (unsigned)MCLBYTES); goto tr_setup; } else if (msg.rm_datalen > (uint32_t)(MHLEN - ETHER_ALIGN)) { m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); } else { m = m_gethdr(M_NOWAIT, MT_DATA); } /* check if we have a buffer */ if (m != NULL) { m->m_len = m->m_pkthdr.len = msg.rm_datalen + ETHER_ALIGN; m_adj(m, ETHER_ALIGN); usbd_copy_out(pc, offset + msg.rm_dataoffset + __offsetof(struct rndis_packet_msg, rm_dataoffset), m->m_data, msg.rm_datalen); /* enqueue */ uether_rxmbuf(&sc->sc_ue, m, msg.rm_datalen); } else { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); } offset += msg.rm_len; actlen -= msg.rm_len; } case USB_ST_SETUP: tr_setup: usbd_xfer_set_frame_len(xfer, 0, RNDIS_RX_MAXLEN); usbd_xfer_set_frames(xfer, 1); usbd_transfer_submit(xfer); uether_rxflush(&sc->sc_ue); /* must be last */ break; default: /* Error */ DPRINTFN(1, "error = %s\n", usbd_errstr(error)); if (error != USB_ERR_CANCELLED) { /* try to clear stall first */ usbd_xfer_set_stall(xfer); usbd_xfer_set_frames(xfer, 0); usbd_transfer_submit(xfer); } break; } } static void urndis_bulk_write_callback(struct usb_xfer *xfer, usb_error_t error) { struct rndis_packet_msg msg; struct urndis_softc *sc = usbd_xfer_softc(xfer); struct ifnet *ifp = uether_getifp(&sc->sc_ue); struct mbuf *m; unsigned x; int actlen; int aframes; usbd_xfer_status(xfer, &actlen, NULL, &aframes, NULL); DPRINTFN(1, "\n"); switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: DPRINTFN(11, "%u bytes in %u frames\n", actlen, aframes); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); /* FALLTHROUGH */ case USB_ST_SETUP: tr_setup: memset(&msg, 0, sizeof(msg)); for (x = 0; x != RNDIS_TX_FRAMES_MAX; x++) { struct usb_page_cache *pc = usbd_xfer_get_frame(xfer, x); usbd_xfer_set_frame_offset(xfer, x * RNDIS_TX_MAXLEN, x); next_pkt: IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) break; if ((m->m_pkthdr.len + sizeof(msg)) > RNDIS_TX_MAXLEN) { DPRINTF("Too big packet\n"); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); /* Free buffer */ m_freem(m); goto next_pkt; } msg.rm_type = htole32(REMOTE_NDIS_PACKET_MSG); msg.rm_len = htole32(sizeof(msg) + m->m_pkthdr.len); msg.rm_dataoffset = htole32(RNDIS_DATA_OFFSET); msg.rm_datalen = htole32(m->m_pkthdr.len); /* copy in all data */ usbd_copy_in(pc, 0, &msg, sizeof(msg)); usbd_m_copy_in(pc, sizeof(msg), m, 0, m->m_pkthdr.len); usbd_xfer_set_frame_len(xfer, x, sizeof(msg) + m->m_pkthdr.len); /* * If there's a BPF listener, bounce a copy of * this frame to him: */ BPF_MTAP(ifp, m); /* Free buffer */ m_freem(m); } if (x != 0) { usbd_xfer_set_frames(xfer, x); usbd_transfer_submit(xfer); } break; default: /* Error */ DPRINTFN(11, "transfer error, %s\n", usbd_errstr(error)); /* count output errors */ if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); if (error != USB_ERR_CANCELLED) { /* try to clear stall first */ usbd_xfer_set_stall(xfer); goto tr_setup; } break; } } static void urndis_intr_read_callback(struct usb_xfer *xfer, usb_error_t error) { int actlen; usbd_xfer_status(xfer, &actlen, NULL, NULL, NULL); switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: DPRINTF("Received %d bytes\n", actlen); /* TODO: decode some indications */ /* FALLTHROUGH */ case USB_ST_SETUP: tr_setup: usbd_xfer_set_frame_len(xfer, 0, usbd_xfer_max_len(xfer)); usbd_transfer_submit(xfer); break; default: /* Error */ if (error != USB_ERR_CANCELLED) { /* start clear stall */ usbd_xfer_set_stall(xfer); goto tr_setup; } break; } } Index: projects/clang390-import/sys/kern/subr_bus.c =================================================================== --- projects/clang390-import/sys/kern/subr_bus.c (revision 305080) +++ projects/clang390-import/sys/kern/subr_bus.c (revision 305081) @@ -1,5559 +1,5579 @@ /*- * Copyright (c) 1997,1998,2003 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_bus.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include SYSCTL_NODE(_hw, OID_AUTO, bus, CTLFLAG_RW, NULL, NULL); SYSCTL_ROOT_NODE(OID_AUTO, dev, CTLFLAG_RW, NULL, NULL); /* * Used to attach drivers to devclasses. */ typedef struct driverlink *driverlink_t; struct driverlink { kobj_class_t driver; TAILQ_ENTRY(driverlink) link; /* list of drivers in devclass */ int pass; TAILQ_ENTRY(driverlink) passlink; }; /* * Forward declarations */ typedef TAILQ_HEAD(devclass_list, devclass) devclass_list_t; typedef TAILQ_HEAD(driver_list, driverlink) driver_list_t; typedef TAILQ_HEAD(device_list, device) device_list_t; struct devclass { TAILQ_ENTRY(devclass) link; devclass_t parent; /* parent in devclass hierarchy */ driver_list_t drivers; /* bus devclasses store drivers for bus */ char *name; device_t *devices; /* array of devices indexed by unit */ int maxunit; /* size of devices array */ int flags; #define DC_HAS_CHILDREN 1 struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *sysctl_tree; }; /** * @brief Implementation of device. */ struct device { /* * A device is a kernel object. The first field must be the * current ops table for the object. */ KOBJ_FIELDS; /* * Device hierarchy. */ TAILQ_ENTRY(device) link; /**< list of devices in parent */ TAILQ_ENTRY(device) devlink; /**< global device list membership */ device_t parent; /**< parent of this device */ device_list_t children; /**< list of child devices */ /* * Details of this device. */ driver_t *driver; /**< current driver */ devclass_t devclass; /**< current device class */ int unit; /**< current unit number */ char* nameunit; /**< name+unit e.g. foodev0 */ char* desc; /**< driver specific description */ int busy; /**< count of calls to device_busy() */ device_state_t state; /**< current device state */ uint32_t devflags; /**< api level flags for device_get_flags() */ u_int flags; /**< internal device flags */ u_int order; /**< order from device_add_child_ordered() */ void *ivars; /**< instance variables */ void *softc; /**< current driver's variables */ struct sysctl_ctx_list sysctl_ctx; /**< state for sysctl variables */ struct sysctl_oid *sysctl_tree; /**< state for sysctl variables */ }; static MALLOC_DEFINE(M_BUS, "bus", "Bus data structures"); static MALLOC_DEFINE(M_BUS_SC, "bus-sc", "Bus data structures, softc"); static void devctl2_init(void); #ifdef BUS_DEBUG static int bus_debug = 1; SYSCTL_INT(_debug, OID_AUTO, bus_debug, CTLFLAG_RWTUN, &bus_debug, 0, "Bus debug level"); #define PDEBUG(a) if (bus_debug) {printf("%s:%d: ", __func__, __LINE__), printf a; printf("\n");} #define DEVICENAME(d) ((d)? device_get_name(d): "no device") #define DRIVERNAME(d) ((d)? d->name : "no driver") #define DEVCLANAME(d) ((d)? d->name : "no devclass") /** * Produce the indenting, indent*2 spaces plus a '.' ahead of that to * prevent syslog from deleting initial spaces */ #define indentprintf(p) do { int iJ; printf("."); for (iJ=0; iJparent ? dc->parent->name : ""; break; default: return (EINVAL); } return (SYSCTL_OUT_STR(req, value)); } static void devclass_sysctl_init(devclass_t dc) { if (dc->sysctl_tree != NULL) return; sysctl_ctx_init(&dc->sysctl_ctx); dc->sysctl_tree = SYSCTL_ADD_NODE(&dc->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev), OID_AUTO, dc->name, CTLFLAG_RD, NULL, ""); SYSCTL_ADD_PROC(&dc->sysctl_ctx, SYSCTL_CHILDREN(dc->sysctl_tree), OID_AUTO, "%parent", CTLTYPE_STRING | CTLFLAG_RD, dc, DEVCLASS_SYSCTL_PARENT, devclass_sysctl_handler, "A", "parent class"); } enum { DEVICE_SYSCTL_DESC, DEVICE_SYSCTL_DRIVER, DEVICE_SYSCTL_LOCATION, DEVICE_SYSCTL_PNPINFO, DEVICE_SYSCTL_PARENT, }; static int device_sysctl_handler(SYSCTL_HANDLER_ARGS) { device_t dev = (device_t)arg1; const char *value; char *buf; int error; buf = NULL; switch (arg2) { case DEVICE_SYSCTL_DESC: value = dev->desc ? dev->desc : ""; break; case DEVICE_SYSCTL_DRIVER: value = dev->driver ? dev->driver->name : ""; break; case DEVICE_SYSCTL_LOCATION: value = buf = malloc(1024, M_BUS, M_WAITOK | M_ZERO); bus_child_location_str(dev, buf, 1024); break; case DEVICE_SYSCTL_PNPINFO: value = buf = malloc(1024, M_BUS, M_WAITOK | M_ZERO); bus_child_pnpinfo_str(dev, buf, 1024); break; case DEVICE_SYSCTL_PARENT: value = dev->parent ? dev->parent->nameunit : ""; break; default: return (EINVAL); } error = SYSCTL_OUT_STR(req, value); if (buf != NULL) free(buf, M_BUS); return (error); } static void device_sysctl_init(device_t dev) { devclass_t dc = dev->devclass; int domain; if (dev->sysctl_tree != NULL) return; devclass_sysctl_init(dc); sysctl_ctx_init(&dev->sysctl_ctx); dev->sysctl_tree = SYSCTL_ADD_NODE(&dev->sysctl_ctx, SYSCTL_CHILDREN(dc->sysctl_tree), OID_AUTO, dev->nameunit + strlen(dc->name), CTLFLAG_RD, NULL, ""); SYSCTL_ADD_PROC(&dev->sysctl_ctx, SYSCTL_CHILDREN(dev->sysctl_tree), OID_AUTO, "%desc", CTLTYPE_STRING | CTLFLAG_RD, dev, DEVICE_SYSCTL_DESC, device_sysctl_handler, "A", "device description"); SYSCTL_ADD_PROC(&dev->sysctl_ctx, SYSCTL_CHILDREN(dev->sysctl_tree), OID_AUTO, "%driver", CTLTYPE_STRING | CTLFLAG_RD, dev, DEVICE_SYSCTL_DRIVER, device_sysctl_handler, "A", "device driver name"); SYSCTL_ADD_PROC(&dev->sysctl_ctx, SYSCTL_CHILDREN(dev->sysctl_tree), OID_AUTO, "%location", CTLTYPE_STRING | CTLFLAG_RD, dev, DEVICE_SYSCTL_LOCATION, device_sysctl_handler, "A", "device location relative to parent"); SYSCTL_ADD_PROC(&dev->sysctl_ctx, SYSCTL_CHILDREN(dev->sysctl_tree), OID_AUTO, "%pnpinfo", CTLTYPE_STRING | CTLFLAG_RD, dev, DEVICE_SYSCTL_PNPINFO, device_sysctl_handler, "A", "device identification"); SYSCTL_ADD_PROC(&dev->sysctl_ctx, SYSCTL_CHILDREN(dev->sysctl_tree), OID_AUTO, "%parent", CTLTYPE_STRING | CTLFLAG_RD, dev, DEVICE_SYSCTL_PARENT, device_sysctl_handler, "A", "parent device"); if (bus_get_domain(dev, &domain) == 0) SYSCTL_ADD_INT(&dev->sysctl_ctx, SYSCTL_CHILDREN(dev->sysctl_tree), OID_AUTO, "%domain", CTLFLAG_RD, NULL, domain, "NUMA domain"); } static void device_sysctl_update(device_t dev) { devclass_t dc = dev->devclass; if (dev->sysctl_tree == NULL) return; sysctl_rename_oid(dev->sysctl_tree, dev->nameunit + strlen(dc->name)); } static void device_sysctl_fini(device_t dev) { if (dev->sysctl_tree == NULL) return; sysctl_ctx_free(&dev->sysctl_ctx); dev->sysctl_tree = NULL; } /* * /dev/devctl implementation */ /* * This design allows only one reader for /dev/devctl. This is not desirable * in the long run, but will get a lot of hair out of this implementation. * Maybe we should make this device a clonable device. * * Also note: we specifically do not attach a device to the device_t tree * to avoid potential chicken and egg problems. One could argue that all * of this belongs to the root node. One could also further argue that the * sysctl interface that we have not might more properly be an ioctl * interface, but at this stage of the game, I'm not inclined to rock that * boat. * * I'm also not sure that the SIGIO support is done correctly or not, as * I copied it from a driver that had SIGIO support that likely hasn't been * tested since 3.4 or 2.2.8! */ /* Deprecated way to adjust queue length */ static int sysctl_devctl_disable(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_hw_bus, OID_AUTO, devctl_disable, CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, NULL, 0, sysctl_devctl_disable, "I", "devctl disable -- deprecated"); #define DEVCTL_DEFAULT_QUEUE_LEN 1000 static int sysctl_devctl_queue(SYSCTL_HANDLER_ARGS); static int devctl_queue_length = DEVCTL_DEFAULT_QUEUE_LEN; SYSCTL_PROC(_hw_bus, OID_AUTO, devctl_queue, CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, NULL, 0, sysctl_devctl_queue, "I", "devctl queue length"); static d_open_t devopen; static d_close_t devclose; static d_read_t devread; static d_ioctl_t devioctl; static d_poll_t devpoll; static d_kqfilter_t devkqfilter; static struct cdevsw dev_cdevsw = { .d_version = D_VERSION, .d_open = devopen, .d_close = devclose, .d_read = devread, .d_ioctl = devioctl, .d_poll = devpoll, .d_kqfilter = devkqfilter, .d_name = "devctl", }; struct dev_event_info { char *dei_data; TAILQ_ENTRY(dev_event_info) dei_link; }; TAILQ_HEAD(devq, dev_event_info); static struct dev_softc { int inuse; int nonblock; int queued; int async; struct mtx mtx; struct cv cv; struct selinfo sel; struct devq devq; struct sigio *sigio; } devsoftc; static void filt_devctl_detach(struct knote *kn); static int filt_devctl_read(struct knote *kn, long hint); struct filterops devctl_rfiltops = { .f_isfd = 1, .f_detach = filt_devctl_detach, .f_event = filt_devctl_read, }; static struct cdev *devctl_dev; static void devinit(void) { devctl_dev = make_dev_credf(MAKEDEV_ETERNAL, &dev_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600, "devctl"); mtx_init(&devsoftc.mtx, "dev mtx", "devd", MTX_DEF); cv_init(&devsoftc.cv, "dev cv"); TAILQ_INIT(&devsoftc.devq); knlist_init_mtx(&devsoftc.sel.si_note, &devsoftc.mtx); devctl2_init(); } static int devopen(struct cdev *dev, int oflags, int devtype, struct thread *td) { mtx_lock(&devsoftc.mtx); if (devsoftc.inuse) { mtx_unlock(&devsoftc.mtx); return (EBUSY); } /* move to init */ devsoftc.inuse = 1; mtx_unlock(&devsoftc.mtx); return (0); } static int devclose(struct cdev *dev, int fflag, int devtype, struct thread *td) { mtx_lock(&devsoftc.mtx); devsoftc.inuse = 0; devsoftc.nonblock = 0; devsoftc.async = 0; cv_broadcast(&devsoftc.cv); funsetown(&devsoftc.sigio); mtx_unlock(&devsoftc.mtx); return (0); } /* * The read channel for this device is used to report changes to * userland in realtime. We are required to free the data as well as * the n1 object because we allocate them separately. Also note that * we return one record at a time. If you try to read this device a * character at a time, you will lose the rest of the data. Listening * programs are expected to cope. */ static int devread(struct cdev *dev, struct uio *uio, int ioflag) { struct dev_event_info *n1; int rv; mtx_lock(&devsoftc.mtx); while (TAILQ_EMPTY(&devsoftc.devq)) { if (devsoftc.nonblock) { mtx_unlock(&devsoftc.mtx); return (EAGAIN); } rv = cv_wait_sig(&devsoftc.cv, &devsoftc.mtx); if (rv) { /* * Need to translate ERESTART to EINTR here? -- jake */ mtx_unlock(&devsoftc.mtx); return (rv); } } n1 = TAILQ_FIRST(&devsoftc.devq); TAILQ_REMOVE(&devsoftc.devq, n1, dei_link); devsoftc.queued--; mtx_unlock(&devsoftc.mtx); rv = uiomove(n1->dei_data, strlen(n1->dei_data), uio); free(n1->dei_data, M_BUS); free(n1, M_BUS); return (rv); } static int devioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { switch (cmd) { case FIONBIO: if (*(int*)data) devsoftc.nonblock = 1; else devsoftc.nonblock = 0; return (0); case FIOASYNC: if (*(int*)data) devsoftc.async = 1; else devsoftc.async = 0; return (0); case FIOSETOWN: return fsetown(*(int *)data, &devsoftc.sigio); case FIOGETOWN: *(int *)data = fgetown(&devsoftc.sigio); return (0); /* (un)Support for other fcntl() calls. */ case FIOCLEX: case FIONCLEX: case FIONREAD: default: break; } return (ENOTTY); } static int devpoll(struct cdev *dev, int events, struct thread *td) { int revents = 0; mtx_lock(&devsoftc.mtx); if (events & (POLLIN | POLLRDNORM)) { if (!TAILQ_EMPTY(&devsoftc.devq)) revents = events & (POLLIN | POLLRDNORM); else selrecord(td, &devsoftc.sel); } mtx_unlock(&devsoftc.mtx); return (revents); } static int devkqfilter(struct cdev *dev, struct knote *kn) { int error; if (kn->kn_filter == EVFILT_READ) { kn->kn_fop = &devctl_rfiltops; knlist_add(&devsoftc.sel.si_note, kn, 0); error = 0; } else error = EINVAL; return (error); } static void filt_devctl_detach(struct knote *kn) { knlist_remove(&devsoftc.sel.si_note, kn, 0); } static int filt_devctl_read(struct knote *kn, long hint) { kn->kn_data = devsoftc.queued; return (kn->kn_data != 0); } /** * @brief Return whether the userland process is running */ boolean_t devctl_process_running(void) { return (devsoftc.inuse == 1); } /** * @brief Queue data to be read from the devctl device * * Generic interface to queue data to the devctl device. It is * assumed that @p data is properly formatted. It is further assumed * that @p data is allocated using the M_BUS malloc type. */ void devctl_queue_data_f(char *data, int flags) { struct dev_event_info *n1 = NULL, *n2 = NULL; if (strlen(data) == 0) goto out; if (devctl_queue_length == 0) goto out; n1 = malloc(sizeof(*n1), M_BUS, flags); if (n1 == NULL) goto out; n1->dei_data = data; mtx_lock(&devsoftc.mtx); if (devctl_queue_length == 0) { mtx_unlock(&devsoftc.mtx); free(n1->dei_data, M_BUS); free(n1, M_BUS); return; } /* Leave at least one spot in the queue... */ while (devsoftc.queued > devctl_queue_length - 1) { n2 = TAILQ_FIRST(&devsoftc.devq); TAILQ_REMOVE(&devsoftc.devq, n2, dei_link); free(n2->dei_data, M_BUS); free(n2, M_BUS); devsoftc.queued--; } TAILQ_INSERT_TAIL(&devsoftc.devq, n1, dei_link); devsoftc.queued++; cv_broadcast(&devsoftc.cv); KNOTE_LOCKED(&devsoftc.sel.si_note, 0); mtx_unlock(&devsoftc.mtx); selwakeup(&devsoftc.sel); if (devsoftc.async && devsoftc.sigio != NULL) pgsigio(&devsoftc.sigio, SIGIO, 0); return; out: /* * We have to free data on all error paths since the caller * assumes it will be free'd when this item is dequeued. */ free(data, M_BUS); return; } void devctl_queue_data(char *data) { devctl_queue_data_f(data, M_NOWAIT); } /** * @brief Send a 'notification' to userland, using standard ways */ void devctl_notify_f(const char *system, const char *subsystem, const char *type, const char *data, int flags) { int len = 0; char *msg; if (system == NULL) return; /* BOGUS! Must specify system. */ if (subsystem == NULL) return; /* BOGUS! Must specify subsystem. */ if (type == NULL) return; /* BOGUS! Must specify type. */ len += strlen(" system=") + strlen(system); len += strlen(" subsystem=") + strlen(subsystem); len += strlen(" type=") + strlen(type); /* add in the data message plus newline. */ if (data != NULL) len += strlen(data); len += 3; /* '!', '\n', and NUL */ msg = malloc(len, M_BUS, flags); if (msg == NULL) return; /* Drop it on the floor */ if (data != NULL) snprintf(msg, len, "!system=%s subsystem=%s type=%s %s\n", system, subsystem, type, data); else snprintf(msg, len, "!system=%s subsystem=%s type=%s\n", system, subsystem, type); devctl_queue_data_f(msg, flags); } void devctl_notify(const char *system, const char *subsystem, const char *type, const char *data) { devctl_notify_f(system, subsystem, type, data, M_NOWAIT); } /* * Common routine that tries to make sending messages as easy as possible. * We allocate memory for the data, copy strings into that, but do not * free it unless there's an error. The dequeue part of the driver should * free the data. We don't send data when the device is disabled. We do * send data, even when we have no listeners, because we wish to avoid * races relating to startup and restart of listening applications. * * devaddq is designed to string together the type of event, with the * object of that event, plus the plug and play info and location info * for that event. This is likely most useful for devices, but less * useful for other consumers of this interface. Those should use * the devctl_queue_data() interface instead. */ static void devaddq(const char *type, const char *what, device_t dev) { char *data = NULL; char *loc = NULL; char *pnp = NULL; const char *parstr; if (!devctl_queue_length)/* Rare race, but lost races safely discard */ return; data = malloc(1024, M_BUS, M_NOWAIT); if (data == NULL) goto bad; /* get the bus specific location of this device */ loc = malloc(1024, M_BUS, M_NOWAIT); if (loc == NULL) goto bad; *loc = '\0'; bus_child_location_str(dev, loc, 1024); /* Get the bus specific pnp info of this device */ pnp = malloc(1024, M_BUS, M_NOWAIT); if (pnp == NULL) goto bad; *pnp = '\0'; bus_child_pnpinfo_str(dev, pnp, 1024); /* Get the parent of this device, or / if high enough in the tree. */ if (device_get_parent(dev) == NULL) parstr = "."; /* Or '/' ? */ else parstr = device_get_nameunit(device_get_parent(dev)); /* String it all together. */ snprintf(data, 1024, "%s%s at %s %s on %s\n", type, what, loc, pnp, parstr); free(loc, M_BUS); free(pnp, M_BUS); devctl_queue_data(data); return; bad: free(pnp, M_BUS); free(loc, M_BUS); free(data, M_BUS); return; } /* * A device was added to the tree. We are called just after it successfully * attaches (that is, probe and attach success for this device). No call * is made if a device is merely parented into the tree. See devnomatch * if probe fails. If attach fails, no notification is sent (but maybe * we should have a different message for this). */ static void devadded(device_t dev) { devaddq("+", device_get_nameunit(dev), dev); } /* * A device was removed from the tree. We are called just before this * happens. */ static void devremoved(device_t dev) { devaddq("-", device_get_nameunit(dev), dev); } /* * Called when there's no match for this device. This is only called * the first time that no match happens, so we don't keep getting this * message. Should that prove to be undesirable, we can change it. * This is called when all drivers that can attach to a given bus * decline to accept this device. Other errors may not be detected. */ static void devnomatch(device_t dev) { devaddq("?", "", dev); } static int sysctl_devctl_disable(SYSCTL_HANDLER_ARGS) { struct dev_event_info *n1; int dis, error; dis = (devctl_queue_length == 0); error = sysctl_handle_int(oidp, &dis, 0, req); if (error || !req->newptr) return (error); if (mtx_initialized(&devsoftc.mtx)) mtx_lock(&devsoftc.mtx); if (dis) { while (!TAILQ_EMPTY(&devsoftc.devq)) { n1 = TAILQ_FIRST(&devsoftc.devq); TAILQ_REMOVE(&devsoftc.devq, n1, dei_link); free(n1->dei_data, M_BUS); free(n1, M_BUS); } devsoftc.queued = 0; devctl_queue_length = 0; } else { devctl_queue_length = DEVCTL_DEFAULT_QUEUE_LEN; } if (mtx_initialized(&devsoftc.mtx)) mtx_unlock(&devsoftc.mtx); return (0); } static int sysctl_devctl_queue(SYSCTL_HANDLER_ARGS) { struct dev_event_info *n1; int q, error; q = devctl_queue_length; error = sysctl_handle_int(oidp, &q, 0, req); if (error || !req->newptr) return (error); if (q < 0) return (EINVAL); if (mtx_initialized(&devsoftc.mtx)) mtx_lock(&devsoftc.mtx); devctl_queue_length = q; while (devsoftc.queued > devctl_queue_length) { n1 = TAILQ_FIRST(&devsoftc.devq); TAILQ_REMOVE(&devsoftc.devq, n1, dei_link); free(n1->dei_data, M_BUS); free(n1, M_BUS); devsoftc.queued--; } if (mtx_initialized(&devsoftc.mtx)) mtx_unlock(&devsoftc.mtx); return (0); } /** * @brief safely quotes strings that might have double quotes in them. * * The devctl protocol relies on quoted strings having matching quotes. * This routine quotes any internal quotes so the resulting string * is safe to pass to snprintf to construct, for example pnp info strings. * Strings are always terminated with a NUL, but may be truncated if longer * than @p len bytes after quotes. * * @param dst Buffer to hold the string. Must be at least @p len bytes long * @param src Original buffer. * @param len Length of buffer pointed to by @dst, including trailing NUL */ void devctl_safe_quote(char *dst, const char *src, size_t len) { char *walker = dst, *ep = dst + len - 1; if (len == 0) return; while (src != NULL && walker < ep) { if (*src == '"' || *src == '\\') { if (ep - walker < 2) break; *walker++ = '\\'; } *walker++ = *src++; } *walker = '\0'; } /* End of /dev/devctl code */ static TAILQ_HEAD(,device) bus_data_devices; static int bus_data_generation = 1; static kobj_method_t null_methods[] = { KOBJMETHOD_END }; DEFINE_CLASS(null, null_methods, 0); /* * Bus pass implementation */ static driver_list_t passes = TAILQ_HEAD_INITIALIZER(passes); int bus_current_pass = BUS_PASS_ROOT; /** * @internal * @brief Register the pass level of a new driver attachment * * Register a new driver attachment's pass level. If no driver * attachment with the same pass level has been added, then @p new * will be added to the global passes list. * * @param new the new driver attachment */ static void driver_register_pass(struct driverlink *new) { struct driverlink *dl; /* We only consider pass numbers during boot. */ if (bus_current_pass == BUS_PASS_DEFAULT) return; /* * Walk the passes list. If we already know about this pass * then there is nothing to do. If we don't, then insert this * driver link into the list. */ TAILQ_FOREACH(dl, &passes, passlink) { if (dl->pass < new->pass) continue; if (dl->pass == new->pass) return; TAILQ_INSERT_BEFORE(dl, new, passlink); return; } TAILQ_INSERT_TAIL(&passes, new, passlink); } /** * @brief Raise the current bus pass * * Raise the current bus pass level to @p pass. Call the BUS_NEW_PASS() * method on the root bus to kick off a new device tree scan for each * new pass level that has at least one driver. */ void bus_set_pass(int pass) { struct driverlink *dl; if (bus_current_pass > pass) panic("Attempt to lower bus pass level"); TAILQ_FOREACH(dl, &passes, passlink) { /* Skip pass values below the current pass level. */ if (dl->pass <= bus_current_pass) continue; /* * Bail once we hit a driver with a pass level that is * too high. */ if (dl->pass > pass) break; /* * Raise the pass level to the next level and rescan * the tree. */ bus_current_pass = dl->pass; BUS_NEW_PASS(root_bus); } /* * If there isn't a driver registered for the requested pass, * then bus_current_pass might still be less than 'pass'. Set * it to 'pass' in that case. */ if (bus_current_pass < pass) bus_current_pass = pass; KASSERT(bus_current_pass == pass, ("Failed to update bus pass level")); } /* * Devclass implementation */ static devclass_list_t devclasses = TAILQ_HEAD_INITIALIZER(devclasses); /** * @internal * @brief Find or create a device class * * If a device class with the name @p classname exists, return it, * otherwise if @p create is non-zero create and return a new device * class. * * If @p parentname is non-NULL, the parent of the devclass is set to * the devclass of that name. * * @param classname the devclass name to find or create * @param parentname the parent devclass name or @c NULL * @param create non-zero to create a devclass */ static devclass_t devclass_find_internal(const char *classname, const char *parentname, int create) { devclass_t dc; PDEBUG(("looking for %s", classname)); if (!classname) return (NULL); TAILQ_FOREACH(dc, &devclasses, link) { if (!strcmp(dc->name, classname)) break; } if (create && !dc) { PDEBUG(("creating %s", classname)); dc = malloc(sizeof(struct devclass) + strlen(classname) + 1, M_BUS, M_NOWAIT | M_ZERO); if (!dc) return (NULL); dc->parent = NULL; dc->name = (char*) (dc + 1); strcpy(dc->name, classname); TAILQ_INIT(&dc->drivers); TAILQ_INSERT_TAIL(&devclasses, dc, link); bus_data_generation_update(); } /* * If a parent class is specified, then set that as our parent so * that this devclass will support drivers for the parent class as * well. If the parent class has the same name don't do this though * as it creates a cycle that can trigger an infinite loop in * device_probe_child() if a device exists for which there is no * suitable driver. */ if (parentname && dc && !dc->parent && strcmp(classname, parentname) != 0) { dc->parent = devclass_find_internal(parentname, NULL, TRUE); dc->parent->flags |= DC_HAS_CHILDREN; } return (dc); } /** * @brief Create a device class * * If a device class with the name @p classname exists, return it, * otherwise create and return a new device class. * * @param classname the devclass name to find or create */ devclass_t devclass_create(const char *classname) { return (devclass_find_internal(classname, NULL, TRUE)); } /** * @brief Find a device class * * If a device class with the name @p classname exists, return it, * otherwise return @c NULL. * * @param classname the devclass name to find */ devclass_t devclass_find(const char *classname) { return (devclass_find_internal(classname, NULL, FALSE)); } /** * @brief Register that a device driver has been added to a devclass * * Register that a device driver has been added to a devclass. This * is called by devclass_add_driver to accomplish the recursive * notification of all the children classes of dc, as well as dc. * Each layer will have BUS_DRIVER_ADDED() called for all instances of * the devclass. * * We do a full search here of the devclass list at each iteration * level to save storing children-lists in the devclass structure. If * we ever move beyond a few dozen devices doing this, we may need to * reevaluate... * * @param dc the devclass to edit * @param driver the driver that was just added */ static void devclass_driver_added(devclass_t dc, driver_t *driver) { devclass_t parent; int i; /* * Call BUS_DRIVER_ADDED for any existing busses in this class. */ for (i = 0; i < dc->maxunit; i++) if (dc->devices[i] && device_is_attached(dc->devices[i])) BUS_DRIVER_ADDED(dc->devices[i], driver); /* * Walk through the children classes. Since we only keep a * single parent pointer around, we walk the entire list of * devclasses looking for children. We set the * DC_HAS_CHILDREN flag when a child devclass is created on * the parent, so we only walk the list for those devclasses * that have children. */ if (!(dc->flags & DC_HAS_CHILDREN)) return; parent = dc; TAILQ_FOREACH(dc, &devclasses, link) { if (dc->parent == parent) devclass_driver_added(dc, driver); } } /** * @brief Add a device driver to a device class * * Add a device driver to a devclass. This is normally called * automatically by DRIVER_MODULE(). The BUS_DRIVER_ADDED() method of * all devices in the devclass will be called to allow them to attempt * to re-probe any unmatched children. * * @param dc the devclass to edit * @param driver the driver to register */ int devclass_add_driver(devclass_t dc, driver_t *driver, int pass, devclass_t *dcp) { driverlink_t dl; const char *parentname; PDEBUG(("%s", DRIVERNAME(driver))); /* Don't allow invalid pass values. */ if (pass <= BUS_PASS_ROOT) return (EINVAL); dl = malloc(sizeof *dl, M_BUS, M_NOWAIT|M_ZERO); if (!dl) return (ENOMEM); /* * Compile the driver's methods. Also increase the reference count * so that the class doesn't get freed when the last instance * goes. This means we can safely use static methods and avoids a * double-free in devclass_delete_driver. */ kobj_class_compile((kobj_class_t) driver); /* * If the driver has any base classes, make the * devclass inherit from the devclass of the driver's * first base class. This will allow the system to * search for drivers in both devclasses for children * of a device using this driver. */ if (driver->baseclasses) parentname = driver->baseclasses[0]->name; else parentname = NULL; *dcp = devclass_find_internal(driver->name, parentname, TRUE); dl->driver = driver; TAILQ_INSERT_TAIL(&dc->drivers, dl, link); driver->refs++; /* XXX: kobj_mtx */ dl->pass = pass; driver_register_pass(dl); devclass_driver_added(dc, driver); bus_data_generation_update(); return (0); } /** * @brief Register that a device driver has been deleted from a devclass * * Register that a device driver has been removed from a devclass. * This is called by devclass_delete_driver to accomplish the * recursive notification of all the children classes of busclass, as * well as busclass. Each layer will attempt to detach the driver * from any devices that are children of the bus's devclass. The function * will return an error if a device fails to detach. * * We do a full search here of the devclass list at each iteration * level to save storing children-lists in the devclass structure. If * we ever move beyond a few dozen devices doing this, we may need to * reevaluate... * * @param busclass the devclass of the parent bus * @param dc the devclass of the driver being deleted * @param driver the driver being deleted */ static int devclass_driver_deleted(devclass_t busclass, devclass_t dc, driver_t *driver) { devclass_t parent; device_t dev; int error, i; /* * Disassociate from any devices. We iterate through all the * devices in the devclass of the driver and detach any which are * using the driver and which have a parent in the devclass which * we are deleting from. * * Note that since a driver can be in multiple devclasses, we * should not detach devices which are not children of devices in * the affected devclass. */ for (i = 0; i < dc->maxunit; i++) { if (dc->devices[i]) { dev = dc->devices[i]; if (dev->driver == driver && dev->parent && dev->parent->devclass == busclass) { if ((error = device_detach(dev)) != 0) return (error); BUS_PROBE_NOMATCH(dev->parent, dev); devnomatch(dev); dev->flags |= DF_DONENOMATCH; } } } /* * Walk through the children classes. Since we only keep a * single parent pointer around, we walk the entire list of * devclasses looking for children. We set the * DC_HAS_CHILDREN flag when a child devclass is created on * the parent, so we only walk the list for those devclasses * that have children. */ if (!(busclass->flags & DC_HAS_CHILDREN)) return (0); parent = busclass; TAILQ_FOREACH(busclass, &devclasses, link) { if (busclass->parent == parent) { error = devclass_driver_deleted(busclass, dc, driver); if (error) return (error); } } return (0); } /** * @brief Delete a device driver from a device class * * Delete a device driver from a devclass. This is normally called * automatically by DRIVER_MODULE(). * * If the driver is currently attached to any devices, * devclass_delete_driver() will first attempt to detach from each * device. If one of the detach calls fails, the driver will not be * deleted. * * @param dc the devclass to edit * @param driver the driver to unregister */ int devclass_delete_driver(devclass_t busclass, driver_t *driver) { devclass_t dc = devclass_find(driver->name); driverlink_t dl; int error; PDEBUG(("%s from devclass %s", driver->name, DEVCLANAME(busclass))); if (!dc) return (0); /* * Find the link structure in the bus' list of drivers. */ TAILQ_FOREACH(dl, &busclass->drivers, link) { if (dl->driver == driver) break; } if (!dl) { PDEBUG(("%s not found in %s list", driver->name, busclass->name)); return (ENOENT); } error = devclass_driver_deleted(busclass, dc, driver); if (error != 0) return (error); TAILQ_REMOVE(&busclass->drivers, dl, link); free(dl, M_BUS); /* XXX: kobj_mtx */ driver->refs--; if (driver->refs == 0) kobj_class_free((kobj_class_t) driver); bus_data_generation_update(); return (0); } /** * @brief Quiesces a set of device drivers from a device class * * Quiesce a device driver from a devclass. This is normally called * automatically by DRIVER_MODULE(). * * If the driver is currently attached to any devices, * devclass_quiesece_driver() will first attempt to quiesce each * device. * * @param dc the devclass to edit * @param driver the driver to unregister */ static int devclass_quiesce_driver(devclass_t busclass, driver_t *driver) { devclass_t dc = devclass_find(driver->name); driverlink_t dl; device_t dev; int i; int error; PDEBUG(("%s from devclass %s", driver->name, DEVCLANAME(busclass))); if (!dc) return (0); /* * Find the link structure in the bus' list of drivers. */ TAILQ_FOREACH(dl, &busclass->drivers, link) { if (dl->driver == driver) break; } if (!dl) { PDEBUG(("%s not found in %s list", driver->name, busclass->name)); return (ENOENT); } /* * Quiesce all devices. We iterate through all the devices in * the devclass of the driver and quiesce any which are using * the driver and which have a parent in the devclass which we * are quiescing. * * Note that since a driver can be in multiple devclasses, we * should not quiesce devices which are not children of * devices in the affected devclass. */ for (i = 0; i < dc->maxunit; i++) { if (dc->devices[i]) { dev = dc->devices[i]; if (dev->driver == driver && dev->parent && dev->parent->devclass == busclass) { if ((error = device_quiesce(dev)) != 0) return (error); } } } return (0); } /** * @internal */ static driverlink_t devclass_find_driver_internal(devclass_t dc, const char *classname) { driverlink_t dl; PDEBUG(("%s in devclass %s", classname, DEVCLANAME(dc))); TAILQ_FOREACH(dl, &dc->drivers, link) { if (!strcmp(dl->driver->name, classname)) return (dl); } PDEBUG(("not found")); return (NULL); } /** * @brief Return the name of the devclass */ const char * devclass_get_name(devclass_t dc) { return (dc->name); } /** * @brief Find a device given a unit number * * @param dc the devclass to search * @param unit the unit number to search for * * @returns the device with the given unit number or @c * NULL if there is no such device */ device_t devclass_get_device(devclass_t dc, int unit) { if (dc == NULL || unit < 0 || unit >= dc->maxunit) return (NULL); return (dc->devices[unit]); } /** * @brief Find the softc field of a device given a unit number * * @param dc the devclass to search * @param unit the unit number to search for * * @returns the softc field of the device with the given * unit number or @c NULL if there is no such * device */ void * devclass_get_softc(devclass_t dc, int unit) { device_t dev; dev = devclass_get_device(dc, unit); if (!dev) return (NULL); return (device_get_softc(dev)); } /** * @brief Get a list of devices in the devclass * * An array containing a list of all the devices in the given devclass * is allocated and returned in @p *devlistp. The number of devices * in the array is returned in @p *devcountp. The caller should free * the array using @c free(p, M_TEMP), even if @p *devcountp is 0. * * @param dc the devclass to examine * @param devlistp points at location for array pointer return * value * @param devcountp points at location for array size return value * * @retval 0 success * @retval ENOMEM the array allocation failed */ int devclass_get_devices(devclass_t dc, device_t **devlistp, int *devcountp) { int count, i; device_t *list; count = devclass_get_count(dc); list = malloc(count * sizeof(device_t), M_TEMP, M_NOWAIT|M_ZERO); if (!list) return (ENOMEM); count = 0; for (i = 0; i < dc->maxunit; i++) { if (dc->devices[i]) { list[count] = dc->devices[i]; count++; } } *devlistp = list; *devcountp = count; return (0); } /** * @brief Get a list of drivers in the devclass * * An array containing a list of pointers to all the drivers in the * given devclass is allocated and returned in @p *listp. The number * of drivers in the array is returned in @p *countp. The caller should * free the array using @c free(p, M_TEMP). * * @param dc the devclass to examine * @param listp gives location for array pointer return value * @param countp gives location for number of array elements * return value * * @retval 0 success * @retval ENOMEM the array allocation failed */ int devclass_get_drivers(devclass_t dc, driver_t ***listp, int *countp) { driverlink_t dl; driver_t **list; int count; count = 0; TAILQ_FOREACH(dl, &dc->drivers, link) count++; list = malloc(count * sizeof(driver_t *), M_TEMP, M_NOWAIT); if (list == NULL) return (ENOMEM); count = 0; TAILQ_FOREACH(dl, &dc->drivers, link) { list[count] = dl->driver; count++; } *listp = list; *countp = count; return (0); } /** * @brief Get the number of devices in a devclass * * @param dc the devclass to examine */ int devclass_get_count(devclass_t dc) { int count, i; count = 0; for (i = 0; i < dc->maxunit; i++) if (dc->devices[i]) count++; return (count); } /** * @brief Get the maximum unit number used in a devclass * * Note that this is one greater than the highest currently-allocated * unit. If a null devclass_t is passed in, -1 is returned to indicate * that not even the devclass has been allocated yet. * * @param dc the devclass to examine */ int devclass_get_maxunit(devclass_t dc) { if (dc == NULL) return (-1); return (dc->maxunit); } /** * @brief Find a free unit number in a devclass * * This function searches for the first unused unit number greater * that or equal to @p unit. * * @param dc the devclass to examine * @param unit the first unit number to check */ int devclass_find_free_unit(devclass_t dc, int unit) { if (dc == NULL) return (unit); while (unit < dc->maxunit && dc->devices[unit] != NULL) unit++; return (unit); } /** * @brief Set the parent of a devclass * * The parent class is normally initialised automatically by * DRIVER_MODULE(). * * @param dc the devclass to edit * @param pdc the new parent devclass */ void devclass_set_parent(devclass_t dc, devclass_t pdc) { dc->parent = pdc; } /** * @brief Get the parent of a devclass * * @param dc the devclass to examine */ devclass_t devclass_get_parent(devclass_t dc) { return (dc->parent); } struct sysctl_ctx_list * devclass_get_sysctl_ctx(devclass_t dc) { return (&dc->sysctl_ctx); } struct sysctl_oid * devclass_get_sysctl_tree(devclass_t dc) { return (dc->sysctl_tree); } /** * @internal * @brief Allocate a unit number * * On entry, @p *unitp is the desired unit number (or @c -1 if any * will do). The allocated unit number is returned in @p *unitp. * @param dc the devclass to allocate from * @param unitp points at the location for the allocated unit * number * * @retval 0 success * @retval EEXIST the requested unit number is already allocated * @retval ENOMEM memory allocation failure */ static int devclass_alloc_unit(devclass_t dc, device_t dev, int *unitp) { const char *s; int unit = *unitp; PDEBUG(("unit %d in devclass %s", unit, DEVCLANAME(dc))); /* Ask the parent bus if it wants to wire this device. */ if (unit == -1) BUS_HINT_DEVICE_UNIT(device_get_parent(dev), dev, dc->name, &unit); /* If we were given a wired unit number, check for existing device */ /* XXX imp XXX */ if (unit != -1) { if (unit >= 0 && unit < dc->maxunit && dc->devices[unit] != NULL) { if (bootverbose) printf("%s: %s%d already exists; skipping it\n", dc->name, dc->name, *unitp); return (EEXIST); } } else { /* Unwired device, find the next available slot for it */ unit = 0; for (unit = 0;; unit++) { /* If there is an "at" hint for a unit then skip it. */ if (resource_string_value(dc->name, unit, "at", &s) == 0) continue; /* If this device slot is already in use, skip it. */ if (unit < dc->maxunit && dc->devices[unit] != NULL) continue; break; } } /* * We've selected a unit beyond the length of the table, so let's * extend the table to make room for all units up to and including * this one. */ if (unit >= dc->maxunit) { device_t *newlist, *oldlist; int newsize; oldlist = dc->devices; newsize = roundup((unit + 1), MINALLOCSIZE / sizeof(device_t)); newlist = malloc(sizeof(device_t) * newsize, M_BUS, M_NOWAIT); if (!newlist) return (ENOMEM); if (oldlist != NULL) bcopy(oldlist, newlist, sizeof(device_t) * dc->maxunit); bzero(newlist + dc->maxunit, sizeof(device_t) * (newsize - dc->maxunit)); dc->devices = newlist; dc->maxunit = newsize; if (oldlist != NULL) free(oldlist, M_BUS); } PDEBUG(("now: unit %d in devclass %s", unit, DEVCLANAME(dc))); *unitp = unit; return (0); } /** * @internal * @brief Add a device to a devclass * * A unit number is allocated for the device (using the device's * preferred unit number if any) and the device is registered in the * devclass. This allows the device to be looked up by its unit * number, e.g. by decoding a dev_t minor number. * * @param dc the devclass to add to * @param dev the device to add * * @retval 0 success * @retval EEXIST the requested unit number is already allocated * @retval ENOMEM memory allocation failure */ static int devclass_add_device(devclass_t dc, device_t dev) { int buflen, error; PDEBUG(("%s in devclass %s", DEVICENAME(dev), DEVCLANAME(dc))); buflen = snprintf(NULL, 0, "%s%d$", dc->name, INT_MAX); if (buflen < 0) return (ENOMEM); dev->nameunit = malloc(buflen, M_BUS, M_NOWAIT|M_ZERO); if (!dev->nameunit) return (ENOMEM); if ((error = devclass_alloc_unit(dc, dev, &dev->unit)) != 0) { free(dev->nameunit, M_BUS); dev->nameunit = NULL; return (error); } dc->devices[dev->unit] = dev; dev->devclass = dc; snprintf(dev->nameunit, buflen, "%s%d", dc->name, dev->unit); return (0); } /** * @internal * @brief Delete a device from a devclass * * The device is removed from the devclass's device list and its unit * number is freed. * @param dc the devclass to delete from * @param dev the device to delete * * @retval 0 success */ static int devclass_delete_device(devclass_t dc, device_t dev) { if (!dc || !dev) return (0); PDEBUG(("%s in devclass %s", DEVICENAME(dev), DEVCLANAME(dc))); if (dev->devclass != dc || dc->devices[dev->unit] != dev) panic("devclass_delete_device: inconsistent device class"); dc->devices[dev->unit] = NULL; if (dev->flags & DF_WILDCARD) dev->unit = -1; dev->devclass = NULL; free(dev->nameunit, M_BUS); dev->nameunit = NULL; return (0); } /** * @internal * @brief Make a new device and add it as a child of @p parent * * @param parent the parent of the new device * @param name the devclass name of the new device or @c NULL * to leave the devclass unspecified * @parem unit the unit number of the new device of @c -1 to * leave the unit number unspecified * * @returns the new device */ static device_t make_device(device_t parent, const char *name, int unit) { device_t dev; devclass_t dc; PDEBUG(("%s at %s as unit %d", name, DEVICENAME(parent), unit)); if (name) { dc = devclass_find_internal(name, NULL, TRUE); if (!dc) { printf("make_device: can't find device class %s\n", name); return (NULL); } } else { dc = NULL; } dev = malloc(sizeof(*dev), M_BUS, M_NOWAIT|M_ZERO); if (!dev) return (NULL); dev->parent = parent; TAILQ_INIT(&dev->children); kobj_init((kobj_t) dev, &null_class); dev->driver = NULL; dev->devclass = NULL; dev->unit = unit; dev->nameunit = NULL; dev->desc = NULL; dev->busy = 0; dev->devflags = 0; dev->flags = DF_ENABLED; dev->order = 0; if (unit == -1) dev->flags |= DF_WILDCARD; if (name) { dev->flags |= DF_FIXEDCLASS; if (devclass_add_device(dc, dev)) { kobj_delete((kobj_t) dev, M_BUS); return (NULL); } } dev->ivars = NULL; dev->softc = NULL; dev->state = DS_NOTPRESENT; TAILQ_INSERT_TAIL(&bus_data_devices, dev, devlink); bus_data_generation_update(); return (dev); } /** * @internal * @brief Print a description of a device. */ static int device_print_child(device_t dev, device_t child) { int retval = 0; if (device_is_alive(child)) retval += BUS_PRINT_CHILD(dev, child); else retval += device_printf(child, " not found\n"); return (retval); } /** * @brief Create a new device * * This creates a new device and adds it as a child of an existing * parent device. The new device will be added after the last existing * child with order zero. * * @param dev the device which will be the parent of the * new child device * @param name devclass name for new device or @c NULL if not * specified * @param unit unit number for new device or @c -1 if not * specified * * @returns the new device */ device_t device_add_child(device_t dev, const char *name, int unit) { return (device_add_child_ordered(dev, 0, name, unit)); } /** * @brief Create a new device * * This creates a new device and adds it as a child of an existing * parent device. The new device will be added after the last existing * child with the same order. * * @param dev the device which will be the parent of the * new child device * @param order a value which is used to partially sort the * children of @p dev - devices created using * lower values of @p order appear first in @p * dev's list of children * @param name devclass name for new device or @c NULL if not * specified * @param unit unit number for new device or @c -1 if not * specified * * @returns the new device */ device_t device_add_child_ordered(device_t dev, u_int order, const char *name, int unit) { device_t child; device_t place; PDEBUG(("%s at %s with order %u as unit %d", name, DEVICENAME(dev), order, unit)); KASSERT(name != NULL || unit == -1, ("child device with wildcard name and specific unit number")); child = make_device(dev, name, unit); if (child == NULL) return (child); child->order = order; TAILQ_FOREACH(place, &dev->children, link) { if (place->order > order) break; } if (place) { /* * The device 'place' is the first device whose order is * greater than the new child. */ TAILQ_INSERT_BEFORE(place, child, link); } else { /* * The new child's order is greater or equal to the order of * any existing device. Add the child to the tail of the list. */ TAILQ_INSERT_TAIL(&dev->children, child, link); } bus_data_generation_update(); return (child); } /** * @brief Delete a device * * This function deletes a device along with all of its children. If * the device currently has a driver attached to it, the device is * detached first using device_detach(). * * @param dev the parent device * @param child the device to delete * * @retval 0 success * @retval non-zero a unit error code describing the error */ int device_delete_child(device_t dev, device_t child) { int error; device_t grandchild; PDEBUG(("%s from %s", DEVICENAME(child), DEVICENAME(dev))); /* remove children first */ while ((grandchild = TAILQ_FIRST(&child->children)) != NULL) { error = device_delete_child(child, grandchild); if (error) return (error); } if ((error = device_detach(child)) != 0) return (error); if (child->devclass) devclass_delete_device(child->devclass, child); if (child->parent) BUS_CHILD_DELETED(dev, child); TAILQ_REMOVE(&dev->children, child, link); TAILQ_REMOVE(&bus_data_devices, child, devlink); kobj_delete((kobj_t) child, M_BUS); bus_data_generation_update(); return (0); } /** * @brief Delete all children devices of the given device, if any. * * This function deletes all children devices of the given device, if * any, using the device_delete_child() function for each device it * finds. If a child device cannot be deleted, this function will * return an error code. * * @param dev the parent device * * @retval 0 success * @retval non-zero a device would not detach */ int device_delete_children(device_t dev) { device_t child; int error; PDEBUG(("Deleting all children of %s", DEVICENAME(dev))); error = 0; while ((child = TAILQ_FIRST(&dev->children)) != NULL) { error = device_delete_child(dev, child); if (error) { PDEBUG(("Failed deleting %s", DEVICENAME(child))); break; } } return (error); } /** * @brief Find a device given a unit number * * This is similar to devclass_get_devices() but only searches for * devices which have @p dev as a parent. * * @param dev the parent device to search * @param unit the unit number to search for. If the unit is -1, * return the first child of @p dev which has name * @p classname (that is, the one with the lowest unit.) * * @returns the device with the given unit number or @c * NULL if there is no such device */ device_t device_find_child(device_t dev, const char *classname, int unit) { devclass_t dc; device_t child; dc = devclass_find(classname); if (!dc) return (NULL); if (unit != -1) { child = devclass_get_device(dc, unit); if (child && child->parent == dev) return (child); } else { for (unit = 0; unit < devclass_get_maxunit(dc); unit++) { child = devclass_get_device(dc, unit); if (child && child->parent == dev) return (child); } } return (NULL); } /** * @internal */ static driverlink_t first_matching_driver(devclass_t dc, device_t dev) { if (dev->devclass) return (devclass_find_driver_internal(dc, dev->devclass->name)); return (TAILQ_FIRST(&dc->drivers)); } /** * @internal */ static driverlink_t next_matching_driver(devclass_t dc, device_t dev, driverlink_t last) { if (dev->devclass) { driverlink_t dl; for (dl = TAILQ_NEXT(last, link); dl; dl = TAILQ_NEXT(dl, link)) if (!strcmp(dev->devclass->name, dl->driver->name)) return (dl); return (NULL); } return (TAILQ_NEXT(last, link)); } /** * @internal */ int device_probe_child(device_t dev, device_t child) { devclass_t dc; driverlink_t best = NULL; driverlink_t dl; int result, pri = 0; int hasclass = (child->devclass != NULL); GIANT_REQUIRED; dc = dev->devclass; if (!dc) panic("device_probe_child: parent device has no devclass"); /* * If the state is already probed, then return. However, don't * return if we can rebid this object. */ if (child->state == DS_ALIVE && (child->flags & DF_REBID) == 0) return (0); for (; dc; dc = dc->parent) { for (dl = first_matching_driver(dc, child); dl; dl = next_matching_driver(dc, child, dl)) { /* If this driver's pass is too high, then ignore it. */ if (dl->pass > bus_current_pass) continue; PDEBUG(("Trying %s", DRIVERNAME(dl->driver))); result = device_set_driver(child, dl->driver); if (result == ENOMEM) return (result); else if (result != 0) continue; if (!hasclass) { if (device_set_devclass(child, dl->driver->name) != 0) { char const * devname = device_get_name(child); if (devname == NULL) devname = "(unknown)"; printf("driver bug: Unable to set " "devclass (class: %s " "devname: %s)\n", dl->driver->name, devname); (void)device_set_driver(child, NULL); continue; } } /* Fetch any flags for the device before probing. */ resource_int_value(dl->driver->name, child->unit, "flags", &child->devflags); result = DEVICE_PROBE(child); /* Reset flags and devclass before the next probe. */ child->devflags = 0; if (!hasclass) (void)device_set_devclass(child, NULL); /* * If the driver returns SUCCESS, there can be * no higher match for this device. */ if (result == 0) { best = dl; pri = 0; break; } /* * Probes that return BUS_PROBE_NOWILDCARD or lower * only match on devices whose driver was explicitly * specified. */ if (result <= BUS_PROBE_NOWILDCARD && !(child->flags & DF_FIXEDCLASS)) { result = ENXIO; } /* * The driver returned an error so it * certainly doesn't match. */ if (result > 0) { (void)device_set_driver(child, NULL); continue; } /* * A priority lower than SUCCESS, remember the * best matching driver. Initialise the value * of pri for the first match. */ if (best == NULL || result > pri) { best = dl; pri = result; continue; } } /* * If we have an unambiguous match in this devclass, * don't look in the parent. */ if (best && pri == 0) break; } /* * If we found a driver, change state and initialise the devclass. */ /* XXX What happens if we rebid and got no best? */ if (best) { /* * If this device was attached, and we were asked to * rescan, and it is a different driver, then we have * to detach the old driver and reattach this new one. * Note, we don't have to check for DF_REBID here * because if the state is > DS_ALIVE, we know it must * be. * * This assumes that all DF_REBID drivers can have * their probe routine called at any time and that * they are idempotent as well as completely benign in * normal operations. * * We also have to make sure that the detach * succeeded, otherwise we fail the operation (or * maybe it should just fail silently? I'm torn). */ if (child->state > DS_ALIVE && best->driver != child->driver) if ((result = device_detach(dev)) != 0) return (result); /* Set the winning driver, devclass, and flags. */ if (!child->devclass) { result = device_set_devclass(child, best->driver->name); if (result != 0) return (result); } result = device_set_driver(child, best->driver); if (result != 0) return (result); resource_int_value(best->driver->name, child->unit, "flags", &child->devflags); if (pri < 0) { /* * A bit bogus. Call the probe method again to make * sure that we have the right description. */ DEVICE_PROBE(child); #if 0 child->flags |= DF_REBID; #endif } else child->flags &= ~DF_REBID; child->state = DS_ALIVE; bus_data_generation_update(); return (0); } return (ENXIO); } /** * @brief Return the parent of a device */ device_t device_get_parent(device_t dev) { return (dev->parent); } /** * @brief Get a list of children of a device * * An array containing a list of all the children of the given device * is allocated and returned in @p *devlistp. The number of devices * in the array is returned in @p *devcountp. The caller should free * the array using @c free(p, M_TEMP). * * @param dev the device to examine * @param devlistp points at location for array pointer return * value * @param devcountp points at location for array size return value * * @retval 0 success * @retval ENOMEM the array allocation failed */ int device_get_children(device_t dev, device_t **devlistp, int *devcountp) { int count; device_t child; device_t *list; count = 0; TAILQ_FOREACH(child, &dev->children, link) { count++; } if (count == 0) { *devlistp = NULL; *devcountp = 0; return (0); } list = malloc(count * sizeof(device_t), M_TEMP, M_NOWAIT|M_ZERO); if (!list) return (ENOMEM); count = 0; TAILQ_FOREACH(child, &dev->children, link) { list[count] = child; count++; } *devlistp = list; *devcountp = count; return (0); } /** * @brief Return the current driver for the device or @c NULL if there * is no driver currently attached */ driver_t * device_get_driver(device_t dev) { return (dev->driver); } /** * @brief Return the current devclass for the device or @c NULL if * there is none. */ devclass_t device_get_devclass(device_t dev) { return (dev->devclass); } /** * @brief Return the name of the device's devclass or @c NULL if there * is none. */ const char * device_get_name(device_t dev) { if (dev != NULL && dev->devclass) return (devclass_get_name(dev->devclass)); return (NULL); } /** * @brief Return a string containing the device's devclass name * followed by an ascii representation of the device's unit number * (e.g. @c "foo2"). */ const char * device_get_nameunit(device_t dev) { return (dev->nameunit); } /** * @brief Return the device's unit number. */ int device_get_unit(device_t dev) { return (dev->unit); } /** * @brief Return the device's description string */ const char * device_get_desc(device_t dev) { return (dev->desc); } /** * @brief Return the device's flags */ uint32_t device_get_flags(device_t dev) { return (dev->devflags); } struct sysctl_ctx_list * device_get_sysctl_ctx(device_t dev) { return (&dev->sysctl_ctx); } struct sysctl_oid * device_get_sysctl_tree(device_t dev) { return (dev->sysctl_tree); } /** * @brief Print the name of the device followed by a colon and a space * * @returns the number of characters printed */ int device_print_prettyname(device_t dev) { const char *name = device_get_name(dev); if (name == NULL) return (printf("unknown: ")); return (printf("%s%d: ", name, device_get_unit(dev))); } /** * @brief Print the name of the device followed by a colon, a space * and the result of calling vprintf() with the value of @p fmt and * the following arguments. * * @returns the number of characters printed */ int device_printf(device_t dev, const char * fmt, ...) { va_list ap; int retval; retval = device_print_prettyname(dev); va_start(ap, fmt); retval += vprintf(fmt, ap); va_end(ap); return (retval); } /** * @internal */ static void device_set_desc_internal(device_t dev, const char* desc, int copy) { if (dev->desc && (dev->flags & DF_DESCMALLOCED)) { free(dev->desc, M_BUS); dev->flags &= ~DF_DESCMALLOCED; dev->desc = NULL; } if (copy && desc) { dev->desc = malloc(strlen(desc) + 1, M_BUS, M_NOWAIT); if (dev->desc) { strcpy(dev->desc, desc); dev->flags |= DF_DESCMALLOCED; } } else { /* Avoid a -Wcast-qual warning */ dev->desc = (char *)(uintptr_t) desc; } bus_data_generation_update(); } /** * @brief Set the device's description * * The value of @c desc should be a string constant that will not * change (at least until the description is changed in a subsequent * call to device_set_desc() or device_set_desc_copy()). */ void device_set_desc(device_t dev, const char* desc) { device_set_desc_internal(dev, desc, FALSE); } /** * @brief Set the device's description * * The string pointed to by @c desc is copied. Use this function if * the device description is generated, (e.g. with sprintf()). */ void device_set_desc_copy(device_t dev, const char* desc) { device_set_desc_internal(dev, desc, TRUE); } /** * @brief Set the device's flags */ void device_set_flags(device_t dev, uint32_t flags) { dev->devflags = flags; } /** * @brief Return the device's softc field * * The softc is allocated and zeroed when a driver is attached, based * on the size field of the driver. */ void * device_get_softc(device_t dev) { return (dev->softc); } /** * @brief Set the device's softc field * * Most drivers do not need to use this since the softc is allocated * automatically when the driver is attached. */ void device_set_softc(device_t dev, void *softc) { if (dev->softc && !(dev->flags & DF_EXTERNALSOFTC)) free(dev->softc, M_BUS_SC); dev->softc = softc; if (dev->softc) dev->flags |= DF_EXTERNALSOFTC; else dev->flags &= ~DF_EXTERNALSOFTC; } /** * @brief Free claimed softc * * Most drivers do not need to use this since the softc is freed * automatically when the driver is detached. */ void device_free_softc(void *softc) { free(softc, M_BUS_SC); } /** * @brief Claim softc * * This function can be used to let the driver free the automatically * allocated softc using "device_free_softc()". This function is * useful when the driver is refcounting the softc and the softc * cannot be freed when the "device_detach" method is called. */ void device_claim_softc(device_t dev) { if (dev->softc) dev->flags |= DF_EXTERNALSOFTC; else dev->flags &= ~DF_EXTERNALSOFTC; } /** * @brief Get the device's ivars field * * The ivars field is used by the parent device to store per-device * state (e.g. the physical location of the device or a list of * resources). */ void * device_get_ivars(device_t dev) { KASSERT(dev != NULL, ("device_get_ivars(NULL, ...)")); return (dev->ivars); } /** * @brief Set the device's ivars field */ void device_set_ivars(device_t dev, void * ivars) { KASSERT(dev != NULL, ("device_set_ivars(NULL, ...)")); dev->ivars = ivars; } /** * @brief Return the device's state */ device_state_t device_get_state(device_t dev) { return (dev->state); } /** * @brief Set the DF_ENABLED flag for the device */ void device_enable(device_t dev) { dev->flags |= DF_ENABLED; } /** * @brief Clear the DF_ENABLED flag for the device */ void device_disable(device_t dev) { dev->flags &= ~DF_ENABLED; } /** * @brief Increment the busy counter for the device */ void device_busy(device_t dev) { if (dev->state < DS_ATTACHING) panic("device_busy: called for unattached device"); if (dev->busy == 0 && dev->parent) device_busy(dev->parent); dev->busy++; if (dev->state == DS_ATTACHED) dev->state = DS_BUSY; } /** * @brief Decrement the busy counter for the device */ void device_unbusy(device_t dev) { if (dev->busy != 0 && dev->state != DS_BUSY && dev->state != DS_ATTACHING) panic("device_unbusy: called for non-busy device %s", device_get_nameunit(dev)); dev->busy--; if (dev->busy == 0) { if (dev->parent) device_unbusy(dev->parent); if (dev->state == DS_BUSY) dev->state = DS_ATTACHED; } } /** * @brief Set the DF_QUIET flag for the device */ void device_quiet(device_t dev) { dev->flags |= DF_QUIET; } /** * @brief Clear the DF_QUIET flag for the device */ void device_verbose(device_t dev) { dev->flags &= ~DF_QUIET; } /** * @brief Return non-zero if the DF_QUIET flag is set on the device */ int device_is_quiet(device_t dev) { return ((dev->flags & DF_QUIET) != 0); } /** * @brief Return non-zero if the DF_ENABLED flag is set on the device */ int device_is_enabled(device_t dev) { return ((dev->flags & DF_ENABLED) != 0); } /** * @brief Return non-zero if the device was successfully probed */ int device_is_alive(device_t dev) { return (dev->state >= DS_ALIVE); } /** * @brief Return non-zero if the device currently has a driver * attached to it */ int device_is_attached(device_t dev) { return (dev->state >= DS_ATTACHED); } /** * @brief Return non-zero if the device is currently suspended. */ int device_is_suspended(device_t dev) { return ((dev->flags & DF_SUSPENDED) != 0); } /** * @brief Set the devclass of a device * @see devclass_add_device(). */ int device_set_devclass(device_t dev, const char *classname) { devclass_t dc; int error; if (!classname) { if (dev->devclass) devclass_delete_device(dev->devclass, dev); return (0); } if (dev->devclass) { printf("device_set_devclass: device class already set\n"); return (EINVAL); } dc = devclass_find_internal(classname, NULL, TRUE); if (!dc) return (ENOMEM); error = devclass_add_device(dc, dev); bus_data_generation_update(); return (error); } /** * @brief Set the devclass of a device and mark the devclass fixed. * @see device_set_devclass() */ int device_set_devclass_fixed(device_t dev, const char *classname) { int error; if (classname == NULL) return (EINVAL); error = device_set_devclass(dev, classname); if (error) return (error); dev->flags |= DF_FIXEDCLASS; return (0); } /** * @brief Set the driver of a device * * @retval 0 success * @retval EBUSY the device already has a driver attached * @retval ENOMEM a memory allocation failure occurred */ int device_set_driver(device_t dev, driver_t *driver) { if (dev->state >= DS_ATTACHED) return (EBUSY); if (dev->driver == driver) return (0); if (dev->softc && !(dev->flags & DF_EXTERNALSOFTC)) { free(dev->softc, M_BUS_SC); dev->softc = NULL; } device_set_desc(dev, NULL); kobj_delete((kobj_t) dev, NULL); dev->driver = driver; if (driver) { kobj_init((kobj_t) dev, (kobj_class_t) driver); if (!(dev->flags & DF_EXTERNALSOFTC) && driver->size > 0) { dev->softc = malloc(driver->size, M_BUS_SC, M_NOWAIT | M_ZERO); if (!dev->softc) { kobj_delete((kobj_t) dev, NULL); kobj_init((kobj_t) dev, &null_class); dev->driver = NULL; return (ENOMEM); } } } else { kobj_init((kobj_t) dev, &null_class); } bus_data_generation_update(); return (0); } /** * @brief Probe a device, and return this status. * * This function is the core of the device autoconfiguration * system. Its purpose is to select a suitable driver for a device and * then call that driver to initialise the hardware appropriately. The * driver is selected by calling the DEVICE_PROBE() method of a set of * candidate drivers and then choosing the driver which returned the * best value. This driver is then attached to the device using * device_attach(). * * The set of suitable drivers is taken from the list of drivers in * the parent device's devclass. If the device was originally created * with a specific class name (see device_add_child()), only drivers * with that name are probed, otherwise all drivers in the devclass * are probed. If no drivers return successful probe values in the * parent devclass, the search continues in the parent of that * devclass (see devclass_get_parent()) if any. * * @param dev the device to initialise * * @retval 0 success * @retval ENXIO no driver was found * @retval ENOMEM memory allocation failure * @retval non-zero some other unix error code * @retval -1 Device already attached */ int device_probe(device_t dev) { int error; GIANT_REQUIRED; if (dev->state >= DS_ALIVE && (dev->flags & DF_REBID) == 0) return (-1); if (!(dev->flags & DF_ENABLED)) { if (bootverbose && device_get_name(dev) != NULL) { device_print_prettyname(dev); printf("not probed (disabled)\n"); } return (-1); } if ((error = device_probe_child(dev->parent, dev)) != 0) { if (bus_current_pass == BUS_PASS_DEFAULT && !(dev->flags & DF_DONENOMATCH)) { BUS_PROBE_NOMATCH(dev->parent, dev); devnomatch(dev); dev->flags |= DF_DONENOMATCH; } return (error); } return (0); } /** * @brief Probe a device and attach a driver if possible * * calls device_probe() and attaches if that was successful. */ int device_probe_and_attach(device_t dev) { int error; GIANT_REQUIRED; error = device_probe(dev); if (error == -1) return (0); else if (error != 0) return (error); CURVNET_SET_QUIET(vnet0); error = device_attach(dev); CURVNET_RESTORE(); return error; } /** * @brief Attach a device driver to a device * * This function is a wrapper around the DEVICE_ATTACH() driver * method. In addition to calling DEVICE_ATTACH(), it initialises the * device's sysctl tree, optionally prints a description of the device * and queues a notification event for user-based device management * services. * * Normally this function is only called internally from * device_probe_and_attach(). * * @param dev the device to initialise * * @retval 0 success * @retval ENXIO no driver was found * @retval ENOMEM memory allocation failure * @retval non-zero some other unix error code */ int device_attach(device_t dev) { uint64_t attachtime; int error; if (resource_disabled(dev->driver->name, dev->unit)) { device_disable(dev); if (bootverbose) device_printf(dev, "disabled via hints entry\n"); return (ENXIO); } device_sysctl_init(dev); if (!device_is_quiet(dev)) device_print_child(dev->parent, dev); attachtime = get_cyclecount(); dev->state = DS_ATTACHING; if ((error = DEVICE_ATTACH(dev)) != 0) { printf("device_attach: %s%d attach returned %d\n", dev->driver->name, dev->unit, error); if (!(dev->flags & DF_FIXEDCLASS)) devclass_delete_device(dev->devclass, dev); (void)device_set_driver(dev, NULL); device_sysctl_fini(dev); KASSERT(dev->busy == 0, ("attach failed but busy")); dev->state = DS_NOTPRESENT; return (error); } attachtime = get_cyclecount() - attachtime; /* * 4 bits per device is a reasonable value for desktop and server * hardware with good get_cyclecount() implementations, but WILL * need to be adjusted on other platforms. */ #define RANDOM_PROBE_BIT_GUESS 4 if (bootverbose) printf("random: harvesting attach, %zu bytes (%d bits) from %s%d\n", sizeof(attachtime), RANDOM_PROBE_BIT_GUESS, dev->driver->name, dev->unit); random_harvest_direct(&attachtime, sizeof(attachtime), RANDOM_PROBE_BIT_GUESS, RANDOM_ATTACH); device_sysctl_update(dev); if (dev->busy) dev->state = DS_BUSY; else dev->state = DS_ATTACHED; dev->flags &= ~DF_DONENOMATCH; devadded(dev); return (0); } /** * @brief Detach a driver from a device * * This function is a wrapper around the DEVICE_DETACH() driver * method. If the call to DEVICE_DETACH() succeeds, it calls * BUS_CHILD_DETACHED() for the parent of @p dev, queues a * notification event for user-based device management services and * cleans up the device's sysctl tree. * * @param dev the device to un-initialise * * @retval 0 success * @retval ENXIO no driver was found * @retval ENOMEM memory allocation failure * @retval non-zero some other unix error code */ int device_detach(device_t dev) { int error; GIANT_REQUIRED; PDEBUG(("%s", DEVICENAME(dev))); if (dev->state == DS_BUSY) return (EBUSY); if (dev->state != DS_ATTACHED) return (0); if ((error = DEVICE_DETACH(dev)) != 0) return (error); devremoved(dev); if (!device_is_quiet(dev)) device_printf(dev, "detached\n"); if (dev->parent) BUS_CHILD_DETACHED(dev->parent, dev); if (!(dev->flags & DF_FIXEDCLASS)) devclass_delete_device(dev->devclass, dev); dev->state = DS_NOTPRESENT; (void)device_set_driver(dev, NULL); device_sysctl_fini(dev); return (0); } /** * @brief Tells a driver to quiesce itself. * * This function is a wrapper around the DEVICE_QUIESCE() driver * method. If the call to DEVICE_QUIESCE() succeeds. * * @param dev the device to quiesce * * @retval 0 success * @retval ENXIO no driver was found * @retval ENOMEM memory allocation failure * @retval non-zero some other unix error code */ int device_quiesce(device_t dev) { PDEBUG(("%s", DEVICENAME(dev))); if (dev->state == DS_BUSY) return (EBUSY); if (dev->state != DS_ATTACHED) return (0); return (DEVICE_QUIESCE(dev)); } /** * @brief Notify a device of system shutdown * * This function calls the DEVICE_SHUTDOWN() driver method if the * device currently has an attached driver. * * @returns the value returned by DEVICE_SHUTDOWN() */ int device_shutdown(device_t dev) { if (dev->state < DS_ATTACHED) return (0); return (DEVICE_SHUTDOWN(dev)); } /** * @brief Set the unit number of a device * * This function can be used to override the unit number used for a * device (e.g. to wire a device to a pre-configured unit number). */ int device_set_unit(device_t dev, int unit) { devclass_t dc; int err; dc = device_get_devclass(dev); if (unit < dc->maxunit && dc->devices[unit]) return (EBUSY); err = devclass_delete_device(dc, dev); if (err) return (err); dev->unit = unit; err = devclass_add_device(dc, dev); if (err) return (err); bus_data_generation_update(); return (0); } /*======================================*/ /* * Some useful method implementations to make life easier for bus drivers. */ void resource_init_map_request_impl(struct resource_map_request *args, size_t sz) { bzero(args, sz); args->size = sz; args->memattr = VM_MEMATTR_UNCACHEABLE; } /** * @brief Initialise a resource list. * * @param rl the resource list to initialise */ void resource_list_init(struct resource_list *rl) { STAILQ_INIT(rl); } /** * @brief Reclaim memory used by a resource list. * * This function frees the memory for all resource entries on the list * (if any). * * @param rl the resource list to free */ void resource_list_free(struct resource_list *rl) { struct resource_list_entry *rle; while ((rle = STAILQ_FIRST(rl)) != NULL) { if (rle->res) panic("resource_list_free: resource entry is busy"); STAILQ_REMOVE_HEAD(rl, link); free(rle, M_BUS); } } /** * @brief Add a resource entry. * * This function adds a resource entry using the given @p type, @p * start, @p end and @p count values. A rid value is chosen by * searching sequentially for the first unused rid starting at zero. * * @param rl the resource list to edit * @param type the resource entry type (e.g. SYS_RES_MEMORY) * @param start the start address of the resource * @param end the end address of the resource * @param count XXX end-start+1 */ int resource_list_add_next(struct resource_list *rl, int type, rman_res_t start, rman_res_t end, rman_res_t count) { int rid; rid = 0; while (resource_list_find(rl, type, rid) != NULL) rid++; resource_list_add(rl, type, rid, start, end, count); return (rid); } /** * @brief Add or modify a resource entry. * * If an existing entry exists with the same type and rid, it will be * modified using the given values of @p start, @p end and @p * count. If no entry exists, a new one will be created using the * given values. The resource list entry that matches is then returned. * * @param rl the resource list to edit * @param type the resource entry type (e.g. SYS_RES_MEMORY) * @param rid the resource identifier * @param start the start address of the resource * @param end the end address of the resource * @param count XXX end-start+1 */ struct resource_list_entry * resource_list_add(struct resource_list *rl, int type, int rid, rman_res_t start, rman_res_t end, rman_res_t count) { struct resource_list_entry *rle; rle = resource_list_find(rl, type, rid); if (!rle) { rle = malloc(sizeof(struct resource_list_entry), M_BUS, M_NOWAIT); if (!rle) panic("resource_list_add: can't record entry"); STAILQ_INSERT_TAIL(rl, rle, link); rle->type = type; rle->rid = rid; rle->res = NULL; rle->flags = 0; } if (rle->res) panic("resource_list_add: resource entry is busy"); rle->start = start; rle->end = end; rle->count = count; return (rle); } /** * @brief Determine if a resource entry is busy. * * Returns true if a resource entry is busy meaning that it has an * associated resource that is not an unallocated "reserved" resource. * * @param rl the resource list to search * @param type the resource entry type (e.g. SYS_RES_MEMORY) * @param rid the resource identifier * * @returns Non-zero if the entry is busy, zero otherwise. */ int resource_list_busy(struct resource_list *rl, int type, int rid) { struct resource_list_entry *rle; rle = resource_list_find(rl, type, rid); if (rle == NULL || rle->res == NULL) return (0); if ((rle->flags & (RLE_RESERVED | RLE_ALLOCATED)) == RLE_RESERVED) { KASSERT(!(rman_get_flags(rle->res) & RF_ACTIVE), ("reserved resource is active")); return (0); } return (1); } /** * @brief Determine if a resource entry is reserved. * * Returns true if a resource entry is reserved meaning that it has an * associated "reserved" resource. The resource can either be * allocated or unallocated. * * @param rl the resource list to search * @param type the resource entry type (e.g. SYS_RES_MEMORY) * @param rid the resource identifier * * @returns Non-zero if the entry is reserved, zero otherwise. */ int resource_list_reserved(struct resource_list *rl, int type, int rid) { struct resource_list_entry *rle; rle = resource_list_find(rl, type, rid); if (rle != NULL && rle->flags & RLE_RESERVED) return (1); return (0); } /** * @brief Find a resource entry by type and rid. * * @param rl the resource list to search * @param type the resource entry type (e.g. SYS_RES_MEMORY) * @param rid the resource identifier * * @returns the resource entry pointer or NULL if there is no such * entry. */ struct resource_list_entry * resource_list_find(struct resource_list *rl, int type, int rid) { struct resource_list_entry *rle; STAILQ_FOREACH(rle, rl, link) { if (rle->type == type && rle->rid == rid) return (rle); } return (NULL); } /** * @brief Delete a resource entry. * * @param rl the resource list to edit * @param type the resource entry type (e.g. SYS_RES_MEMORY) * @param rid the resource identifier */ void resource_list_delete(struct resource_list *rl, int type, int rid) { struct resource_list_entry *rle = resource_list_find(rl, type, rid); if (rle) { if (rle->res != NULL) panic("resource_list_delete: resource has not been released"); STAILQ_REMOVE(rl, rle, resource_list_entry, link); free(rle, M_BUS); } } /** * @brief Allocate a reserved resource * * This can be used by busses to force the allocation of resources * that are always active in the system even if they are not allocated * by a driver (e.g. PCI BARs). This function is usually called when * adding a new child to the bus. The resource is allocated from the * parent bus when it is reserved. The resource list entry is marked * with RLE_RESERVED to note that it is a reserved resource. * * Subsequent attempts to allocate the resource with * resource_list_alloc() will succeed the first time and will set * RLE_ALLOCATED to note that it has been allocated. When a reserved * resource that has been allocated is released with * resource_list_release() the resource RLE_ALLOCATED is cleared, but * the actual resource remains allocated. The resource can be released to * the parent bus by calling resource_list_unreserve(). * * @param rl the resource list to allocate from * @param bus the parent device of @p child * @param child the device for which the resource is being reserved * @param type the type of resource to allocate * @param rid a pointer to the resource identifier * @param start hint at the start of the resource range - pass * @c 0 for any start address * @param end hint at the end of the resource range - pass * @c ~0 for any end address * @param count hint at the size of range required - pass @c 1 * for any size * @param flags any extra flags to control the resource * allocation - see @c RF_XXX flags in * for details * * @returns the resource which was allocated or @c NULL if no * resource could be allocated */ struct resource * resource_list_reserve(struct resource_list *rl, device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource_list_entry *rle = NULL; int passthrough = (device_get_parent(child) != bus); struct resource *r; if (passthrough) panic( "resource_list_reserve() should only be called for direct children"); if (flags & RF_ACTIVE) panic( "resource_list_reserve() should only reserve inactive resources"); r = resource_list_alloc(rl, bus, child, type, rid, start, end, count, flags); if (r != NULL) { rle = resource_list_find(rl, type, *rid); rle->flags |= RLE_RESERVED; } return (r); } /** * @brief Helper function for implementing BUS_ALLOC_RESOURCE() * * Implement BUS_ALLOC_RESOURCE() by looking up a resource from the list * and passing the allocation up to the parent of @p bus. This assumes * that the first entry of @c device_get_ivars(child) is a struct * resource_list. This also handles 'passthrough' allocations where a * child is a remote descendant of bus by passing the allocation up to * the parent of bus. * * Typically, a bus driver would store a list of child resources * somewhere in the child device's ivars (see device_get_ivars()) and * its implementation of BUS_ALLOC_RESOURCE() would find that list and * then call resource_list_alloc() to perform the allocation. * * @param rl the resource list to allocate from * @param bus the parent device of @p child * @param child the device which is requesting an allocation * @param type the type of resource to allocate * @param rid a pointer to the resource identifier * @param start hint at the start of the resource range - pass * @c 0 for any start address * @param end hint at the end of the resource range - pass * @c ~0 for any end address * @param count hint at the size of range required - pass @c 1 * for any size * @param flags any extra flags to control the resource * allocation - see @c RF_XXX flags in * for details * * @returns the resource which was allocated or @c NULL if no * resource could be allocated */ struct resource * resource_list_alloc(struct resource_list *rl, device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource_list_entry *rle = NULL; int passthrough = (device_get_parent(child) != bus); int isdefault = RMAN_IS_DEFAULT_RANGE(start, end); if (passthrough) { return (BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type, rid, start, end, count, flags)); } rle = resource_list_find(rl, type, *rid); if (!rle) return (NULL); /* no resource of that type/rid */ if (rle->res) { if (rle->flags & RLE_RESERVED) { if (rle->flags & RLE_ALLOCATED) return (NULL); if ((flags & RF_ACTIVE) && bus_activate_resource(child, type, *rid, rle->res) != 0) return (NULL); rle->flags |= RLE_ALLOCATED; return (rle->res); } device_printf(bus, "resource entry %#x type %d for child %s is busy\n", *rid, type, device_get_nameunit(child)); return (NULL); } if (isdefault) { start = rle->start; count = ulmax(count, rle->count); end = ulmax(rle->end, start + count - 1); } rle->res = BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type, rid, start, end, count, flags); /* * Record the new range. */ if (rle->res) { rle->start = rman_get_start(rle->res); rle->end = rman_get_end(rle->res); rle->count = count; } return (rle->res); } /** * @brief Helper function for implementing BUS_RELEASE_RESOURCE() * * Implement BUS_RELEASE_RESOURCE() using a resource list. Normally * used with resource_list_alloc(). * * @param rl the resource list which was allocated from * @param bus the parent device of @p child * @param child the device which is requesting a release * @param type the type of resource to release * @param rid the resource identifier * @param res the resource to release * * @retval 0 success * @retval non-zero a standard unix error code indicating what * error condition prevented the operation */ int resource_list_release(struct resource_list *rl, device_t bus, device_t child, int type, int rid, struct resource *res) { struct resource_list_entry *rle = NULL; int passthrough = (device_get_parent(child) != bus); int error; if (passthrough) { return (BUS_RELEASE_RESOURCE(device_get_parent(bus), child, type, rid, res)); } rle = resource_list_find(rl, type, rid); if (!rle) panic("resource_list_release: can't find resource"); if (!rle->res) panic("resource_list_release: resource entry is not busy"); if (rle->flags & RLE_RESERVED) { if (rle->flags & RLE_ALLOCATED) { if (rman_get_flags(res) & RF_ACTIVE) { error = bus_deactivate_resource(child, type, rid, res); if (error) return (error); } rle->flags &= ~RLE_ALLOCATED; return (0); } return (EINVAL); } error = BUS_RELEASE_RESOURCE(device_get_parent(bus), child, type, rid, res); if (error) return (error); rle->res = NULL; return (0); } /** * @brief Release all active resources of a given type * * Release all active resources of a specified type. This is intended * to be used to cleanup resources leaked by a driver after detach or * a failed attach. * * @param rl the resource list which was allocated from * @param bus the parent device of @p child * @param child the device whose active resources are being released * @param type the type of resources to release * * @retval 0 success * @retval EBUSY at least one resource was active */ int resource_list_release_active(struct resource_list *rl, device_t bus, device_t child, int type) { struct resource_list_entry *rle; int error, retval; retval = 0; STAILQ_FOREACH(rle, rl, link) { if (rle->type != type) continue; if (rle->res == NULL) continue; if ((rle->flags & (RLE_RESERVED | RLE_ALLOCATED)) == RLE_RESERVED) continue; retval = EBUSY; error = resource_list_release(rl, bus, child, type, rman_get_rid(rle->res), rle->res); if (error != 0) device_printf(bus, "Failed to release active resource: %d\n", error); } return (retval); } /** * @brief Fully release a reserved resource * * Fully releases a resource reserved via resource_list_reserve(). * * @param rl the resource list which was allocated from * @param bus the parent device of @p child * @param child the device whose reserved resource is being released * @param type the type of resource to release * @param rid the resource identifier * @param res the resource to release * * @retval 0 success * @retval non-zero a standard unix error code indicating what * error condition prevented the operation */ int resource_list_unreserve(struct resource_list *rl, device_t bus, device_t child, int type, int rid) { struct resource_list_entry *rle = NULL; int passthrough = (device_get_parent(child) != bus); if (passthrough) panic( "resource_list_unreserve() should only be called for direct children"); rle = resource_list_find(rl, type, rid); if (!rle) panic("resource_list_unreserve: can't find resource"); if (!(rle->flags & RLE_RESERVED)) return (EINVAL); if (rle->flags & RLE_ALLOCATED) return (EBUSY); rle->flags &= ~RLE_RESERVED; return (resource_list_release(rl, bus, child, type, rid, rle->res)); } /** * @brief Print a description of resources in a resource list * * Print all resources of a specified type, for use in BUS_PRINT_CHILD(). * The name is printed if at least one resource of the given type is available. * The format is used to print resource start and end. * * @param rl the resource list to print * @param name the name of @p type, e.g. @c "memory" * @param type type type of resource entry to print * @param format printf(9) format string to print resource * start and end values * * @returns the number of characters printed */ int resource_list_print_type(struct resource_list *rl, const char *name, int type, const char *format) { struct resource_list_entry *rle; int printed, retval; printed = 0; retval = 0; /* Yes, this is kinda cheating */ STAILQ_FOREACH(rle, rl, link) { if (rle->type == type) { if (printed == 0) retval += printf(" %s ", name); else retval += printf(","); printed++; retval += printf(format, rle->start); if (rle->count > 1) { retval += printf("-"); retval += printf(format, rle->start + rle->count - 1); } } } return (retval); } /** * @brief Releases all the resources in a list. * * @param rl The resource list to purge. * * @returns nothing */ void resource_list_purge(struct resource_list *rl) { struct resource_list_entry *rle; while ((rle = STAILQ_FIRST(rl)) != NULL) { if (rle->res) bus_release_resource(rman_get_device(rle->res), rle->type, rle->rid, rle->res); STAILQ_REMOVE_HEAD(rl, link); free(rle, M_BUS); } } device_t bus_generic_add_child(device_t dev, u_int order, const char *name, int unit) { return (device_add_child_ordered(dev, order, name, unit)); } /** * @brief Helper function for implementing DEVICE_PROBE() * * This function can be used to help implement the DEVICE_PROBE() for * a bus (i.e. a device which has other devices attached to it). It * calls the DEVICE_IDENTIFY() method of each driver in the device's * devclass. */ int bus_generic_probe(device_t dev) { devclass_t dc = dev->devclass; driverlink_t dl; TAILQ_FOREACH(dl, &dc->drivers, link) { /* * If this driver's pass is too high, then ignore it. * For most drivers in the default pass, this will * never be true. For early-pass drivers they will * only call the identify routines of eligible drivers * when this routine is called. Drivers for later * passes should have their identify routines called * on early-pass busses during BUS_NEW_PASS(). */ if (dl->pass > bus_current_pass) continue; DEVICE_IDENTIFY(dl->driver, dev); } return (0); } /** * @brief Helper function for implementing DEVICE_ATTACH() * * This function can be used to help implement the DEVICE_ATTACH() for * a bus. It calls device_probe_and_attach() for each of the device's * children. */ int bus_generic_attach(device_t dev) { device_t child; TAILQ_FOREACH(child, &dev->children, link) { device_probe_and_attach(child); } return (0); } /** * @brief Helper function for implementing DEVICE_DETACH() * * This function can be used to help implement the DEVICE_DETACH() for * a bus. It calls device_detach() for each of the device's * children. */ int bus_generic_detach(device_t dev) { device_t child; int error; if (dev->state != DS_ATTACHED) return (EBUSY); TAILQ_FOREACH(child, &dev->children, link) { if ((error = device_detach(child)) != 0) return (error); } return (0); } /** * @brief Helper function for implementing DEVICE_SHUTDOWN() * * This function can be used to help implement the DEVICE_SHUTDOWN() * for a bus. It calls device_shutdown() for each of the device's * children. */ int bus_generic_shutdown(device_t dev) { device_t child; TAILQ_FOREACH(child, &dev->children, link) { device_shutdown(child); } return (0); } /** * @brief Default function for suspending a child device. * * This function is to be used by a bus's DEVICE_SUSPEND_CHILD(). */ int bus_generic_suspend_child(device_t dev, device_t child) { int error; error = DEVICE_SUSPEND(child); if (error == 0) child->flags |= DF_SUSPENDED; return (error); } /** * @brief Default function for resuming a child device. * * This function is to be used by a bus's DEVICE_RESUME_CHILD(). */ int bus_generic_resume_child(device_t dev, device_t child) { DEVICE_RESUME(child); child->flags &= ~DF_SUSPENDED; return (0); } /** * @brief Helper function for implementing DEVICE_SUSPEND() * * This function can be used to help implement the DEVICE_SUSPEND() * for a bus. It calls DEVICE_SUSPEND() for each of the device's * children. If any call to DEVICE_SUSPEND() fails, the suspend * operation is aborted and any devices which were suspended are * resumed immediately by calling their DEVICE_RESUME() methods. */ int bus_generic_suspend(device_t dev) { int error; device_t child, child2; TAILQ_FOREACH(child, &dev->children, link) { error = BUS_SUSPEND_CHILD(dev, child); if (error) { for (child2 = TAILQ_FIRST(&dev->children); child2 && child2 != child; child2 = TAILQ_NEXT(child2, link)) BUS_RESUME_CHILD(dev, child2); return (error); } } return (0); } /** * @brief Helper function for implementing DEVICE_RESUME() * * This function can be used to help implement the DEVICE_RESUME() for * a bus. It calls DEVICE_RESUME() on each of the device's children. */ int bus_generic_resume(device_t dev) { device_t child; TAILQ_FOREACH(child, &dev->children, link) { BUS_RESUME_CHILD(dev, child); /* if resume fails, there's nothing we can usefully do... */ } return (0); } /** * @brief Helper function for implementing BUS_PRINT_CHILD(). * * This function prints the first part of the ascii representation of * @p child, including its name, unit and description (if any - see * device_set_desc()). * * @returns the number of characters printed */ int bus_print_child_header(device_t dev, device_t child) { int retval = 0; if (device_get_desc(child)) { retval += device_printf(child, "<%s>", device_get_desc(child)); } else { retval += printf("%s", device_get_nameunit(child)); } return (retval); } /** * @brief Helper function for implementing BUS_PRINT_CHILD(). * * This function prints the last part of the ascii representation of * @p child, which consists of the string @c " on " followed by the * name and unit of the @p dev. * * @returns the number of characters printed */ int bus_print_child_footer(device_t dev, device_t child) { return (printf(" on %s\n", device_get_nameunit(dev))); } /** * @brief Helper function for implementing BUS_PRINT_CHILD(). * * This function prints out the VM domain for the given device. * * @returns the number of characters printed */ int bus_print_child_domain(device_t dev, device_t child) { int domain; /* No domain? Don't print anything */ if (BUS_GET_DOMAIN(dev, child, &domain) != 0) return (0); return (printf(" numa-domain %d", domain)); } /** * @brief Helper function for implementing BUS_PRINT_CHILD(). * * This function simply calls bus_print_child_header() followed by * bus_print_child_footer(). * * @returns the number of characters printed */ int bus_generic_print_child(device_t dev, device_t child) { int retval = 0; retval += bus_print_child_header(dev, child); retval += bus_print_child_domain(dev, child); retval += bus_print_child_footer(dev, child); return (retval); } /** * @brief Stub function for implementing BUS_READ_IVAR(). * * @returns ENOENT */ int bus_generic_read_ivar(device_t dev, device_t child, int index, uintptr_t * result) { return (ENOENT); } /** * @brief Stub function for implementing BUS_WRITE_IVAR(). * * @returns ENOENT */ int bus_generic_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { return (ENOENT); } /** * @brief Stub function for implementing BUS_GET_RESOURCE_LIST(). * * @returns NULL */ struct resource_list * bus_generic_get_resource_list(device_t dev, device_t child) { return (NULL); } /** * @brief Helper function for implementing BUS_DRIVER_ADDED(). * * This implementation of BUS_DRIVER_ADDED() simply calls the driver's * DEVICE_IDENTIFY() method to allow it to add new children to the bus * and then calls device_probe_and_attach() for each unattached child. */ void bus_generic_driver_added(device_t dev, driver_t *driver) { device_t child; DEVICE_IDENTIFY(driver, dev); TAILQ_FOREACH(child, &dev->children, link) { if (child->state == DS_NOTPRESENT || (child->flags & DF_REBID)) device_probe_and_attach(child); } } /** * @brief Helper function for implementing BUS_NEW_PASS(). * * This implementing of BUS_NEW_PASS() first calls the identify * routines for any drivers that probe at the current pass. Then it * walks the list of devices for this bus. If a device is already * attached, then it calls BUS_NEW_PASS() on that device. If the * device is not already attached, it attempts to attach a driver to * it. */ void bus_generic_new_pass(device_t dev) { driverlink_t dl; devclass_t dc; device_t child; dc = dev->devclass; TAILQ_FOREACH(dl, &dc->drivers, link) { if (dl->pass == bus_current_pass) DEVICE_IDENTIFY(dl->driver, dev); } TAILQ_FOREACH(child, &dev->children, link) { if (child->state >= DS_ATTACHED) BUS_NEW_PASS(child); else if (child->state == DS_NOTPRESENT) device_probe_and_attach(child); } } /** * @brief Helper function for implementing BUS_SETUP_INTR(). * * This simple implementation of BUS_SETUP_INTR() simply calls the * BUS_SETUP_INTR() method of the parent of @p dev. */ int bus_generic_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_SETUP_INTR(dev->parent, child, irq, flags, filter, intr, arg, cookiep)); return (EINVAL); } /** * @brief Helper function for implementing BUS_TEARDOWN_INTR(). * * This simple implementation of BUS_TEARDOWN_INTR() simply calls the * BUS_TEARDOWN_INTR() method of the parent of @p dev. */ int bus_generic_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_TEARDOWN_INTR(dev->parent, child, irq, cookie)); return (EINVAL); } /** * @brief Helper function for implementing BUS_ADJUST_RESOURCE(). * * This simple implementation of BUS_ADJUST_RESOURCE() simply calls the * BUS_ADJUST_RESOURCE() method of the parent of @p dev. */ int bus_generic_adjust_resource(device_t dev, device_t child, int type, struct resource *r, rman_res_t start, rman_res_t end) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_ADJUST_RESOURCE(dev->parent, child, type, r, start, end)); return (EINVAL); } /** * @brief Helper function for implementing BUS_ALLOC_RESOURCE(). * * This simple implementation of BUS_ALLOC_RESOURCE() simply calls the * BUS_ALLOC_RESOURCE() method of the parent of @p dev. */ struct resource * bus_generic_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_ALLOC_RESOURCE(dev->parent, child, type, rid, start, end, count, flags)); return (NULL); } /** * @brief Helper function for implementing BUS_RELEASE_RESOURCE(). * * This simple implementation of BUS_RELEASE_RESOURCE() simply calls the * BUS_RELEASE_RESOURCE() method of the parent of @p dev. */ int bus_generic_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_RELEASE_RESOURCE(dev->parent, child, type, rid, r)); return (EINVAL); } /** * @brief Helper function for implementing BUS_ACTIVATE_RESOURCE(). * * This simple implementation of BUS_ACTIVATE_RESOURCE() simply calls the * BUS_ACTIVATE_RESOURCE() method of the parent of @p dev. */ int bus_generic_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_ACTIVATE_RESOURCE(dev->parent, child, type, rid, r)); return (EINVAL); } /** * @brief Helper function for implementing BUS_DEACTIVATE_RESOURCE(). * * This simple implementation of BUS_DEACTIVATE_RESOURCE() simply calls the * BUS_DEACTIVATE_RESOURCE() method of the parent of @p dev. */ int bus_generic_deactivate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_DEACTIVATE_RESOURCE(dev->parent, child, type, rid, r)); return (EINVAL); } /** * @brief Helper function for implementing BUS_MAP_RESOURCE(). * * This simple implementation of BUS_MAP_RESOURCE() simply calls the * BUS_MAP_RESOURCE() method of the parent of @p dev. */ int bus_generic_map_resource(device_t dev, device_t child, int type, struct resource *r, struct resource_map_request *args, struct resource_map *map) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_MAP_RESOURCE(dev->parent, child, type, r, args, map)); return (EINVAL); } /** * @brief Helper function for implementing BUS_UNMAP_RESOURCE(). * * This simple implementation of BUS_UNMAP_RESOURCE() simply calls the * BUS_UNMAP_RESOURCE() method of the parent of @p dev. */ int bus_generic_unmap_resource(device_t dev, device_t child, int type, struct resource *r, struct resource_map *map) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_UNMAP_RESOURCE(dev->parent, child, type, r, map)); return (EINVAL); } /** * @brief Helper function for implementing BUS_BIND_INTR(). * * This simple implementation of BUS_BIND_INTR() simply calls the * BUS_BIND_INTR() method of the parent of @p dev. */ int bus_generic_bind_intr(device_t dev, device_t child, struct resource *irq, int cpu) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_BIND_INTR(dev->parent, child, irq, cpu)); return (EINVAL); } /** * @brief Helper function for implementing BUS_CONFIG_INTR(). * * This simple implementation of BUS_CONFIG_INTR() simply calls the * BUS_CONFIG_INTR() method of the parent of @p dev. */ int bus_generic_config_intr(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_CONFIG_INTR(dev->parent, irq, trig, pol)); return (EINVAL); } /** * @brief Helper function for implementing BUS_DESCRIBE_INTR(). * * This simple implementation of BUS_DESCRIBE_INTR() simply calls the * BUS_DESCRIBE_INTR() method of the parent of @p dev. */ int bus_generic_describe_intr(device_t dev, device_t child, struct resource *irq, void *cookie, const char *descr) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent) return (BUS_DESCRIBE_INTR(dev->parent, child, irq, cookie, descr)); return (EINVAL); } /** * @brief Helper function for implementing BUS_GET_CPUS(). * * This simple implementation of BUS_GET_CPUS() simply calls the * BUS_GET_CPUS() method of the parent of @p dev. */ int bus_generic_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsize, cpuset_t *cpuset) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent != NULL) return (BUS_GET_CPUS(dev->parent, child, op, setsize, cpuset)); return (EINVAL); } /** * @brief Helper function for implementing BUS_GET_DMA_TAG(). * * This simple implementation of BUS_GET_DMA_TAG() simply calls the * BUS_GET_DMA_TAG() method of the parent of @p dev. */ bus_dma_tag_t bus_generic_get_dma_tag(device_t dev, device_t child) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent != NULL) return (BUS_GET_DMA_TAG(dev->parent, child)); return (NULL); } /** * @brief Helper function for implementing BUS_GET_BUS_TAG(). * * This simple implementation of BUS_GET_BUS_TAG() simply calls the * BUS_GET_BUS_TAG() method of the parent of @p dev. */ bus_space_tag_t bus_generic_get_bus_tag(device_t dev, device_t child) { /* Propagate up the bus hierarchy until someone handles it. */ if (dev->parent != NULL) return (BUS_GET_BUS_TAG(dev->parent, child)); return ((bus_space_tag_t)0); } /** * @brief Helper function for implementing BUS_GET_RESOURCE(). * * This implementation of BUS_GET_RESOURCE() uses the * resource_list_find() function to do most of the work. It calls * BUS_GET_RESOURCE_LIST() to find a suitable resource list to * search. */ int bus_generic_rl_get_resource(device_t dev, device_t child, int type, int rid, rman_res_t *startp, rman_res_t *countp) { struct resource_list * rl = NULL; struct resource_list_entry * rle = NULL; rl = BUS_GET_RESOURCE_LIST(dev, child); if (!rl) return (EINVAL); rle = resource_list_find(rl, type, rid); if (!rle) return (ENOENT); if (startp) *startp = rle->start; if (countp) *countp = rle->count; return (0); } /** * @brief Helper function for implementing BUS_SET_RESOURCE(). * * This implementation of BUS_SET_RESOURCE() uses the * resource_list_add() function to do most of the work. It calls * BUS_GET_RESOURCE_LIST() to find a suitable resource list to * edit. */ int bus_generic_rl_set_resource(device_t dev, device_t child, int type, int rid, rman_res_t start, rman_res_t count) { struct resource_list * rl = NULL; rl = BUS_GET_RESOURCE_LIST(dev, child); if (!rl) return (EINVAL); resource_list_add(rl, type, rid, start, (start + count - 1), count); return (0); } /** * @brief Helper function for implementing BUS_DELETE_RESOURCE(). * * This implementation of BUS_DELETE_RESOURCE() uses the * resource_list_delete() function to do most of the work. It calls * BUS_GET_RESOURCE_LIST() to find a suitable resource list to * edit. */ void bus_generic_rl_delete_resource(device_t dev, device_t child, int type, int rid) { struct resource_list * rl = NULL; rl = BUS_GET_RESOURCE_LIST(dev, child); if (!rl) return; resource_list_delete(rl, type, rid); return; } /** * @brief Helper function for implementing BUS_RELEASE_RESOURCE(). * * This implementation of BUS_RELEASE_RESOURCE() uses the * resource_list_release() function to do most of the work. It calls * BUS_GET_RESOURCE_LIST() to find a suitable resource list. */ int bus_generic_rl_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct resource_list * rl = NULL; if (device_get_parent(child) != dev) return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child, type, rid, r)); rl = BUS_GET_RESOURCE_LIST(dev, child); if (!rl) return (EINVAL); return (resource_list_release(rl, dev, child, type, rid, r)); } /** * @brief Helper function for implementing BUS_ALLOC_RESOURCE(). * * This implementation of BUS_ALLOC_RESOURCE() uses the * resource_list_alloc() function to do most of the work. It calls * BUS_GET_RESOURCE_LIST() to find a suitable resource list. */ struct resource * bus_generic_rl_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource_list * rl = NULL; if (device_get_parent(child) != dev) return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child, type, rid, start, end, count, flags)); rl = BUS_GET_RESOURCE_LIST(dev, child); if (!rl) return (NULL); return (resource_list_alloc(rl, dev, child, type, rid, start, end, count, flags)); } /** * @brief Helper function for implementing BUS_CHILD_PRESENT(). * * This simple implementation of BUS_CHILD_PRESENT() simply calls the * BUS_CHILD_PRESENT() method of the parent of @p dev. */ int bus_generic_child_present(device_t dev, device_t child) { return (BUS_CHILD_PRESENT(device_get_parent(dev), dev)); } int bus_generic_get_domain(device_t dev, device_t child, int *domain) { if (dev->parent) return (BUS_GET_DOMAIN(dev->parent, dev, domain)); return (ENOENT); } /** * @brief Helper function for implementing BUS_RESCAN(). * * This null implementation of BUS_RESCAN() always fails to indicate * the bus does not support rescanning. */ int bus_null_rescan(device_t dev) { return (ENXIO); } /* * Some convenience functions to make it easier for drivers to use the * resource-management functions. All these really do is hide the * indirection through the parent's method table, making for slightly * less-wordy code. In the future, it might make sense for this code * to maintain some sort of a list of resources allocated by each device. */ int bus_alloc_resources(device_t dev, struct resource_spec *rs, struct resource **res) { int i; for (i = 0; rs[i].type != -1; i++) res[i] = NULL; for (i = 0; rs[i].type != -1; i++) { res[i] = bus_alloc_resource_any(dev, rs[i].type, &rs[i].rid, rs[i].flags); if (res[i] == NULL && !(rs[i].flags & RF_OPTIONAL)) { bus_release_resources(dev, rs, res); return (ENXIO); } } return (0); } void bus_release_resources(device_t dev, const struct resource_spec *rs, struct resource **res) { int i; for (i = 0; rs[i].type != -1; i++) if (res[i] != NULL) { bus_release_resource( dev, rs[i].type, rs[i].rid, res[i]); res[i] = NULL; } } /** * @brief Wrapper function for BUS_ALLOC_RESOURCE(). * * This function simply calls the BUS_ALLOC_RESOURCE() method of the * parent of @p dev. */ struct resource * bus_alloc_resource(device_t dev, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res; if (dev->parent == NULL) return (NULL); res = BUS_ALLOC_RESOURCE(dev->parent, dev, type, rid, start, end, count, flags); return (res); } /** * @brief Wrapper function for BUS_ADJUST_RESOURCE(). * * This function simply calls the BUS_ADJUST_RESOURCE() method of the * parent of @p dev. */ int bus_adjust_resource(device_t dev, int type, struct resource *r, rman_res_t start, rman_res_t end) { if (dev->parent == NULL) return (EINVAL); return (BUS_ADJUST_RESOURCE(dev->parent, dev, type, r, start, end)); } /** * @brief Wrapper function for BUS_ACTIVATE_RESOURCE(). * * This function simply calls the BUS_ACTIVATE_RESOURCE() method of the * parent of @p dev. */ int bus_activate_resource(device_t dev, int type, int rid, struct resource *r) { if (dev->parent == NULL) return (EINVAL); return (BUS_ACTIVATE_RESOURCE(dev->parent, dev, type, rid, r)); } /** * @brief Wrapper function for BUS_DEACTIVATE_RESOURCE(). * * This function simply calls the BUS_DEACTIVATE_RESOURCE() method of the * parent of @p dev. */ int bus_deactivate_resource(device_t dev, int type, int rid, struct resource *r) { if (dev->parent == NULL) return (EINVAL); return (BUS_DEACTIVATE_RESOURCE(dev->parent, dev, type, rid, r)); } /** * @brief Wrapper function for BUS_MAP_RESOURCE(). * * This function simply calls the BUS_MAP_RESOURCE() method of the * parent of @p dev. */ int bus_map_resource(device_t dev, int type, struct resource *r, struct resource_map_request *args, struct resource_map *map) { if (dev->parent == NULL) return (EINVAL); return (BUS_MAP_RESOURCE(dev->parent, dev, type, r, args, map)); } /** * @brief Wrapper function for BUS_UNMAP_RESOURCE(). * * This function simply calls the BUS_UNMAP_RESOURCE() method of the * parent of @p dev. */ int bus_unmap_resource(device_t dev, int type, struct resource *r, struct resource_map *map) { if (dev->parent == NULL) return (EINVAL); return (BUS_UNMAP_RESOURCE(dev->parent, dev, type, r, map)); } /** * @brief Wrapper function for BUS_RELEASE_RESOURCE(). * * This function simply calls the BUS_RELEASE_RESOURCE() method of the * parent of @p dev. */ int bus_release_resource(device_t dev, int type, int rid, struct resource *r) { int rv; if (dev->parent == NULL) return (EINVAL); rv = BUS_RELEASE_RESOURCE(dev->parent, dev, type, rid, r); return (rv); } /** * @brief Wrapper function for BUS_SETUP_INTR(). * * This function simply calls the BUS_SETUP_INTR() method of the * parent of @p dev. */ int bus_setup_intr(device_t dev, struct resource *r, int flags, driver_filter_t filter, driver_intr_t handler, void *arg, void **cookiep) { int error; if (dev->parent == NULL) return (EINVAL); error = BUS_SETUP_INTR(dev->parent, dev, r, flags, filter, handler, arg, cookiep); if (error != 0) return (error); if (handler != NULL && !(flags & INTR_MPSAFE)) device_printf(dev, "[GIANT-LOCKED]\n"); return (0); } /** * @brief Wrapper function for BUS_TEARDOWN_INTR(). * * This function simply calls the BUS_TEARDOWN_INTR() method of the * parent of @p dev. */ int bus_teardown_intr(device_t dev, struct resource *r, void *cookie) { if (dev->parent == NULL) return (EINVAL); return (BUS_TEARDOWN_INTR(dev->parent, dev, r, cookie)); } /** * @brief Wrapper function for BUS_BIND_INTR(). * * This function simply calls the BUS_BIND_INTR() method of the * parent of @p dev. */ int bus_bind_intr(device_t dev, struct resource *r, int cpu) { if (dev->parent == NULL) return (EINVAL); return (BUS_BIND_INTR(dev->parent, dev, r, cpu)); } /** * @brief Wrapper function for BUS_DESCRIBE_INTR(). * * This function first formats the requested description into a * temporary buffer and then calls the BUS_DESCRIBE_INTR() method of * the parent of @p dev. */ int bus_describe_intr(device_t dev, struct resource *irq, void *cookie, const char *fmt, ...) { va_list ap; char descr[MAXCOMLEN + 1]; if (dev->parent == NULL) return (EINVAL); va_start(ap, fmt); vsnprintf(descr, sizeof(descr), fmt, ap); va_end(ap); return (BUS_DESCRIBE_INTR(dev->parent, dev, irq, cookie, descr)); } /** * @brief Wrapper function for BUS_SET_RESOURCE(). * * This function simply calls the BUS_SET_RESOURCE() method of the * parent of @p dev. */ int bus_set_resource(device_t dev, int type, int rid, rman_res_t start, rman_res_t count) { return (BUS_SET_RESOURCE(device_get_parent(dev), dev, type, rid, start, count)); } /** * @brief Wrapper function for BUS_GET_RESOURCE(). * * This function simply calls the BUS_GET_RESOURCE() method of the * parent of @p dev. */ int bus_get_resource(device_t dev, int type, int rid, rman_res_t *startp, rman_res_t *countp) { return (BUS_GET_RESOURCE(device_get_parent(dev), dev, type, rid, startp, countp)); } /** * @brief Wrapper function for BUS_GET_RESOURCE(). * * This function simply calls the BUS_GET_RESOURCE() method of the * parent of @p dev and returns the start value. */ rman_res_t bus_get_resource_start(device_t dev, int type, int rid) { rman_res_t start; rman_res_t count; int error; error = BUS_GET_RESOURCE(device_get_parent(dev), dev, type, rid, &start, &count); if (error) return (0); return (start); } /** * @brief Wrapper function for BUS_GET_RESOURCE(). * * This function simply calls the BUS_GET_RESOURCE() method of the * parent of @p dev and returns the count value. */ rman_res_t bus_get_resource_count(device_t dev, int type, int rid) { rman_res_t start; rman_res_t count; int error; error = BUS_GET_RESOURCE(device_get_parent(dev), dev, type, rid, &start, &count); if (error) return (0); return (count); } /** * @brief Wrapper function for BUS_DELETE_RESOURCE(). * * This function simply calls the BUS_DELETE_RESOURCE() method of the * parent of @p dev. */ void bus_delete_resource(device_t dev, int type, int rid) { BUS_DELETE_RESOURCE(device_get_parent(dev), dev, type, rid); } /** * @brief Wrapper function for BUS_CHILD_PRESENT(). * * This function simply calls the BUS_CHILD_PRESENT() method of the * parent of @p dev. */ int bus_child_present(device_t child) { return (BUS_CHILD_PRESENT(device_get_parent(child), child)); } /** * @brief Wrapper function for BUS_CHILD_PNPINFO_STR(). * * This function simply calls the BUS_CHILD_PNPINFO_STR() method of the * parent of @p dev. */ int bus_child_pnpinfo_str(device_t child, char *buf, size_t buflen) { device_t parent; parent = device_get_parent(child); if (parent == NULL) { *buf = '\0'; return (0); } return (BUS_CHILD_PNPINFO_STR(parent, child, buf, buflen)); } /** * @brief Wrapper function for BUS_CHILD_LOCATION_STR(). * * This function simply calls the BUS_CHILD_LOCATION_STR() method of the * parent of @p dev. */ int bus_child_location_str(device_t child, char *buf, size_t buflen) { device_t parent; parent = device_get_parent(child); if (parent == NULL) { *buf = '\0'; return (0); } return (BUS_CHILD_LOCATION_STR(parent, child, buf, buflen)); } /** * @brief Wrapper function for BUS_GET_CPUS(). * * This function simply calls the BUS_GET_CPUS() method of the * parent of @p dev. */ int bus_get_cpus(device_t dev, enum cpu_sets op, size_t setsize, cpuset_t *cpuset) { device_t parent; parent = device_get_parent(dev); if (parent == NULL) return (EINVAL); return (BUS_GET_CPUS(parent, dev, op, setsize, cpuset)); } /** * @brief Wrapper function for BUS_GET_DMA_TAG(). * * This function simply calls the BUS_GET_DMA_TAG() method of the * parent of @p dev. */ bus_dma_tag_t bus_get_dma_tag(device_t dev) { device_t parent; parent = device_get_parent(dev); if (parent == NULL) return (NULL); return (BUS_GET_DMA_TAG(parent, dev)); } /** * @brief Wrapper function for BUS_GET_BUS_TAG(). * * This function simply calls the BUS_GET_BUS_TAG() method of the * parent of @p dev. */ bus_space_tag_t bus_get_bus_tag(device_t dev) { device_t parent; parent = device_get_parent(dev); if (parent == NULL) return ((bus_space_tag_t)0); return (BUS_GET_BUS_TAG(parent, dev)); } /** * @brief Wrapper function for BUS_GET_DOMAIN(). * * This function simply calls the BUS_GET_DOMAIN() method of the * parent of @p dev. */ int bus_get_domain(device_t dev, int *domain) { return (BUS_GET_DOMAIN(device_get_parent(dev), dev, domain)); } /* Resume all devices and then notify userland that we're up again. */ static int root_resume(device_t dev) { int error; error = bus_generic_resume(dev); if (error == 0) devctl_notify("kern", "power", "resume", NULL); return (error); } static int root_print_child(device_t dev, device_t child) { int retval = 0; retval += bus_print_child_header(dev, child); retval += printf("\n"); return (retval); } static int root_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep) { /* * If an interrupt mapping gets to here something bad has happened. */ panic("root_setup_intr"); } /* * If we get here, assume that the device is permanent and really is * present in the system. Removable bus drivers are expected to intercept * this call long before it gets here. We return -1 so that drivers that * really care can check vs -1 or some ERRNO returned higher in the food * chain. */ static int root_child_present(device_t dev, device_t child) { return (-1); } static int root_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsize, cpuset_t *cpuset) { switch (op) { case INTR_CPUS: /* Default to returning the set of all CPUs. */ if (setsize != sizeof(cpuset_t)) return (EINVAL); *cpuset = all_cpus; return (0); default: return (EINVAL); } } static kobj_method_t root_methods[] = { /* Device interface */ KOBJMETHOD(device_shutdown, bus_generic_shutdown), KOBJMETHOD(device_suspend, bus_generic_suspend), KOBJMETHOD(device_resume, root_resume), /* Bus interface */ KOBJMETHOD(bus_print_child, root_print_child), KOBJMETHOD(bus_read_ivar, bus_generic_read_ivar), KOBJMETHOD(bus_write_ivar, bus_generic_write_ivar), KOBJMETHOD(bus_setup_intr, root_setup_intr), KOBJMETHOD(bus_child_present, root_child_present), KOBJMETHOD(bus_get_cpus, root_get_cpus), KOBJMETHOD_END }; static driver_t root_driver = { "root", root_methods, 1, /* no softc */ }; device_t root_bus; devclass_t root_devclass; static int root_bus_module_handler(module_t mod, int what, void* arg) { switch (what) { case MOD_LOAD: TAILQ_INIT(&bus_data_devices); kobj_class_compile((kobj_class_t) &root_driver); root_bus = make_device(NULL, "root", 0); root_bus->desc = "System root bus"; kobj_init((kobj_t) root_bus, (kobj_class_t) &root_driver); root_bus->driver = &root_driver; root_bus->state = DS_ATTACHED; root_devclass = devclass_find_internal("root", NULL, FALSE); devinit(); return (0); case MOD_SHUTDOWN: device_shutdown(root_bus); return (0); default: return (EOPNOTSUPP); } return (0); } static moduledata_t root_bus_mod = { "rootbus", root_bus_module_handler, NULL }; DECLARE_MODULE(rootbus, root_bus_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); /** * @brief Automatically configure devices * * This function begins the autoconfiguration process by calling * device_probe_and_attach() for each child of the @c root0 device. */ void root_bus_configure(void) { PDEBUG((".")); /* Eventually this will be split up, but this is sufficient for now. */ bus_set_pass(BUS_PASS_DEFAULT); } /** * @brief Module handler for registering device drivers * * This module handler is used to automatically register device * drivers when modules are loaded. If @p what is MOD_LOAD, it calls * devclass_add_driver() for the driver described by the * driver_module_data structure pointed to by @p arg */ int driver_module_handler(module_t mod, int what, void *arg) { struct driver_module_data *dmd; devclass_t bus_devclass; kobj_class_t driver; int error, pass; dmd = (struct driver_module_data *)arg; bus_devclass = devclass_find_internal(dmd->dmd_busname, NULL, TRUE); error = 0; switch (what) { case MOD_LOAD: if (dmd->dmd_chainevh) error = dmd->dmd_chainevh(mod,what,dmd->dmd_chainarg); pass = dmd->dmd_pass; driver = dmd->dmd_driver; PDEBUG(("Loading module: driver %s on bus %s (pass %d)", DRIVERNAME(driver), dmd->dmd_busname, pass)); error = devclass_add_driver(bus_devclass, driver, pass, dmd->dmd_devclass); break; case MOD_UNLOAD: PDEBUG(("Unloading module: driver %s from bus %s", DRIVERNAME(dmd->dmd_driver), dmd->dmd_busname)); error = devclass_delete_driver(bus_devclass, dmd->dmd_driver); if (!error && dmd->dmd_chainevh) error = dmd->dmd_chainevh(mod,what,dmd->dmd_chainarg); break; case MOD_QUIESCE: PDEBUG(("Quiesce module: driver %s from bus %s", DRIVERNAME(dmd->dmd_driver), dmd->dmd_busname)); error = devclass_quiesce_driver(bus_devclass, dmd->dmd_driver); if (!error && dmd->dmd_chainevh) error = dmd->dmd_chainevh(mod,what,dmd->dmd_chainarg); break; default: error = EOPNOTSUPP; break; } return (error); } /** * @brief Enumerate all hinted devices for this bus. * * Walks through the hints for this bus and calls the bus_hinted_child * routine for each one it fines. It searches first for the specific * bus that's being probed for hinted children (eg isa0), and then for * generic children (eg isa). * * @param dev bus device to enumerate */ void bus_enumerate_hinted_children(device_t bus) { int i; const char *dname, *busname; int dunit; /* * enumerate all devices on the specific bus */ busname = device_get_nameunit(bus); i = 0; while (resource_find_match(&i, &dname, &dunit, "at", busname) == 0) BUS_HINTED_CHILD(bus, dname, dunit); /* * and all the generic ones. */ busname = device_get_name(bus); i = 0; while (resource_find_match(&i, &dname, &dunit, "at", busname) == 0) BUS_HINTED_CHILD(bus, dname, dunit); } #ifdef BUS_DEBUG /* the _short versions avoid iteration by not calling anything that prints * more than oneliners. I love oneliners. */ static void print_device_short(device_t dev, int indent) { if (!dev) return; indentprintf(("device %d: <%s> %sparent,%schildren,%s%s%s%s%s,%sivars,%ssoftc,busy=%d\n", dev->unit, dev->desc, (dev->parent? "":"no "), (TAILQ_EMPTY(&dev->children)? "no ":""), (dev->flags&DF_ENABLED? "enabled,":"disabled,"), (dev->flags&DF_FIXEDCLASS? "fixed,":""), (dev->flags&DF_WILDCARD? "wildcard,":""), (dev->flags&DF_DESCMALLOCED? "descmalloced,":""), (dev->flags&DF_REBID? "rebiddable,":""), (dev->ivars? "":"no "), (dev->softc? "":"no "), dev->busy)); } static void print_device(device_t dev, int indent) { if (!dev) return; print_device_short(dev, indent); indentprintf(("Parent:\n")); print_device_short(dev->parent, indent+1); indentprintf(("Driver:\n")); print_driver_short(dev->driver, indent+1); indentprintf(("Devclass:\n")); print_devclass_short(dev->devclass, indent+1); } void print_device_tree_short(device_t dev, int indent) /* print the device and all its children (indented) */ { device_t child; if (!dev) return; print_device_short(dev, indent); TAILQ_FOREACH(child, &dev->children, link) { print_device_tree_short(child, indent+1); } } void print_device_tree(device_t dev, int indent) /* print the device and all its children (indented) */ { device_t child; if (!dev) return; print_device(dev, indent); TAILQ_FOREACH(child, &dev->children, link) { print_device_tree(child, indent+1); } } static void print_driver_short(driver_t *driver, int indent) { if (!driver) return; indentprintf(("driver %s: softc size = %zd\n", driver->name, driver->size)); } static void print_driver(driver_t *driver, int indent) { if (!driver) return; print_driver_short(driver, indent); } static void print_driver_list(driver_list_t drivers, int indent) { driverlink_t driver; TAILQ_FOREACH(driver, &drivers, link) { print_driver(driver->driver, indent); } } static void print_devclass_short(devclass_t dc, int indent) { if ( !dc ) return; indentprintf(("devclass %s: max units = %d\n", dc->name, dc->maxunit)); } static void print_devclass(devclass_t dc, int indent) { int i; if ( !dc ) return; print_devclass_short(dc, indent); indentprintf(("Drivers:\n")); print_driver_list(dc->drivers, indent+1); indentprintf(("Devices:\n")); for (i = 0; i < dc->maxunit; i++) if (dc->devices[i]) print_device(dc->devices[i], indent+1); } void print_devclass_list_short(void) { devclass_t dc; printf("Short listing of devclasses, drivers & devices:\n"); TAILQ_FOREACH(dc, &devclasses, link) { print_devclass_short(dc, 0); } } void print_devclass_list(void) { devclass_t dc; printf("Full listing of devclasses, drivers & devices:\n"); TAILQ_FOREACH(dc, &devclasses, link) { print_devclass(dc, 0); } } #endif /* * User-space access to the device tree. * * We implement a small set of nodes: * * hw.bus Single integer read method to obtain the * current generation count. * hw.bus.devices Reads the entire device tree in flat space. * hw.bus.rman Resource manager interface * * We might like to add the ability to scan devclasses and/or drivers to * determine what else is currently loaded/available. */ static int sysctl_bus(SYSCTL_HANDLER_ARGS) { struct u_businfo ubus; ubus.ub_version = BUS_USER_VERSION; ubus.ub_generation = bus_data_generation; return (SYSCTL_OUT(req, &ubus, sizeof(ubus))); } SYSCTL_NODE(_hw_bus, OID_AUTO, info, CTLFLAG_RW, sysctl_bus, "bus-related data"); static int sysctl_devices(SYSCTL_HANDLER_ARGS) { int *name = (int *)arg1; u_int namelen = arg2; int index; device_t dev; struct u_device udev; /* XXX this is a bit big */ int error; if (namelen != 2) return (EINVAL); if (bus_data_generation_check(name[0])) return (EINVAL); index = name[1]; /* * Scan the list of devices, looking for the requested index. */ TAILQ_FOREACH(dev, &bus_data_devices, devlink) { if (index-- == 0) break; } if (dev == NULL) return (ENOENT); /* * Populate the return array. */ bzero(&udev, sizeof(udev)); udev.dv_handle = (uintptr_t)dev; udev.dv_parent = (uintptr_t)dev->parent; if (dev->nameunit != NULL) strlcpy(udev.dv_name, dev->nameunit, sizeof(udev.dv_name)); if (dev->desc != NULL) strlcpy(udev.dv_desc, dev->desc, sizeof(udev.dv_desc)); if (dev->driver != NULL && dev->driver->name != NULL) strlcpy(udev.dv_drivername, dev->driver->name, sizeof(udev.dv_drivername)); bus_child_pnpinfo_str(dev, udev.dv_pnpinfo, sizeof(udev.dv_pnpinfo)); bus_child_location_str(dev, udev.dv_location, sizeof(udev.dv_location)); udev.dv_devflags = dev->devflags; udev.dv_flags = dev->flags; udev.dv_state = dev->state; error = SYSCTL_OUT(req, &udev, sizeof(udev)); return (error); } SYSCTL_NODE(_hw_bus, OID_AUTO, devices, CTLFLAG_RD, sysctl_devices, "system device tree"); int bus_data_generation_check(int generation) { if (generation != bus_data_generation) return (1); /* XXX generate optimised lists here? */ return (0); } void bus_data_generation_update(void) { bus_data_generation++; } int bus_free_resource(device_t dev, int type, struct resource *r) { if (r == NULL) return (0); return (bus_release_resource(dev, type, rman_get_rid(r), r)); } device_t device_lookup_by_name(const char *name) { device_t dev; TAILQ_FOREACH(dev, &bus_data_devices, devlink) { if (dev->nameunit != NULL && strcmp(dev->nameunit, name) == 0) return (dev); } return (NULL); } /* * /dev/devctl2 implementation. The existing /dev/devctl device has * implicit semantics on open, so it could not be reused for this. * Another option would be to call this /dev/bus? */ static int find_device(struct devreq *req, device_t *devp) { device_t dev; /* * First, ensure that the name is nul terminated. */ if (memchr(req->dr_name, '\0', sizeof(req->dr_name)) == NULL) return (EINVAL); /* * Second, try to find an attached device whose name matches * 'name'. */ dev = device_lookup_by_name(req->dr_name); if (dev != NULL) { *devp = dev; return (0); } /* Finally, give device enumerators a chance. */ dev = NULL; EVENTHANDLER_INVOKE(dev_lookup, req->dr_name, &dev); if (dev == NULL) return (ENOENT); *devp = dev; return (0); } static bool driver_exists(device_t bus, const char *driver) { devclass_t dc; for (dc = bus->devclass; dc != NULL; dc = dc->parent) { if (devclass_find_driver_internal(dc, driver) != NULL) return (true); } return (false); } static int devctl2_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag, struct thread *td) { struct devreq *req; device_t dev; int error, old; /* Locate the device to control. */ mtx_lock(&Giant); req = (struct devreq *)data; switch (cmd) { case DEV_ATTACH: case DEV_DETACH: case DEV_ENABLE: case DEV_DISABLE: case DEV_SUSPEND: case DEV_RESUME: case DEV_SET_DRIVER: + case DEV_CLEAR_DRIVER: case DEV_RESCAN: case DEV_DELETE: error = priv_check(td, PRIV_DRIVER); if (error == 0) error = find_device(req, &dev); break; default: error = ENOTTY; break; } if (error) { mtx_unlock(&Giant); return (error); } /* Perform the requested operation. */ switch (cmd) { case DEV_ATTACH: if (device_is_attached(dev) && (dev->flags & DF_REBID) == 0) error = EBUSY; else if (!device_is_enabled(dev)) error = ENXIO; else error = device_probe_and_attach(dev); break; case DEV_DETACH: if (!device_is_attached(dev)) { error = ENXIO; break; } if (!(req->dr_flags & DEVF_FORCE_DETACH)) { error = device_quiesce(dev); if (error) break; } error = device_detach(dev); break; case DEV_ENABLE: if (device_is_enabled(dev)) { error = EBUSY; break; } /* * If the device has been probed but not attached (e.g. * when it has been disabled by a loader hint), just * attach the device rather than doing a full probe. */ device_enable(dev); if (device_is_alive(dev)) { /* * If the device was disabled via a hint, clear * the hint. */ if (resource_disabled(dev->driver->name, dev->unit)) resource_unset_value(dev->driver->name, dev->unit, "disabled"); error = device_attach(dev); } else error = device_probe_and_attach(dev); break; case DEV_DISABLE: if (!device_is_enabled(dev)) { error = ENXIO; break; } if (!(req->dr_flags & DEVF_FORCE_DETACH)) { error = device_quiesce(dev); if (error) break; } /* * Force DF_FIXEDCLASS on around detach to preserve * the existing name. */ old = dev->flags; dev->flags |= DF_FIXEDCLASS; error = device_detach(dev); if (!(old & DF_FIXEDCLASS)) dev->flags &= ~DF_FIXEDCLASS; if (error == 0) device_disable(dev); break; case DEV_SUSPEND: if (device_is_suspended(dev)) { error = EBUSY; break; } if (device_get_parent(dev) == NULL) { error = EINVAL; break; } error = BUS_SUSPEND_CHILD(device_get_parent(dev), dev); break; case DEV_RESUME: if (!device_is_suspended(dev)) { error = EINVAL; break; } if (device_get_parent(dev) == NULL) { error = EINVAL; break; } error = BUS_RESUME_CHILD(device_get_parent(dev), dev); break; case DEV_SET_DRIVER: { devclass_t dc; char driver[128]; error = copyinstr(req->dr_data, driver, sizeof(driver), NULL); if (error) break; if (driver[0] == '\0') { error = EINVAL; break; } if (dev->devclass != NULL && strcmp(driver, dev->devclass->name) == 0) /* XXX: Could possibly force DF_FIXEDCLASS on? */ break; /* * Scan drivers for this device's bus looking for at * least one matching driver. */ if (dev->parent == NULL) { error = EINVAL; break; } if (!driver_exists(dev->parent, driver)) { error = ENOENT; break; } dc = devclass_create(driver); if (dc == NULL) { error = ENOMEM; break; } /* Detach device if necessary. */ if (device_is_attached(dev)) { if (req->dr_flags & DEVF_SET_DRIVER_DETACH) error = device_detach(dev); else error = EBUSY; if (error) break; } /* Clear any previously-fixed device class and unit. */ if (dev->flags & DF_FIXEDCLASS) devclass_delete_device(dev->devclass, dev); dev->flags |= DF_WILDCARD; dev->unit = -1; /* Force the new device class. */ error = devclass_add_device(dc, dev); if (error) break; dev->flags |= DF_FIXEDCLASS; error = device_probe_and_attach(dev); break; } + case DEV_CLEAR_DRIVER: + if (!(dev->flags & DF_FIXEDCLASS)) { + error = 0; + break; + } + if (device_is_attached(dev)) { + if (req->dr_flags & DEVF_CLEAR_DRIVER_DETACH) + error = device_detach(dev); + else + error = EBUSY; + if (error) + break; + } + + dev->flags &= ~DF_FIXEDCLASS; + dev->flags |= DF_WILDCARD; + devclass_delete_device(dev->devclass, dev); + error = device_probe_and_attach(dev); + break; case DEV_RESCAN: if (!device_is_attached(dev)) { error = ENXIO; break; } error = BUS_RESCAN(dev); break; case DEV_DELETE: { device_t parent; parent = device_get_parent(dev); if (parent == NULL) { error = EINVAL; break; } if (!(req->dr_flags & DEVF_FORCE_DELETE)) { if (bus_child_present(dev) != 0) { error = EBUSY; break; } } error = device_delete_child(parent, dev); break; } } mtx_unlock(&Giant); return (error); } static struct cdevsw devctl2_cdevsw = { .d_version = D_VERSION, .d_ioctl = devctl2_ioctl, .d_name = "devctl2", }; static void devctl2_init(void) { make_dev_credf(MAKEDEV_ETERNAL, &devctl2_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600, "devctl2"); } Index: projects/clang390-import/sys/kern/subr_smp.c =================================================================== --- projects/clang390-import/sys/kern/subr_smp.c (revision 305080) +++ projects/clang390-import/sys/kern/subr_smp.c (revision 305081) @@ -1,1140 +1,1133 @@ /*- * Copyright (c) 2001, John Baldwin . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This module holds the global variables and machine independent functions * used for the kernel SMP support. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "opt_sched.h" #ifdef SMP MALLOC_DEFINE(M_TOPO, "toponodes", "SMP topology data"); volatile cpuset_t stopped_cpus; volatile cpuset_t started_cpus; volatile cpuset_t suspended_cpus; cpuset_t hlt_cpus_mask; cpuset_t logical_cpus_mask; void (*cpustop_restartfunc)(void); #endif static int sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS); /* This is used in modules that need to work in both SMP and UP. */ cpuset_t all_cpus; int mp_ncpus; /* export this for libkvm consumers. */ int mp_maxcpus = MAXCPU; volatile int smp_started; u_int mp_maxid; static SYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD|CTLFLAG_CAPRD, NULL, "Kernel SMP"); SYSCTL_INT(_kern_smp, OID_AUTO, maxid, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_maxid, 0, "Max CPU ID."); SYSCTL_INT(_kern_smp, OID_AUTO, maxcpus, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_maxcpus, 0, "Max number of CPUs that the system was compiled for."); SYSCTL_PROC(_kern_smp, OID_AUTO, active, CTLFLAG_RD | CTLTYPE_INT, NULL, 0, sysctl_kern_smp_active, "I", "Indicates system is running in SMP mode"); int smp_disabled = 0; /* has smp been disabled? */ SYSCTL_INT(_kern_smp, OID_AUTO, disabled, CTLFLAG_RDTUN|CTLFLAG_CAPRD, &smp_disabled, 0, "SMP has been disabled from the loader"); int smp_cpus = 1; /* how many cpu's running */ SYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD|CTLFLAG_CAPRD, &smp_cpus, 0, "Number of CPUs online"); int smp_topology = 0; /* Which topology we're using. */ SYSCTL_INT(_kern_smp, OID_AUTO, topology, CTLFLAG_RDTUN, &smp_topology, 0, "Topology override setting; 0 is default provided by hardware."); #ifdef SMP /* Enable forwarding of a signal to a process running on a different CPU */ static int forward_signal_enabled = 1; SYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, &forward_signal_enabled, 0, "Forwarding of a signal to a process on a different CPU"); /* Variables needed for SMP rendezvous. */ static volatile int smp_rv_ncpus; static void (*volatile smp_rv_setup_func)(void *arg); static void (*volatile smp_rv_action_func)(void *arg); static void (*volatile smp_rv_teardown_func)(void *arg); static void *volatile smp_rv_func_arg; static volatile int smp_rv_waiters[4]; /* * Shared mutex to restrict busywaits between smp_rendezvous() and * smp(_targeted)_tlb_shootdown(). A deadlock occurs if both of these * functions trigger at once and cause multiple CPUs to busywait with * interrupts disabled. */ struct mtx smp_ipi_mtx; /* * Let the MD SMP code initialize mp_maxid very early if it can. */ static void mp_setmaxid(void *dummy) { cpu_mp_setmaxid(); KASSERT(mp_ncpus >= 1, ("%s: CPU count < 1", __func__)); KASSERT(mp_ncpus > 1 || mp_maxid == 0, ("%s: one CPU but mp_maxid is not zero", __func__)); KASSERT(mp_maxid >= mp_ncpus - 1, ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid, mp_ncpus)); } SYSINIT(cpu_mp_setmaxid, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_setmaxid, NULL); /* * Call the MD SMP initialization code. */ static void mp_start(void *dummy) { mtx_init(&smp_ipi_mtx, "smp rendezvous", NULL, MTX_SPIN); /* Probe for MP hardware. */ if (smp_disabled != 0 || cpu_mp_probe() == 0) { mp_ncpus = 1; CPU_SETOF(PCPU_GET(cpuid), &all_cpus); return; } cpu_mp_start(); printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", mp_ncpus); cpu_mp_announce(); } SYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_THIRD, mp_start, NULL); void forward_signal(struct thread *td) { int id; /* * signotify() has already set TDF_ASTPENDING and TDF_NEEDSIGCHECK on * this thread, so all we need to do is poke it if it is currently * executing so that it executes ast(). */ THREAD_LOCK_ASSERT(td, MA_OWNED); KASSERT(TD_IS_RUNNING(td), ("forward_signal: thread is not TDS_RUNNING")); CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc); if (!smp_started || cold || panicstr) return; if (!forward_signal_enabled) return; /* No need to IPI ourself. */ if (td == curthread) return; id = td->td_oncpu; if (id == NOCPU) return; ipi_cpu(id, IPI_AST); } /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * */ static int generic_stop_cpus(cpuset_t map, u_int type) { #ifdef KTR char cpusetbuf[CPUSETBUFSIZ]; #endif static volatile u_int stopping_cpu = NOCPU; int i; volatile cpuset_t *cpus; KASSERT( #if defined(__amd64__) || defined(__i386__) type == IPI_STOP || type == IPI_STOP_HARD || type == IPI_SUSPEND, #else type == IPI_STOP || type == IPI_STOP_HARD, #endif ("%s: invalid stop type", __func__)); if (!smp_started) return (0); CTR2(KTR_SMP, "stop_cpus(%s) with %u type", cpusetobj_strprint(cpusetbuf, &map), type); #if defined(__amd64__) || defined(__i386__) /* * When suspending, ensure there are are no IPIs in progress. * IPIs that have been issued, but not yet delivered (e.g. * not pending on a vCPU when running under virtualization) * will be lost, violating FreeBSD's assumption of reliable * IPI delivery. */ if (type == IPI_SUSPEND) mtx_lock_spin(&smp_ipi_mtx); #endif if (stopping_cpu != PCPU_GET(cpuid)) while (atomic_cmpset_int(&stopping_cpu, NOCPU, PCPU_GET(cpuid)) == 0) while (stopping_cpu != NOCPU) cpu_spinwait(); /* spin */ /* send the stop IPI to all CPUs in map */ ipi_selected(map, type); #if defined(__amd64__) || defined(__i386__) if (type == IPI_SUSPEND) cpus = &suspended_cpus; else #endif cpus = &stopped_cpus; i = 0; while (!CPU_SUBSET(cpus, &map)) { /* spin */ cpu_spinwait(); i++; if (i == 100000000) { printf("timeout stopping cpus\n"); break; } } #if defined(__amd64__) || defined(__i386__) if (type == IPI_SUSPEND) mtx_unlock_spin(&smp_ipi_mtx); #endif stopping_cpu = NOCPU; return (1); } int stop_cpus(cpuset_t map) { return (generic_stop_cpus(map, IPI_STOP)); } int stop_cpus_hard(cpuset_t map) { return (generic_stop_cpus(map, IPI_STOP_HARD)); } #if defined(__amd64__) || defined(__i386__) int suspend_cpus(cpuset_t map) { return (generic_stop_cpus(map, IPI_SUSPEND)); } #endif /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ static int generic_restart_cpus(cpuset_t map, u_int type) { #ifdef KTR char cpusetbuf[CPUSETBUFSIZ]; #endif volatile cpuset_t *cpus; KASSERT( #if defined(__amd64__) || defined(__i386__) type == IPI_STOP || type == IPI_STOP_HARD || type == IPI_SUSPEND, #else type == IPI_STOP || type == IPI_STOP_HARD, #endif ("%s: invalid stop type", __func__)); if (!smp_started) return 0; CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map)); #if defined(__amd64__) || defined(__i386__) if (type == IPI_SUSPEND) cpus = &suspended_cpus; else #endif cpus = &stopped_cpus; /* signal other cpus to restart */ CPU_COPY_STORE_REL(&map, &started_cpus); /* wait for each to clear its bit */ while (CPU_OVERLAP(cpus, &map)) cpu_spinwait(); return 1; } int restart_cpus(cpuset_t map) { return (generic_restart_cpus(map, IPI_STOP)); } #if defined(__amd64__) || defined(__i386__) int resume_cpus(cpuset_t map) { return (generic_restart_cpus(map, IPI_SUSPEND)); } #endif /* * All-CPU rendezvous. CPUs are signalled, all execute the setup function * (if specified), rendezvous, execute the action function (if specified), * rendezvous again, execute the teardown function (if specified), and then * resume. * * Note that the supplied external functions _must_ be reentrant and aware * that they are running in parallel and in an unknown lock context. */ void smp_rendezvous_action(void) { struct thread *td; void *local_func_arg; void (*local_setup_func)(void*); void (*local_action_func)(void*); void (*local_teardown_func)(void*); #ifdef INVARIANTS int owepreempt; #endif /* Ensure we have up-to-date values. */ atomic_add_acq_int(&smp_rv_waiters[0], 1); while (smp_rv_waiters[0] < smp_rv_ncpus) cpu_spinwait(); /* Fetch rendezvous parameters after acquire barrier. */ local_func_arg = smp_rv_func_arg; local_setup_func = smp_rv_setup_func; local_action_func = smp_rv_action_func; local_teardown_func = smp_rv_teardown_func; /* * Use a nested critical section to prevent any preemptions * from occurring during a rendezvous action routine. * Specifically, if a rendezvous handler is invoked via an IPI * and the interrupted thread was in the critical_exit() * function after setting td_critnest to 0 but before * performing a deferred preemption, this routine can be * invoked with td_critnest set to 0 and td_owepreempt true. * In that case, a critical_exit() during the rendezvous * action would trigger a preemption which is not permitted in * a rendezvous action. To fix this, wrap all of the * rendezvous action handlers in a critical section. We * cannot use a regular critical section however as having * critical_exit() preempt from this routine would also be * problematic (the preemption must not occur before the IPI * has been acknowledged via an EOI). Instead, we * intentionally ignore td_owepreempt when leaving the * critical section. This should be harmless because we do * not permit rendezvous action routines to schedule threads, * and thus td_owepreempt should never transition from 0 to 1 * during this routine. */ td = curthread; td->td_critnest++; #ifdef INVARIANTS owepreempt = td->td_owepreempt; #endif /* * If requested, run a setup function before the main action * function. Ensure all CPUs have completed the setup * function before moving on to the action function. */ if (local_setup_func != smp_no_rendevous_barrier) { if (smp_rv_setup_func != NULL) smp_rv_setup_func(smp_rv_func_arg); atomic_add_int(&smp_rv_waiters[1], 1); while (smp_rv_waiters[1] < smp_rv_ncpus) cpu_spinwait(); } if (local_action_func != NULL) local_action_func(local_func_arg); if (local_teardown_func != smp_no_rendevous_barrier) { /* * Signal that the main action has been completed. If a * full exit rendezvous is requested, then all CPUs will * wait here until all CPUs have finished the main action. */ atomic_add_int(&smp_rv_waiters[2], 1); while (smp_rv_waiters[2] < smp_rv_ncpus) cpu_spinwait(); if (local_teardown_func != NULL) local_teardown_func(local_func_arg); } /* * Signal that the rendezvous is fully completed by this CPU. * This means that no member of smp_rv_* pseudo-structure will be * accessed by this target CPU after this point; in particular, * memory pointed by smp_rv_func_arg. * * The release semantic ensures that all accesses performed by * the current CPU are visible when smp_rendezvous_cpus() * returns, by synchronizing with the * atomic_load_acq_int(&smp_rv_waiters[3]). */ atomic_add_rel_int(&smp_rv_waiters[3], 1); td->td_critnest--; KASSERT(owepreempt == td->td_owepreempt, ("rendezvous action changed td_owepreempt")); } void smp_rendezvous_cpus(cpuset_t map, void (* setup_func)(void *), void (* action_func)(void *), void (* teardown_func)(void *), void *arg) { int curcpumap, i, ncpus = 0; /* Look comments in the !SMP case. */ if (!smp_started) { spinlock_enter(); if (setup_func != NULL) setup_func(arg); if (action_func != NULL) action_func(arg); if (teardown_func != NULL) teardown_func(arg); spinlock_exit(); return; } CPU_FOREACH(i) { if (CPU_ISSET(i, &map)) ncpus++; } if (ncpus == 0) panic("ncpus is 0 with non-zero map"); mtx_lock_spin(&smp_ipi_mtx); /* Pass rendezvous parameters via global variables. */ smp_rv_ncpus = ncpus; smp_rv_setup_func = setup_func; smp_rv_action_func = action_func; smp_rv_teardown_func = teardown_func; smp_rv_func_arg = arg; smp_rv_waiters[1] = 0; smp_rv_waiters[2] = 0; smp_rv_waiters[3] = 0; atomic_store_rel_int(&smp_rv_waiters[0], 0); /* * Signal other processors, which will enter the IPI with * interrupts off. */ curcpumap = CPU_ISSET(curcpu, &map); CPU_CLR(curcpu, &map); ipi_selected(map, IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ if (curcpumap != 0) smp_rendezvous_action(); /* * Ensure that the master CPU waits for all the other * CPUs to finish the rendezvous, so that smp_rv_* * pseudo-structure and the arg are guaranteed to not * be in use. * * Load acquire synchronizes with the release add in * smp_rendezvous_action(), which ensures that our caller sees * all memory actions done by the called functions on other * CPUs. */ while (atomic_load_acq_int(&smp_rv_waiters[3]) < ncpus) cpu_spinwait(); mtx_unlock_spin(&smp_ipi_mtx); } void smp_rendezvous(void (* setup_func)(void *), void (* action_func)(void *), void (* teardown_func)(void *), void *arg) { smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func, arg); } static struct cpu_group group[MAXCPU * MAX_CACHE_LEVELS + 1]; struct cpu_group * smp_topo(void) { char cpusetbuf[CPUSETBUFSIZ], cpusetbuf2[CPUSETBUFSIZ]; struct cpu_group *top; /* * Check for a fake topology request for debugging purposes. */ switch (smp_topology) { case 1: /* Dual core with no sharing. */ top = smp_topo_1level(CG_SHARE_NONE, 2, 0); break; case 2: /* No topology, all cpus are equal. */ top = smp_topo_none(); break; case 3: /* Dual core with shared L2. */ top = smp_topo_1level(CG_SHARE_L2, 2, 0); break; case 4: /* quad core, shared l3 among each package, private l2. */ top = smp_topo_1level(CG_SHARE_L3, 4, 0); break; case 5: /* quad core, 2 dualcore parts on each package share l2. */ top = smp_topo_2level(CG_SHARE_NONE, 2, CG_SHARE_L2, 2, 0); break; case 6: /* Single-core 2xHTT */ top = smp_topo_1level(CG_SHARE_L1, 2, CG_FLAG_HTT); break; case 7: /* quad core with a shared l3, 8 threads sharing L2. */ top = smp_topo_2level(CG_SHARE_L3, 4, CG_SHARE_L2, 8, CG_FLAG_SMT); break; default: /* Default, ask the system what it wants. */ top = cpu_topo(); break; } /* * Verify the returned topology. */ if (top->cg_count != mp_ncpus) panic("Built bad topology at %p. CPU count %d != %d", top, top->cg_count, mp_ncpus); if (CPU_CMP(&top->cg_mask, &all_cpus)) panic("Built bad topology at %p. CPU mask (%s) != (%s)", top, cpusetobj_strprint(cpusetbuf, &top->cg_mask), cpusetobj_strprint(cpusetbuf2, &all_cpus)); return (top); } struct cpu_group * smp_topo_alloc(u_int count) { static u_int index; u_int curr; curr = index; index += count; return (&group[curr]); } struct cpu_group * smp_topo_none(void) { struct cpu_group *top; top = &group[0]; top->cg_parent = NULL; top->cg_child = NULL; top->cg_mask = all_cpus; top->cg_count = mp_ncpus; top->cg_children = 0; top->cg_level = CG_SHARE_NONE; top->cg_flags = 0; return (top); } static int smp_topo_addleaf(struct cpu_group *parent, struct cpu_group *child, int share, int count, int flags, int start) { char cpusetbuf[CPUSETBUFSIZ], cpusetbuf2[CPUSETBUFSIZ]; cpuset_t mask; int i; CPU_ZERO(&mask); for (i = 0; i < count; i++, start++) CPU_SET(start, &mask); child->cg_parent = parent; child->cg_child = NULL; child->cg_children = 0; child->cg_level = share; child->cg_count = count; child->cg_flags = flags; child->cg_mask = mask; parent->cg_children++; for (; parent != NULL; parent = parent->cg_parent) { if (CPU_OVERLAP(&parent->cg_mask, &child->cg_mask)) panic("Duplicate children in %p. mask (%s) child (%s)", parent, cpusetobj_strprint(cpusetbuf, &parent->cg_mask), cpusetobj_strprint(cpusetbuf2, &child->cg_mask)); CPU_OR(&parent->cg_mask, &child->cg_mask); parent->cg_count += child->cg_count; } return (start); } struct cpu_group * smp_topo_1level(int share, int count, int flags) { struct cpu_group *child; struct cpu_group *top; int packages; int cpu; int i; cpu = 0; top = &group[0]; packages = mp_ncpus / count; top->cg_child = child = &group[1]; top->cg_level = CG_SHARE_NONE; for (i = 0; i < packages; i++, child++) cpu = smp_topo_addleaf(top, child, share, count, flags, cpu); return (top); } struct cpu_group * smp_topo_2level(int l2share, int l2count, int l1share, int l1count, int l1flags) { struct cpu_group *top; struct cpu_group *l1g; struct cpu_group *l2g; int cpu; int i; int j; cpu = 0; top = &group[0]; l2g = &group[1]; top->cg_child = l2g; top->cg_level = CG_SHARE_NONE; top->cg_children = mp_ncpus / (l2count * l1count); l1g = l2g + top->cg_children; for (i = 0; i < top->cg_children; i++, l2g++) { l2g->cg_parent = top; l2g->cg_child = l1g; l2g->cg_level = l2share; for (j = 0; j < l2count; j++, l1g++) cpu = smp_topo_addleaf(l2g, l1g, l1share, l1count, l1flags, cpu); } return (top); } struct cpu_group * smp_topo_find(struct cpu_group *top, int cpu) { struct cpu_group *cg; cpuset_t mask; int children; int i; CPU_SETOF(cpu, &mask); cg = top; for (;;) { if (!CPU_OVERLAP(&cg->cg_mask, &mask)) return (NULL); if (cg->cg_children == 0) return (cg); children = cg->cg_children; for (i = 0, cg = cg->cg_child; i < children; cg++, i++) if (CPU_OVERLAP(&cg->cg_mask, &mask)) break; } return (NULL); } #else /* !SMP */ void smp_rendezvous_cpus(cpuset_t map, void (*setup_func)(void *), void (*action_func)(void *), void (*teardown_func)(void *), void *arg) { /* * In the !SMP case we just need to ensure the same initial conditions * as the SMP case. */ spinlock_enter(); if (setup_func != NULL) setup_func(arg); if (action_func != NULL) action_func(arg); if (teardown_func != NULL) teardown_func(arg); spinlock_exit(); } void smp_rendezvous(void (*setup_func)(void *), void (*action_func)(void *), void (*teardown_func)(void *), void *arg) { - /* Look comments in the smp_rendezvous_cpus() case. */ - spinlock_enter(); - if (setup_func != NULL) - setup_func(arg); - if (action_func != NULL) - action_func(arg); - if (teardown_func != NULL) - teardown_func(arg); - spinlock_exit(); + smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func, + arg); } /* * Provide dummy SMP support for UP kernels. Modules that need to use SMP * APIs will still work using this dummy support. */ static void mp_setvariables_for_up(void *dummy) { mp_ncpus = 1; mp_maxid = PCPU_GET(cpuid); CPU_SETOF(mp_maxid, &all_cpus); KASSERT(PCPU_GET(cpuid) == 0, ("UP must have a CPU ID of zero")); } SYSINIT(cpu_mp_setvariables, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_setvariables_for_up, NULL); #endif /* SMP */ void smp_no_rendevous_barrier(void *dummy) { #ifdef SMP KASSERT((!smp_started),("smp_no_rendevous called and smp is started")); #endif } /* * Wait specified idle threads to switch once. This ensures that even * preempted threads have cycled through the switch function once, * exiting their codepaths. This allows us to change global pointers * with no other synchronization. */ int quiesce_cpus(cpuset_t map, const char *wmesg, int prio) { struct pcpu *pcpu; u_int gen[MAXCPU]; int error; int cpu; error = 0; for (cpu = 0; cpu <= mp_maxid; cpu++) { if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu)) continue; pcpu = pcpu_find(cpu); gen[cpu] = pcpu->pc_idlethread->td_generation; } for (cpu = 0; cpu <= mp_maxid; cpu++) { if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu)) continue; pcpu = pcpu_find(cpu); thread_lock(curthread); sched_bind(curthread, cpu); thread_unlock(curthread); while (gen[cpu] == pcpu->pc_idlethread->td_generation) { error = tsleep(quiesce_cpus, prio, wmesg, 1); if (error != EWOULDBLOCK) goto out; error = 0; } } out: thread_lock(curthread); sched_unbind(curthread); thread_unlock(curthread); return (error); } int quiesce_all_cpus(const char *wmesg, int prio) { return quiesce_cpus(all_cpus, wmesg, prio); } /* Extra care is taken with this sysctl because the data type is volatile */ static int sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS) { int error, active; active = smp_started; error = SYSCTL_OUT(req, &active, sizeof(active)); return (error); } #ifdef SMP void topo_init_node(struct topo_node *node) { bzero(node, sizeof(*node)); TAILQ_INIT(&node->children); } void topo_init_root(struct topo_node *root) { topo_init_node(root); root->type = TOPO_TYPE_SYSTEM; } /* * Add a child node with the given ID under the given parent. * Do nothing if there is already a child with that ID. */ struct topo_node * topo_add_node_by_hwid(struct topo_node *parent, int hwid, topo_node_type type, uintptr_t subtype) { struct topo_node *node; TAILQ_FOREACH_REVERSE(node, &parent->children, topo_children, siblings) { if (node->hwid == hwid && node->type == type && node->subtype == subtype) { return (node); } } node = malloc(sizeof(*node), M_TOPO, M_WAITOK); topo_init_node(node); node->parent = parent; node->hwid = hwid; node->type = type; node->subtype = subtype; TAILQ_INSERT_TAIL(&parent->children, node, siblings); parent->nchildren++; return (node); } /* * Find a child node with the given ID under the given parent. */ struct topo_node * topo_find_node_by_hwid(struct topo_node *parent, int hwid, topo_node_type type, uintptr_t subtype) { struct topo_node *node; TAILQ_FOREACH(node, &parent->children, siblings) { if (node->hwid == hwid && node->type == type && node->subtype == subtype) { return (node); } } return (NULL); } /* * Given a node change the order of its parent's child nodes such * that the node becomes the firt child while preserving the cyclic * order of the children. In other words, the given node is promoted * by rotation. */ void topo_promote_child(struct topo_node *child) { struct topo_node *next; struct topo_node *node; struct topo_node *parent; parent = child->parent; next = TAILQ_NEXT(child, siblings); TAILQ_REMOVE(&parent->children, child, siblings); TAILQ_INSERT_HEAD(&parent->children, child, siblings); while (next != NULL) { node = next; next = TAILQ_NEXT(node, siblings); TAILQ_REMOVE(&parent->children, node, siblings); TAILQ_INSERT_AFTER(&parent->children, child, node, siblings); child = node; } } /* * Iterate to the next node in the depth-first search (traversal) of * the topology tree. */ struct topo_node * topo_next_node(struct topo_node *top, struct topo_node *node) { struct topo_node *next; if ((next = TAILQ_FIRST(&node->children)) != NULL) return (next); if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); while ((node = node->parent) != top) if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); return (NULL); } /* * Iterate to the next node in the depth-first search of the topology tree, * but without descending below the current node. */ struct topo_node * topo_next_nonchild_node(struct topo_node *top, struct topo_node *node) { struct topo_node *next; if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); while ((node = node->parent) != top) if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); return (NULL); } /* * Assign the given ID to the given topology node that represents a logical * processor. */ void topo_set_pu_id(struct topo_node *node, cpuid_t id) { KASSERT(node->type == TOPO_TYPE_PU, ("topo_set_pu_id: wrong node type: %u", node->type)); KASSERT(CPU_EMPTY(&node->cpuset) && node->cpu_count == 0, ("topo_set_pu_id: cpuset already not empty")); node->id = id; CPU_SET(id, &node->cpuset); node->cpu_count = 1; node->subtype = 1; while ((node = node->parent) != NULL) { KASSERT(!CPU_ISSET(id, &node->cpuset), ("logical ID %u is already set in node %p", id, node)); CPU_SET(id, &node->cpuset); node->cpu_count++; } } /* * Check if the topology is uniform, that is, each package has the same number * of cores in it and each core has the same number of threads (logical * processors) in it. If so, calculate the number of package, the number of * cores per package and the number of logical processors per core. * 'all' parameter tells whether to include administratively disabled logical * processors into the analysis. */ int topo_analyze(struct topo_node *topo_root, int all, int *pkg_count, int *cores_per_pkg, int *thrs_per_core) { struct topo_node *pkg_node; struct topo_node *core_node; struct topo_node *pu_node; int thrs_per_pkg; int cpp_counter; int tpc_counter; int tpp_counter; *pkg_count = 0; *cores_per_pkg = -1; *thrs_per_core = -1; thrs_per_pkg = -1; pkg_node = topo_root; while (pkg_node != NULL) { if (pkg_node->type != TOPO_TYPE_PKG) { pkg_node = topo_next_node(topo_root, pkg_node); continue; } if (!all && CPU_EMPTY(&pkg_node->cpuset)) { pkg_node = topo_next_nonchild_node(topo_root, pkg_node); continue; } (*pkg_count)++; cpp_counter = 0; tpp_counter = 0; core_node = pkg_node; while (core_node != NULL) { if (core_node->type == TOPO_TYPE_CORE) { if (!all && CPU_EMPTY(&core_node->cpuset)) { core_node = topo_next_nonchild_node(pkg_node, core_node); continue; } cpp_counter++; tpc_counter = 0; pu_node = core_node; while (pu_node != NULL) { if (pu_node->type == TOPO_TYPE_PU && (all || !CPU_EMPTY(&pu_node->cpuset))) tpc_counter++; pu_node = topo_next_node(core_node, pu_node); } if (*thrs_per_core == -1) *thrs_per_core = tpc_counter; else if (*thrs_per_core != tpc_counter) return (0); core_node = topo_next_nonchild_node(pkg_node, core_node); } else { /* PU node directly under PKG. */ if (core_node->type == TOPO_TYPE_PU && (all || !CPU_EMPTY(&core_node->cpuset))) tpp_counter++; core_node = topo_next_node(pkg_node, core_node); } } if (*cores_per_pkg == -1) *cores_per_pkg = cpp_counter; else if (*cores_per_pkg != cpp_counter) return (0); if (thrs_per_pkg == -1) thrs_per_pkg = tpp_counter; else if (thrs_per_pkg != tpp_counter) return (0); pkg_node = topo_next_nonchild_node(topo_root, pkg_node); } KASSERT(*pkg_count > 0, ("bug in topology or analysis")); if (*cores_per_pkg == 0) { KASSERT(*thrs_per_core == -1 && thrs_per_pkg > 0, ("bug in topology or analysis")); *thrs_per_core = thrs_per_pkg; } return (1); } #endif /* SMP */ Index: projects/clang390-import/sys/net/rndis.h =================================================================== --- projects/clang390-import/sys/net/rndis.h (revision 305080) +++ projects/clang390-import/sys/net/rndis.h (revision 305081) @@ -1,283 +1,287 @@ /* $FreeBSD$ */ /* $OpenBSD: if_urndisreg.h,v 1.19 2013/11/21 14:08:05 mpi Exp $ */ /* * Copyright (c) 2010 Jonathan Armani * Copyright (c) 2010 Fabien Romano * Copyright (c) 2010 Michael Knudsen * All rights reserved. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #ifndef _NET_RNDIS_H_ #define _NET_RNDIS_H_ /* Canonical major/minor version as of 22th Aug. 2016. */ #define RNDIS_VERSION_MAJOR 0x00000001 #define RNDIS_VERSION_MINOR 0x00000000 #define RNDIS_STATUS_SUCCESS 0x00000000L #define RNDIS_STATUS_PENDING 0x00000103L #define RNDIS_STATUS_MEDIA_CONNECT 0x4001000BL #define RNDIS_STATUS_MEDIA_DISCONNECT 0x4001000CL #define RNDIS_STATUS_BUFFER_OVERFLOW 0x80000005L #define RNDIS_STATUS_FAILURE 0xC0000001L #define RNDIS_STATUS_NOT_SUPPORTED 0xC00000BBL #define RNDIS_STATUS_RESOURCES 0xC000009AL #define RNDIS_STATUS_INVALID_DATA 0xC0010015L #define OID_GEN_SUPPORTED_LIST 0x00010101 #define OID_GEN_HARDWARE_STATUS 0x00010102 #define OID_GEN_MEDIA_SUPPORTED 0x00010103 #define OID_GEN_MEDIA_IN_USE 0x00010104 #define OID_GEN_MAXIMUM_LOOKAHEAD 0x00010105 #define OID_GEN_MAXIMUM_FRAME_SIZE 0x00010106 #define OID_GEN_LINK_SPEED 0x00010107 #define OID_GEN_TRANSMIT_BUFFER_SPACE 0x00010108 #define OID_GEN_RECEIVE_BUFFER_SPACE 0x00010109 #define OID_GEN_TRANSMIT_BLOCK_SIZE 0x0001010A #define OID_GEN_RECEIVE_BLOCK_SIZE 0x0001010B #define OID_GEN_VENDOR_ID 0x0001010C #define OID_GEN_VENDOR_DESCRIPTION 0x0001010D #define OID_GEN_CURRENT_PACKET_FILTER 0x0001010E #define OID_GEN_CURRENT_LOOKAHEAD 0x0001010F #define OID_GEN_DRIVER_VERSION 0x00010110 #define OID_GEN_MAXIMUM_TOTAL_SIZE 0x00010111 #define OID_GEN_PROTOCOL_OPTIONS 0x00010112 #define OID_GEN_MAC_OPTIONS 0x00010113 #define OID_GEN_MEDIA_CONNECT_STATUS 0x00010114 #define OID_GEN_MAXIMUM_SEND_PACKETS 0x00010115 #define OID_GEN_VENDOR_DRIVER_VERSION 0x00010116 #define OID_GEN_SUPPORTED_GUIDS 0x00010117 #define OID_GEN_NETWORK_LAYER_ADDRESSES 0x00010118 #define OID_GEN_TRANSPORT_HEADER_OFFSET 0x00010119 +#define OID_GEN_RECEIVE_SCALE_CAPABILITIES 0x00010203 +#define OID_GEN_RECEIVE_SCALE_PARAMETERS 0x00010204 #define OID_GEN_MACHINE_NAME 0x0001021A #define OID_GEN_RNDIS_CONFIG_PARAMETER 0x0001021B #define OID_GEN_VLAN_ID 0x0001021C #define OID_802_3_PERMANENT_ADDRESS 0x01010101 #define OID_802_3_CURRENT_ADDRESS 0x01010102 #define OID_802_3_MULTICAST_LIST 0x01010103 #define OID_802_3_MAXIMUM_LIST_SIZE 0x01010104 #define OID_802_3_MAC_OPTIONS 0x01010105 #define OID_802_3_RCV_ERROR_ALIGNMENT 0x01020101 #define OID_802_3_XMIT_ONE_COLLISION 0x01020102 #define OID_802_3_XMIT_MORE_COLLISIONS 0x01020103 #define OID_802_3_XMIT_DEFERRED 0x01020201 #define OID_802_3_XMIT_MAX_COLLISIONS 0x01020202 #define OID_802_3_RCV_OVERRUN 0x01020203 #define OID_802_3_XMIT_UNDERRUN 0x01020204 #define OID_802_3_XMIT_HEARTBEAT_FAILURE 0x01020205 #define OID_802_3_XMIT_TIMES_CRS_LOST 0x01020206 #define OID_802_3_XMIT_LATE_COLLISIONS 0x01020207 +#define OID_TCP_OFFLOAD_PARAMETERS 0xFC01020C + #define RNDIS_MEDIUM_802_3 0x00000000 /* Device flags */ #define RNDIS_DF_CONNECTIONLESS 0x00000001 #define RNDIS_DF_CONNECTION_ORIENTED 0x00000002 /* * RNDIS data message */ #define REMOTE_NDIS_PACKET_MSG 0x00000001 struct rndis_packet_msg { uint32_t rm_type; uint32_t rm_len; uint32_t rm_dataoffset; uint32_t rm_datalen; uint32_t rm_oobdataoffset; uint32_t rm_oobdatalen; uint32_t rm_oobdataelements; uint32_t rm_pktinfooffset; uint32_t rm_pktinfolen; uint32_t rm_vchandle; uint32_t rm_reserved; }; /* * RNDIS control messages */ struct rndis_comp_hdr { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_status; }; /* Initialize the device. */ #define REMOTE_NDIS_INITIALIZE_MSG 0x00000002 #define REMOTE_NDIS_INITIALIZE_CMPLT 0x80000002 struct rndis_init_req { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_ver_major; uint32_t rm_ver_minor; uint32_t rm_max_xfersz; }; struct rndis_init_comp { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_status; uint32_t rm_ver_major; uint32_t rm_ver_minor; uint32_t rm_devflags; uint32_t rm_medium; uint32_t rm_pktmaxcnt; uint32_t rm_pktmaxsz; uint32_t rm_align; uint32_t rm_aflistoffset; uint32_t rm_aflistsz; }; #define RNDIS_INIT_COMP_SIZE_MIN \ __offsetof(struct rndis_init_comp, rm_aflistsz) /* Halt the device. No response sent. */ #define REMOTE_NDIS_HALT_MSG 0x00000003 struct rndis_halt_req { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; }; /* Send a query object. */ #define REMOTE_NDIS_QUERY_MSG 0x00000004 #define REMOTE_NDIS_QUERY_CMPLT 0x80000004 struct rndis_query_req { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_oid; uint32_t rm_infobuflen; uint32_t rm_infobufoffset; uint32_t rm_devicevchdl; }; #define RNDIS_QUERY_REQ_INFOBUFOFFSET \ (sizeof(struct rndis_query_req) - \ __offsetof(struct rndis_query_req, rm_rid)) struct rndis_query_comp { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_status; uint32_t rm_infobuflen; uint32_t rm_infobufoffset; }; #define RNDIS_QUERY_COMP_INFOBUFABS(ofs) \ ((ofs) + __offsetof(struct rndis_query_req, rm_rid)) /* Send a set object request. */ #define REMOTE_NDIS_SET_MSG 0x00000005 #define REMOTE_NDIS_SET_CMPLT 0x80000005 struct rndis_set_req { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_oid; uint32_t rm_infobuflen; uint32_t rm_infobufoffset; uint32_t rm_devicevchdl; }; #define RNDIS_SET_REQ_INFOBUFOFFSET \ (sizeof(struct rndis_set_req) - \ __offsetof(struct rndis_set_req, rm_rid)) struct rndis_set_comp { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_status; }; #define REMOTE_NDIS_SET_PARAM_NUMERIC 0x00000000 #define REMOTE_NDIS_SET_PARAM_STRING 0x00000002 struct rndis_set_parameter { uint32_t rm_nameoffset; uint32_t rm_namelen; uint32_t rm_type; uint32_t rm_valueoffset; uint32_t rm_valuelen; }; /* Perform a soft reset on the device. */ #define REMOTE_NDIS_RESET_MSG 0x00000006 #define REMOTE_NDIS_RESET_CMPLT 0x80000006 struct rndis_reset_req { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; }; struct rndis_reset_comp { uint32_t rm_type; uint32_t rm_len; uint32_t rm_status; uint32_t rm_adrreset; }; /* 802.3 link-state or undefined message error. */ #define REMOTE_NDIS_INDICATE_STATUS_MSG 0x00000007 /* Keepalive messsage. May be sent by device. */ #define REMOTE_NDIS_KEEPALIVE_MSG 0x00000008 #define REMOTE_NDIS_KEEPALIVE_CMPLT 0x80000008 struct rndis_keepalive_req { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; }; struct rndis_keepalive_comp { uint32_t rm_type; uint32_t rm_len; uint32_t rm_rid; uint32_t rm_status; }; /* packet filter bits used by OID_GEN_CURRENT_PACKET_FILTER */ -#define RNDIS_PACKET_TYPE_DIRECTED 0x00000001 -#define RNDIS_PACKET_TYPE_MULTICAST 0x00000002 -#define RNDIS_PACKET_TYPE_ALL_MULTICAST 0x00000004 -#define RNDIS_PACKET_TYPE_BROADCAST 0x00000008 -#define RNDIS_PACKET_TYPE_SOURCE_ROUTING 0x00000010 -#define RNDIS_PACKET_TYPE_PROMISCUOUS 0x00000020 -#define RNDIS_PACKET_TYPE_SMT 0x00000040 -#define RNDIS_PACKET_TYPE_ALL_LOCAL 0x00000080 -#define RNDIS_PACKET_TYPE_GROUP 0x00001000 -#define RNDIS_PACKET_TYPE_ALL_FUNCTIONAL 0x00002000 -#define RNDIS_PACKET_TYPE_FUNCTIONAL 0x00004000 -#define RNDIS_PACKET_TYPE_MAC_FRAME 0x00008000 +#define NDIS_PACKET_TYPE_DIRECTED 0x00000001 +#define NDIS_PACKET_TYPE_MULTICAST 0x00000002 +#define NDIS_PACKET_TYPE_ALL_MULTICAST 0x00000004 +#define NDIS_PACKET_TYPE_BROADCAST 0x00000008 +#define NDIS_PACKET_TYPE_SOURCE_ROUTING 0x00000010 +#define NDIS_PACKET_TYPE_PROMISCUOUS 0x00000020 +#define NDIS_PACKET_TYPE_SMT 0x00000040 +#define NDIS_PACKET_TYPE_ALL_LOCAL 0x00000080 +#define NDIS_PACKET_TYPE_GROUP 0x00001000 +#define NDIS_PACKET_TYPE_ALL_FUNCTIONAL 0x00002000 +#define NDIS_PACKET_TYPE_FUNCTIONAL 0x00004000 +#define NDIS_PACKET_TYPE_MAC_FRAME 0x00008000 /* RNDIS offsets */ #define RNDIS_HEADER_OFFSET 8 /* bytes */ #define RNDIS_DATA_OFFSET \ ((uint32_t)(sizeof(struct rndis_packet_msg) - RNDIS_HEADER_OFFSET)) #endif /* !_NET_RNDIS_H_ */ Index: projects/clang390-import/sys/powerpc/mpc85xx/platform_mpc85xx.c =================================================================== --- projects/clang390-import/sys/powerpc/mpc85xx/platform_mpc85xx.c (revision 305080) +++ projects/clang390-import/sys/powerpc/mpc85xx/platform_mpc85xx.c (revision 305081) @@ -1,569 +1,556 @@ /*- * Copyright (c) 2008-2012 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "platform_if.h" #ifdef SMP extern void *ap_pcpu; extern vm_paddr_t kernload; /* Kernel physical load address */ extern uint8_t __boot_page[]; /* Boot page body */ extern uint32_t bp_kernload; struct cpu_release { uint32_t entry_h; uint32_t entry_l; uint32_t r3_h; uint32_t r3_l; uint32_t reserved; uint32_t pir; }; #endif extern uint32_t *bootinfo; vm_offset_t ccsrbar_va; static int cpu, maxcpu; static int mpc85xx_probe(platform_t); static void mpc85xx_mem_regions(platform_t, struct mem_region *phys, int *physsz, struct mem_region *avail, int *availsz); static u_long mpc85xx_timebase_freq(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_first_cpu(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_next_cpu(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_get_bsp(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_start_cpu(platform_t, struct pcpu *cpu); static void mpc85xx_idle(platform_t, int cpu); static int mpc85xx_idle_wakeup(platform_t plat, int cpu); static void mpc85xx_reset(platform_t); static platform_method_t mpc85xx_methods[] = { PLATFORMMETHOD(platform_probe, mpc85xx_probe), PLATFORMMETHOD(platform_attach, mpc85xx_attach), PLATFORMMETHOD(platform_mem_regions, mpc85xx_mem_regions), PLATFORMMETHOD(platform_timebase_freq, mpc85xx_timebase_freq), PLATFORMMETHOD(platform_smp_first_cpu, mpc85xx_smp_first_cpu), PLATFORMMETHOD(platform_smp_next_cpu, mpc85xx_smp_next_cpu), PLATFORMMETHOD(platform_smp_get_bsp, mpc85xx_smp_get_bsp), PLATFORMMETHOD(platform_smp_start_cpu, mpc85xx_smp_start_cpu), PLATFORMMETHOD(platform_reset, mpc85xx_reset), PLATFORMMETHOD(platform_idle, mpc85xx_idle), PLATFORMMETHOD(platform_idle_wakeup, mpc85xx_idle_wakeup), PLATFORMMETHOD_END }; DEFINE_CLASS_0(mpc85xx, mpc85xx_platform, mpc85xx_methods, 0); PLATFORM_DEF(mpc85xx_platform); static int mpc85xx_probe(platform_t plat) { - u_int pvr = mfpvr() >> 16; + u_int pvr = (mfpvr() >> 16) & 0xFFFF; - if ((pvr & 0xfff0) == FSL_E500v1) - return (BUS_PROBE_DEFAULT); - + switch (pvr) { + case FSL_E500v1: + case FSL_E500v2: + case FSL_E500mc: + case FSL_E5500: + case FSL_E6500: + return (BUS_PROBE_DEFAULT); + } return (ENXIO); } int mpc85xx_attach(platform_t plat) { phandle_t cpus, child, ccsr; const char *soc_name_guesses[] = {"/soc", "soc", NULL}; const char **name; pcell_t ranges[6], acells, pacells, scells; - uint32_t sr; uint64_t ccsrbar, ccsrsize; - int i, law_max, tgt; + int i; if ((cpus = OF_finddevice("/cpus")) != -1) { for (maxcpu = 0, child = OF_child(cpus); child != 0; child = OF_peer(child), maxcpu++) ; } else maxcpu = 1; /* * Locate CCSR region. Irritatingly, there is no way to find it * unless you already know where it is. Try to infer its location * from the device tree. */ ccsr = -1; for (name = soc_name_guesses; *name != NULL && ccsr == -1; name++) ccsr = OF_finddevice(*name); if (ccsr == -1) { char type[64]; /* That didn't work. Search for devices of type "soc" */ child = OF_child(OF_peer(0)); for (OF_child(child); child != 0; child = OF_peer(child)) { if (OF_getprop(child, "device_type", type, sizeof(type)) <= 0) continue; if (strcmp(type, "soc") == 0) { ccsr = child; break; } } } if (ccsr == -1) panic("Could not locate CCSR window!"); OF_getprop(ccsr, "#size-cells", &scells, sizeof(scells)); OF_getprop(ccsr, "#address-cells", &acells, sizeof(acells)); OF_searchprop(OF_parent(ccsr), "#address-cells", &pacells, sizeof(pacells)); OF_getprop(ccsr, "ranges", ranges, sizeof(ranges)); ccsrbar = ccsrsize = 0; for (i = acells; i < acells + pacells; i++) { ccsrbar <<= 32; ccsrbar |= ranges[i]; } for (i = acells + pacells; i < acells + pacells + scells; i++) { ccsrsize <<= 32; ccsrsize |= ranges[i]; } ccsrbar_va = pmap_early_io_map(ccsrbar, ccsrsize); mpc85xx_fix_errata(ccsrbar_va); mpc85xx_enable_l3_cache(); - - /* - * Clear local access windows. Skip DRAM entries, so we don't shoot - * ourselves in the foot. - */ - law_max = law_getmax(); - for (i = 0; i < law_max; i++) { - sr = ccsr_read4(OCP85XX_LAWSR(i)); - if ((sr & OCP85XX_ENA_MASK) == 0) - continue; - tgt = (sr & 0x01f00000) >> 20; - if (tgt == OCP85XX_TGTIF_RAM1 || tgt == OCP85XX_TGTIF_RAM2 || - tgt == OCP85XX_TGTIF_RAM_INTL) - continue; - - ccsr_write4(OCP85XX_LAWSR(i), sr & OCP85XX_DIS_MASK); - } return (0); } void mpc85xx_mem_regions(platform_t plat, struct mem_region *phys, int *physsz, struct mem_region *avail, int *availsz) { ofw_mem_regions(phys, physsz, avail, availsz); } static u_long mpc85xx_timebase_freq(platform_t plat, struct cpuref *cpuref) { u_long ticks; phandle_t cpus, child; pcell_t freq; if (bootinfo != NULL) { if (bootinfo[0] == 1) { /* Backward compatibility. See 8-STABLE. */ ticks = bootinfo[3] >> 3; } else { /* Compatibility with Juniper's loader. */ ticks = bootinfo[5] >> 3; } } else ticks = 0; if ((cpus = OF_finddevice("/cpus")) == -1) goto out; if ((child = OF_child(cpus)) == 0) goto out; switch (OF_getproplen(child, "timebase-frequency")) { case 4: { uint32_t tbase; OF_getprop(child, "timebase-frequency", &tbase, sizeof(tbase)); ticks = tbase; return (ticks); } case 8: { uint64_t tbase; OF_getprop(child, "timebase-frequency", &tbase, sizeof(tbase)); ticks = tbase; return (ticks); } default: break; } freq = 0; if (OF_getprop(child, "bus-frequency", (void *)&freq, sizeof(freq)) <= 0) goto out; if (freq == 0) goto out; /* * Time Base and Decrementer are updated every 8 CCB bus clocks. * HID0[SEL_TBCLK] = 0 */ if (mpc85xx_is_qoriq()) ticks = freq / 32; else ticks = freq / 8; out: if (ticks <= 0) panic("Unable to determine timebase frequency!"); return (ticks); } static int mpc85xx_smp_first_cpu(platform_t plat, struct cpuref *cpuref) { cpu = 0; cpuref->cr_cpuid = cpu; cpuref->cr_hwref = cpuref->cr_cpuid; if (bootverbose) printf("powerpc_smp_first_cpu: cpuid %d\n", cpuref->cr_cpuid); cpu++; return (0); } static int mpc85xx_smp_next_cpu(platform_t plat, struct cpuref *cpuref) { if (cpu >= maxcpu) return (ENOENT); cpuref->cr_cpuid = cpu++; cpuref->cr_hwref = cpuref->cr_cpuid; if (bootverbose) printf("powerpc_smp_next_cpu: cpuid %d\n", cpuref->cr_cpuid); return (0); } static int mpc85xx_smp_get_bsp(platform_t plat, struct cpuref *cpuref) { cpuref->cr_cpuid = mfspr(SPR_PIR); cpuref->cr_hwref = cpuref->cr_cpuid; return (0); } #ifdef SMP static int mpc85xx_smp_start_cpu_epapr(platform_t plat, struct pcpu *pc) { vm_paddr_t rel_pa, bptr; volatile struct cpu_release *rel; vm_offset_t rel_va, rel_page; phandle_t node; int i; /* If we're calling this, the node already exists. */ node = OF_finddevice("/cpus"); for (i = 0, node = OF_child(node); i < pc->pc_cpuid; i++, node = OF_peer(node)) ; if (OF_getencprop(node, "cpu-release-addr", (pcell_t *)&rel_pa, sizeof(rel_pa)) == -1) { return (ENOENT); } rel_page = kva_alloc(PAGE_SIZE); if (rel_page == 0) return (ENOMEM); critical_enter(); rel_va = rel_page + (rel_pa & PAGE_MASK); pmap_kenter(rel_page, rel_pa & ~PAGE_MASK); rel = (struct cpu_release *)rel_va; bptr = ((vm_paddr_t)(uintptr_t)__boot_page - KERNBASE) + kernload; cpu_flush_dcache(__DEVOLATILE(struct cpu_release *,rel), sizeof(*rel)); rel->pir = pc->pc_cpuid; __asm __volatile("sync"); rel->entry_h = (bptr >> 32); rel->entry_l = bptr; __asm __volatile("sync"); cpu_flush_dcache(__DEVOLATILE(struct cpu_release *,rel), sizeof(*rel)); if (bootverbose) printf("Waking up CPU %d via CPU release page %p\n", pc->pc_cpuid, rel); critical_exit(); pmap_kremove(rel_page); kva_free(rel_page, PAGE_SIZE); return (0); } #endif static int mpc85xx_smp_start_cpu(platform_t plat, struct pcpu *pc) { #ifdef SMP vm_paddr_t bptr; uint32_t reg; int timeout; uintptr_t brr; int cpuid; int epapr_boot = 0; uint32_t tgt; if (mpc85xx_is_qoriq()) { reg = ccsr_read4(OCP85XX_COREDISR); cpuid = pc->pc_cpuid; if ((reg & (1 << cpuid)) != 0) { printf("%s: CPU %d is disabled!\n", __func__, pc->pc_cpuid); return (-1); } brr = OCP85XX_BRR; } else { brr = OCP85XX_EEBPCR; cpuid = pc->pc_cpuid + 24; } bp_kernload = kernload; /* * bp_kernload is in the boot page. Sync the cache because ePAPR * booting has the other core(s) already running. */ cpu_flush_dcache(&bp_kernload, sizeof(bp_kernload)); ap_pcpu = pc; __asm __volatile("msync; isync"); /* First try the ePAPR way. */ if (mpc85xx_smp_start_cpu_epapr(plat, pc) == 0) { epapr_boot = 1; goto spin_wait; } reg = ccsr_read4(brr); if ((reg & (1 << cpuid)) != 0) { printf("SMP: CPU %d already out of hold-off state!\n", pc->pc_cpuid); return (ENXIO); } /* Flush caches to have our changes hit DRAM. */ cpu_flush_dcache(__boot_page, 4096); bptr = ((vm_paddr_t)(uintptr_t)__boot_page - KERNBASE) + kernload; KASSERT((bptr & 0xfff) == 0, ("%s: boot page is not aligned (%#jx)", __func__, (uintmax_t)bptr)); if (mpc85xx_is_qoriq()) { /* * Read DDR controller configuration to select proper BPTR target ID. * * On P5020 bit 29 of DDR1_CS0_CONFIG enables DDR controllers * interleaving. If this bit is set, we have to use * OCP85XX_TGTIF_RAM_INTL as BPTR target ID. On other QorIQ DPAA SoCs, * this bit is reserved and always 0. */ reg = ccsr_read4(OCP85XX_DDR1_CS0_CONFIG); if (reg & (1 << 29)) tgt = OCP85XX_TGTIF_RAM_INTL; else tgt = OCP85XX_TGTIF_RAM1; /* * Set BSTR to the physical address of the boot page */ ccsr_write4(OCP85XX_BSTRH, bptr >> 32); ccsr_write4(OCP85XX_BSTRL, bptr); ccsr_write4(OCP85XX_BSTAR, OCP85XX_ENA_MASK | (tgt << OCP85XX_TRGT_SHIFT_QORIQ) | (ffsl(PAGE_SIZE) - 2)); /* Read back OCP85XX_BSTAR to synchronize write */ ccsr_read4(OCP85XX_BSTAR); /* * Enable and configure time base on new CPU. */ /* Set TB clock source to platform clock / 32 */ reg = ccsr_read4(CCSR_CTBCKSELR); ccsr_write4(CCSR_CTBCKSELR, reg & ~(1 << pc->pc_cpuid)); /* Enable TB */ reg = ccsr_read4(CCSR_CTBENR); ccsr_write4(CCSR_CTBENR, reg | (1 << pc->pc_cpuid)); } else { /* * Set BPTR to the physical address of the boot page */ bptr = (bptr >> 12) | 0x80000000u; ccsr_write4(OCP85XX_BPTR, bptr); __asm __volatile("isync; msync"); } /* * Release AP from hold-off state */ reg = ccsr_read4(brr); ccsr_write4(brr, reg | (1 << cpuid)); __asm __volatile("isync; msync"); spin_wait: timeout = 500; while (!pc->pc_awake && timeout--) DELAY(1000); /* wait 1ms */ /* * Disable boot page translation so that the 4K page at the default * address (= 0xfffff000) isn't permanently remapped and thus not * usable otherwise. */ if (!epapr_boot) { if (mpc85xx_is_qoriq()) ccsr_write4(OCP85XX_BSTAR, 0); else ccsr_write4(OCP85XX_BPTR, 0); __asm __volatile("isync; msync"); } if (!pc->pc_awake) panic("SMP: CPU %d didn't wake up.\n", pc->pc_cpuid); return ((pc->pc_awake) ? 0 : EBUSY); #else /* No SMP support */ return (ENXIO); #endif } static void mpc85xx_reset(platform_t plat) { /* * Try the dedicated reset register first. * If the SoC doesn't have one, we'll fall * back to using the debug control register. */ ccsr_write4(OCP85XX_RSTCR, 2); /* Clear DBCR0, disables debug interrupts and events. */ mtspr(SPR_DBCR0, 0); __asm __volatile("isync"); /* Enable Debug Interrupts in MSR. */ mtmsr(mfmsr() | PSL_DE); /* Enable debug interrupts and issue reset. */ mtspr(SPR_DBCR0, mfspr(SPR_DBCR0) | DBCR0_IDM | DBCR0_RST_SYSTEM); printf("Reset failed...\n"); while (1) ; } static void mpc85xx_idle(platform_t plat, int cpu) { uint32_t reg; if (mpc85xx_is_qoriq()) { reg = ccsr_read4(OCP85XX_RCPM_CDOZCR); ccsr_write4(OCP85XX_RCPM_CDOZCR, reg | (1 << cpu)); ccsr_read4(OCP85XX_RCPM_CDOZCR); } else { reg = mfmsr(); /* Freescale E500 core RM section 6.4.1. */ __asm __volatile("msync; mtmsr %0; isync" :: "r" (reg | PSL_WE)); } } static int mpc85xx_idle_wakeup(platform_t plat, int cpu) { uint32_t reg; if (mpc85xx_is_qoriq()) { reg = ccsr_read4(OCP85XX_RCPM_CDOZCR); ccsr_write4(OCP85XX_RCPM_CDOZCR, reg & ~(1 << cpu)); ccsr_read4(OCP85XX_RCPM_CDOZCR); return (1); } return (0); } Index: projects/clang390-import/sys/sys/bus.h =================================================================== --- projects/clang390-import/sys/sys/bus.h (revision 305080) +++ projects/clang390-import/sys/sys/bus.h (revision 305081) @@ -1,933 +1,937 @@ /*- * Copyright (c) 1997,1998,2003 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_BUS_H_ #define _SYS_BUS_H_ #include #include #include #include /** * @defgroup NEWBUS newbus - a generic framework for managing devices * @{ */ /** * @brief Interface information structure. */ struct u_businfo { int ub_version; /**< @brief interface version */ #define BUS_USER_VERSION 1 int ub_generation; /**< @brief generation count */ }; /** * @brief State of the device. */ typedef enum device_state { DS_NOTPRESENT = 10, /**< @brief not probed or probe failed */ DS_ALIVE = 20, /**< @brief probe succeeded */ DS_ATTACHING = 25, /**< @brief currently attaching */ DS_ATTACHED = 30, /**< @brief attach method called */ DS_BUSY = 40 /**< @brief device is open */ } device_state_t; /** * @brief Device information exported to userspace. */ struct u_device { uintptr_t dv_handle; uintptr_t dv_parent; char dv_name[32]; /**< @brief Name of device in tree. */ char dv_desc[32]; /**< @brief Driver description */ char dv_drivername[32]; /**< @brief Driver name */ char dv_pnpinfo[128]; /**< @brief Plug and play info */ char dv_location[128]; /**< @brief Where is the device? */ uint32_t dv_devflags; /**< @brief API Flags for device */ uint16_t dv_flags; /**< @brief flags for dev state */ device_state_t dv_state; /**< @brief State of attachment */ /* XXX more driver info? */ }; /* Flags exported via dv_flags. */ #define DF_ENABLED 0x01 /* device should be probed/attached */ #define DF_FIXEDCLASS 0x02 /* devclass specified at create time */ #define DF_WILDCARD 0x04 /* unit was originally wildcard */ #define DF_DESCMALLOCED 0x08 /* description was malloced */ #define DF_QUIET 0x10 /* don't print verbose attach message */ #define DF_DONENOMATCH 0x20 /* don't execute DEVICE_NOMATCH again */ #define DF_EXTERNALSOFTC 0x40 /* softc not allocated by us */ #define DF_REBID 0x80 /* Can rebid after attach */ #define DF_SUSPENDED 0x100 /* Device is suspended. */ /** * @brief Device request structure used for ioctl's. * * Used for ioctl's on /dev/devctl2. All device ioctl's * must have parameter definitions which begin with dr_name. */ struct devreq_buffer { void *buffer; size_t length; }; struct devreq { char dr_name[128]; int dr_flags; /* request-specific flags */ union { struct devreq_buffer dru_buffer; void *dru_data; } dr_dru; #define dr_buffer dr_dru.dru_buffer /* variable-sized buffer */ #define dr_data dr_dru.dru_data /* fixed-size buffer */ }; #define DEV_ATTACH _IOW('D', 1, struct devreq) #define DEV_DETACH _IOW('D', 2, struct devreq) #define DEV_ENABLE _IOW('D', 3, struct devreq) #define DEV_DISABLE _IOW('D', 4, struct devreq) #define DEV_SUSPEND _IOW('D', 5, struct devreq) #define DEV_RESUME _IOW('D', 6, struct devreq) #define DEV_SET_DRIVER _IOW('D', 7, struct devreq) +#define DEV_CLEAR_DRIVER _IOW('D', 8, struct devreq) #define DEV_RESCAN _IOW('D', 9, struct devreq) #define DEV_DELETE _IOW('D', 10, struct devreq) /* Flags for DEV_DETACH and DEV_DISABLE. */ #define DEVF_FORCE_DETACH 0x0000001 /* Flags for DEV_SET_DRIVER. */ #define DEVF_SET_DRIVER_DETACH 0x0000001 /* Detach existing driver. */ + +/* Flags for DEV_CLEAR_DRIVER. */ +#define DEVF_CLEAR_DRIVER_DETACH 0x0000001 /* Detach existing driver. */ /* Flags for DEV_DELETE. */ #define DEVF_FORCE_DELETE 0x0000001 #ifdef _KERNEL #include #include /** * devctl hooks. Typically one should use the devctl_notify * hook to send the message. However, devctl_queue_data is also * included in case devctl_notify isn't sufficiently general. */ boolean_t devctl_process_running(void); void devctl_notify_f(const char *__system, const char *__subsystem, const char *__type, const char *__data, int __flags); void devctl_notify(const char *__system, const char *__subsystem, const char *__type, const char *__data); void devctl_queue_data_f(char *__data, int __flags); void devctl_queue_data(char *__data); void devctl_safe_quote(char *__dst, const char *__src, size_t len); /** * Device name parsers. Hook to allow device enumerators to map * scheme-specific names to a device. */ typedef void (*dev_lookup_fn)(void *arg, const char *name, device_t *result); EVENTHANDLER_DECLARE(dev_lookup, dev_lookup_fn); /** * @brief A device driver (included mainly for compatibility with * FreeBSD 4.x). */ typedef struct kobj_class driver_t; /** * @brief A device class * * The devclass object has two main functions in the system. The first * is to manage the allocation of unit numbers for device instances * and the second is to hold the list of device drivers for a * particular bus type. Each devclass has a name and there cannot be * two devclasses with the same name. This ensures that unique unit * numbers are allocated to device instances. * * Drivers that support several different bus attachments (e.g. isa, * pci, pccard) should all use the same devclass to ensure that unit * numbers do not conflict. * * Each devclass may also have a parent devclass. This is used when * searching for device drivers to allow a form of inheritance. When * matching drivers with devices, first the driver list of the parent * device's devclass is searched. If no driver is found in that list, * the search continues in the parent devclass (if any). */ typedef struct devclass *devclass_t; /** * @brief A device method */ #define device_method_t kobj_method_t /** * @brief Driver interrupt filter return values * * If a driver provides an interrupt filter routine it must return an * integer consisting of oring together zero or more of the following * flags: * * FILTER_STRAY - this device did not trigger the interrupt * FILTER_HANDLED - the interrupt has been fully handled and can be EOId * FILTER_SCHEDULE_THREAD - the threaded interrupt handler should be * scheduled to execute * * If the driver does not provide a filter, then the interrupt code will * act is if the filter had returned FILTER_SCHEDULE_THREAD. Note that it * is illegal to specify any other flag with FILTER_STRAY and that it is * illegal to not specify either of FILTER_HANDLED or FILTER_SCHEDULE_THREAD * if FILTER_STRAY is not specified. */ #define FILTER_STRAY 0x01 #define FILTER_HANDLED 0x02 #define FILTER_SCHEDULE_THREAD 0x04 /** * @brief Driver interrupt service routines * * The filter routine is run in primary interrupt context and may not * block or use regular mutexes. It may only use spin mutexes for * synchronization. The filter may either completely handle the * interrupt or it may perform some of the work and defer more * expensive work to the regular interrupt handler. If a filter * routine is not registered by the driver, then the regular interrupt * handler is always used to handle interrupts from this device. * * The regular interrupt handler executes in its own thread context * and may use regular mutexes. However, it is prohibited from * sleeping on a sleep queue. */ typedef int driver_filter_t(void*); typedef void driver_intr_t(void*); /** * @brief Interrupt type bits. * * These flags are used both by newbus interrupt * registration (nexus.c) and also in struct intrec, which defines * interrupt properties. * * XXX We should probably revisit this and remove the vestiges of the * spls implicit in names like INTR_TYPE_TTY. In the meantime, don't * confuse things by renaming them (Grog, 18 July 2000). * * Buses which do interrupt remapping will want to change their type * to reflect what sort of devices are underneath. */ enum intr_type { INTR_TYPE_TTY = 1, INTR_TYPE_BIO = 2, INTR_TYPE_NET = 4, INTR_TYPE_CAM = 8, INTR_TYPE_MISC = 16, INTR_TYPE_CLK = 32, INTR_TYPE_AV = 64, INTR_EXCL = 256, /* exclusive interrupt */ INTR_MPSAFE = 512, /* this interrupt is SMP safe */ INTR_ENTROPY = 1024, /* this interrupt provides entropy */ INTR_MD1 = 4096, /* flag reserved for MD use */ INTR_MD2 = 8192, /* flag reserved for MD use */ INTR_MD3 = 16384, /* flag reserved for MD use */ INTR_MD4 = 32768 /* flag reserved for MD use */ }; enum intr_trigger { INTR_TRIGGER_CONFORM = 0, INTR_TRIGGER_EDGE = 1, INTR_TRIGGER_LEVEL = 2 }; enum intr_polarity { INTR_POLARITY_CONFORM = 0, INTR_POLARITY_HIGH = 1, INTR_POLARITY_LOW = 2 }; /** * CPU sets supported by bus_get_cpus(). Note that not all sets may be * supported for a given device. If a request is not supported by a * device (or its parents), then bus_get_cpus() will fail with EINVAL. */ enum cpu_sets { LOCAL_CPUS = 0, INTR_CPUS }; typedef int (*devop_t)(void); /** * @brief This structure is deprecated. * * Use the kobj(9) macro DEFINE_CLASS to * declare classes which implement device drivers. */ struct driver { KOBJ_CLASS_FIELDS; }; /** * @brief A resource mapping. */ struct resource_map { bus_space_tag_t r_bustag; bus_space_handle_t r_bushandle; bus_size_t r_size; void *r_vaddr; }; /** * @brief Optional properties of a resource mapping request. */ struct resource_map_request { size_t size; rman_res_t offset; rman_res_t length; vm_memattr_t memattr; }; void resource_init_map_request_impl(struct resource_map_request *_args, size_t _sz); #define resource_init_map_request(rmr) \ resource_init_map_request_impl((rmr), sizeof(*(rmr))) /* * Definitions for drivers which need to keep simple lists of resources * for their child devices. */ struct resource; /** * @brief An entry for a single resource in a resource list. */ struct resource_list_entry { STAILQ_ENTRY(resource_list_entry) link; int type; /**< @brief type argument to alloc_resource */ int rid; /**< @brief resource identifier */ int flags; /**< @brief resource flags */ struct resource *res; /**< @brief the real resource when allocated */ rman_res_t start; /**< @brief start of resource range */ rman_res_t end; /**< @brief end of resource range */ rman_res_t count; /**< @brief count within range */ }; STAILQ_HEAD(resource_list, resource_list_entry); #define RLE_RESERVED 0x0001 /* Reserved by the parent bus. */ #define RLE_ALLOCATED 0x0002 /* Reserved resource is allocated. */ #define RLE_PREFETCH 0x0004 /* Resource is a prefetch range. */ void resource_list_init(struct resource_list *rl); void resource_list_free(struct resource_list *rl); struct resource_list_entry * resource_list_add(struct resource_list *rl, int type, int rid, rman_res_t start, rman_res_t end, rman_res_t count); int resource_list_add_next(struct resource_list *rl, int type, rman_res_t start, rman_res_t end, rman_res_t count); int resource_list_busy(struct resource_list *rl, int type, int rid); int resource_list_reserved(struct resource_list *rl, int type, int rid); struct resource_list_entry* resource_list_find(struct resource_list *rl, int type, int rid); void resource_list_delete(struct resource_list *rl, int type, int rid); struct resource * resource_list_alloc(struct resource_list *rl, device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags); int resource_list_release(struct resource_list *rl, device_t bus, device_t child, int type, int rid, struct resource *res); int resource_list_release_active(struct resource_list *rl, device_t bus, device_t child, int type); struct resource * resource_list_reserve(struct resource_list *rl, device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags); int resource_list_unreserve(struct resource_list *rl, device_t bus, device_t child, int type, int rid); void resource_list_purge(struct resource_list *rl); int resource_list_print_type(struct resource_list *rl, const char *name, int type, const char *format); /* * The root bus, to which all top-level busses are attached. */ extern device_t root_bus; extern devclass_t root_devclass; void root_bus_configure(void); /* * Useful functions for implementing busses. */ int bus_generic_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *r); device_t bus_generic_add_child(device_t dev, u_int order, const char *name, int unit); int bus_generic_adjust_resource(device_t bus, device_t child, int type, struct resource *r, rman_res_t start, rman_res_t end); struct resource * bus_generic_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags); int bus_generic_attach(device_t dev); int bus_generic_bind_intr(device_t dev, device_t child, struct resource *irq, int cpu); int bus_generic_child_present(device_t dev, device_t child); int bus_generic_config_intr(device_t, int, enum intr_trigger, enum intr_polarity); int bus_generic_describe_intr(device_t dev, device_t child, struct resource *irq, void *cookie, const char *descr); int bus_generic_deactivate_resource(device_t dev, device_t child, int type, int rid, struct resource *r); int bus_generic_detach(device_t dev); void bus_generic_driver_added(device_t dev, driver_t *driver); int bus_generic_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsize, struct _cpuset *cpuset); bus_dma_tag_t bus_generic_get_dma_tag(device_t dev, device_t child); bus_space_tag_t bus_generic_get_bus_tag(device_t dev, device_t child); int bus_generic_get_domain(device_t dev, device_t child, int *domain); struct resource_list * bus_generic_get_resource_list (device_t, device_t); int bus_generic_map_resource(device_t dev, device_t child, int type, struct resource *r, struct resource_map_request *args, struct resource_map *map); void bus_generic_new_pass(device_t dev); int bus_print_child_header(device_t dev, device_t child); int bus_print_child_domain(device_t dev, device_t child); int bus_print_child_footer(device_t dev, device_t child); int bus_generic_print_child(device_t dev, device_t child); int bus_generic_probe(device_t dev); int bus_generic_read_ivar(device_t dev, device_t child, int which, uintptr_t *result); int bus_generic_release_resource(device_t bus, device_t child, int type, int rid, struct resource *r); int bus_generic_resume(device_t dev); int bus_generic_resume_child(device_t dev, device_t child); int bus_generic_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep); struct resource * bus_generic_rl_alloc_resource (device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); void bus_generic_rl_delete_resource (device_t, device_t, int, int); int bus_generic_rl_get_resource (device_t, device_t, int, int, rman_res_t *, rman_res_t *); int bus_generic_rl_set_resource (device_t, device_t, int, int, rman_res_t, rman_res_t); int bus_generic_rl_release_resource (device_t, device_t, int, int, struct resource *); int bus_generic_shutdown(device_t dev); int bus_generic_suspend(device_t dev); int bus_generic_suspend_child(device_t dev, device_t child); int bus_generic_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie); int bus_generic_unmap_resource(device_t dev, device_t child, int type, struct resource *r, struct resource_map *map); int bus_generic_write_ivar(device_t dev, device_t child, int which, uintptr_t value); int bus_null_rescan(device_t dev); /* * Wrapper functions for the BUS_*_RESOURCE methods to make client code * a little simpler. */ struct resource_spec { int type; int rid; int flags; }; int bus_alloc_resources(device_t dev, struct resource_spec *rs, struct resource **res); void bus_release_resources(device_t dev, const struct resource_spec *rs, struct resource **res); int bus_adjust_resource(device_t child, int type, struct resource *r, rman_res_t start, rman_res_t end); struct resource *bus_alloc_resource(device_t dev, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags); int bus_activate_resource(device_t dev, int type, int rid, struct resource *r); int bus_deactivate_resource(device_t dev, int type, int rid, struct resource *r); int bus_map_resource(device_t dev, int type, struct resource *r, struct resource_map_request *args, struct resource_map *map); int bus_unmap_resource(device_t dev, int type, struct resource *r, struct resource_map *map); int bus_get_cpus(device_t dev, enum cpu_sets op, size_t setsize, struct _cpuset *cpuset); bus_dma_tag_t bus_get_dma_tag(device_t dev); bus_space_tag_t bus_get_bus_tag(device_t dev); int bus_get_domain(device_t dev, int *domain); int bus_release_resource(device_t dev, int type, int rid, struct resource *r); int bus_free_resource(device_t dev, int type, struct resource *r); int bus_setup_intr(device_t dev, struct resource *r, int flags, driver_filter_t filter, driver_intr_t handler, void *arg, void **cookiep); int bus_teardown_intr(device_t dev, struct resource *r, void *cookie); int bus_bind_intr(device_t dev, struct resource *r, int cpu); int bus_describe_intr(device_t dev, struct resource *irq, void *cookie, const char *fmt, ...) __printflike(4, 5); int bus_set_resource(device_t dev, int type, int rid, rman_res_t start, rman_res_t count); int bus_get_resource(device_t dev, int type, int rid, rman_res_t *startp, rman_res_t *countp); rman_res_t bus_get_resource_start(device_t dev, int type, int rid); rman_res_t bus_get_resource_count(device_t dev, int type, int rid); void bus_delete_resource(device_t dev, int type, int rid); int bus_child_present(device_t child); int bus_child_pnpinfo_str(device_t child, char *buf, size_t buflen); int bus_child_location_str(device_t child, char *buf, size_t buflen); void bus_enumerate_hinted_children(device_t bus); static __inline struct resource * bus_alloc_resource_any(device_t dev, int type, int *rid, u_int flags) { return (bus_alloc_resource(dev, type, rid, 0, ~0, 1, flags)); } static __inline struct resource * bus_alloc_resource_anywhere(device_t dev, int type, int *rid, rman_res_t count, u_int flags) { return (bus_alloc_resource(dev, type, rid, 0, ~0, count, flags)); } /* * Access functions for device. */ device_t device_add_child(device_t dev, const char *name, int unit); device_t device_add_child_ordered(device_t dev, u_int order, const char *name, int unit); void device_busy(device_t dev); int device_delete_child(device_t dev, device_t child); int device_delete_children(device_t dev); int device_attach(device_t dev); int device_detach(device_t dev); void device_disable(device_t dev); void device_enable(device_t dev); device_t device_find_child(device_t dev, const char *classname, int unit); const char *device_get_desc(device_t dev); devclass_t device_get_devclass(device_t dev); driver_t *device_get_driver(device_t dev); u_int32_t device_get_flags(device_t dev); device_t device_get_parent(device_t dev); int device_get_children(device_t dev, device_t **listp, int *countp); void *device_get_ivars(device_t dev); void device_set_ivars(device_t dev, void *ivars); const char *device_get_name(device_t dev); const char *device_get_nameunit(device_t dev); void *device_get_softc(device_t dev); device_state_t device_get_state(device_t dev); int device_get_unit(device_t dev); struct sysctl_ctx_list *device_get_sysctl_ctx(device_t dev); struct sysctl_oid *device_get_sysctl_tree(device_t dev); int device_is_alive(device_t dev); /* did probe succeed? */ int device_is_attached(device_t dev); /* did attach succeed? */ int device_is_enabled(device_t dev); int device_is_suspended(device_t dev); int device_is_quiet(device_t dev); device_t device_lookup_by_name(const char *name); int device_print_prettyname(device_t dev); int device_printf(device_t dev, const char *, ...) __printflike(2, 3); int device_probe(device_t dev); int device_probe_and_attach(device_t dev); int device_probe_child(device_t bus, device_t dev); int device_quiesce(device_t dev); void device_quiet(device_t dev); void device_set_desc(device_t dev, const char* desc); void device_set_desc_copy(device_t dev, const char* desc); int device_set_devclass(device_t dev, const char *classname); int device_set_devclass_fixed(device_t dev, const char *classname); int device_set_driver(device_t dev, driver_t *driver); void device_set_flags(device_t dev, u_int32_t flags); void device_set_softc(device_t dev, void *softc); void device_free_softc(void *softc); void device_claim_softc(device_t dev); int device_set_unit(device_t dev, int unit); /* XXX DONT USE XXX */ int device_shutdown(device_t dev); void device_unbusy(device_t dev); void device_verbose(device_t dev); /* * Access functions for devclass. */ int devclass_add_driver(devclass_t dc, driver_t *driver, int pass, devclass_t *dcp); devclass_t devclass_create(const char *classname); int devclass_delete_driver(devclass_t busclass, driver_t *driver); devclass_t devclass_find(const char *classname); const char *devclass_get_name(devclass_t dc); device_t devclass_get_device(devclass_t dc, int unit); void *devclass_get_softc(devclass_t dc, int unit); int devclass_get_devices(devclass_t dc, device_t **listp, int *countp); int devclass_get_drivers(devclass_t dc, driver_t ***listp, int *countp); int devclass_get_count(devclass_t dc); int devclass_get_maxunit(devclass_t dc); int devclass_find_free_unit(devclass_t dc, int unit); void devclass_set_parent(devclass_t dc, devclass_t pdc); devclass_t devclass_get_parent(devclass_t dc); struct sysctl_ctx_list *devclass_get_sysctl_ctx(devclass_t dc); struct sysctl_oid *devclass_get_sysctl_tree(devclass_t dc); /* * Access functions for device resources. */ int resource_int_value(const char *name, int unit, const char *resname, int *result); int resource_long_value(const char *name, int unit, const char *resname, long *result); int resource_string_value(const char *name, int unit, const char *resname, const char **result); int resource_disabled(const char *name, int unit); int resource_find_match(int *anchor, const char **name, int *unit, const char *resname, const char *value); int resource_find_dev(int *anchor, const char *name, int *unit, const char *resname, const char *value); int resource_set_int(const char *name, int unit, const char *resname, int value); int resource_set_long(const char *name, int unit, const char *resname, long value); int resource_set_string(const char *name, int unit, const char *resname, const char *value); int resource_unset_value(const char *name, int unit, const char *resname); /* * Functions for maintaining and checking consistency of * bus information exported to userspace. */ int bus_data_generation_check(int generation); void bus_data_generation_update(void); /** * Some convenience defines for probe routines to return. These are just * suggested values, and there's nothing magical about them. * BUS_PROBE_SPECIFIC is for devices that cannot be reprobed, and that no * possible other driver may exist (typically legacy drivers who don't fallow * all the rules, or special needs drivers). BUS_PROBE_VENDOR is the * suggested value that vendor supplied drivers use. This is for source or * binary drivers that are not yet integrated into the FreeBSD tree. Its use * in the base OS is prohibited. BUS_PROBE_DEFAULT is the normal return value * for drivers to use. It is intended that nearly all of the drivers in the * tree should return this value. BUS_PROBE_LOW_PRIORITY are for drivers that * have special requirements like when there are two drivers that support * overlapping series of hardware devices. In this case the one that supports * the older part of the line would return this value, while the one that * supports the newer ones would return BUS_PROBE_DEFAULT. BUS_PROBE_GENERIC * is for drivers that wish to have a generic form and a specialized form, * like is done with the pci bus and the acpi pci bus. BUS_PROBE_HOOVER is * for those busses that implement a generic device place-holder for devices on * the bus that have no more specific driver for them (aka ugen). * BUS_PROBE_NOWILDCARD or lower means that the device isn't really bidding * for a device node, but accepts only devices that its parent has told it * use this driver. */ #define BUS_PROBE_SPECIFIC 0 /* Only I can use this device */ #define BUS_PROBE_VENDOR (-10) /* Vendor supplied driver */ #define BUS_PROBE_DEFAULT (-20) /* Base OS default driver */ #define BUS_PROBE_LOW_PRIORITY (-40) /* Older, less desirable drivers */ #define BUS_PROBE_GENERIC (-100) /* generic driver for dev */ #define BUS_PROBE_HOOVER (-1000000) /* Driver for any dev on bus */ #define BUS_PROBE_NOWILDCARD (-2000000000) /* No wildcard device matches */ /** * During boot, the device tree is scanned multiple times. Each scan, * or pass, drivers may be attached to devices. Each driver * attachment is assigned a pass number. Drivers may only probe and * attach to devices if their pass number is less than or equal to the * current system-wide pass number. The default pass is the last pass * and is used by most drivers. Drivers needed by the scheduler are * probed in earlier passes. */ #define BUS_PASS_ROOT 0 /* Used to attach root0. */ #define BUS_PASS_BUS 10 /* Busses and bridges. */ #define BUS_PASS_CPU 20 /* CPU devices. */ #define BUS_PASS_RESOURCE 30 /* Resource discovery. */ #define BUS_PASS_INTERRUPT 40 /* Interrupt controllers. */ #define BUS_PASS_TIMER 50 /* Timers and clocks. */ #define BUS_PASS_SCHEDULER 60 /* Start scheduler. */ #define BUS_PASS_DEFAULT __INT_MAX /* Everything else. */ #define BUS_PASS_ORDER_FIRST 0 #define BUS_PASS_ORDER_EARLY 2 #define BUS_PASS_ORDER_MIDDLE 5 #define BUS_PASS_ORDER_LATE 7 #define BUS_PASS_ORDER_LAST 9 extern int bus_current_pass; void bus_set_pass(int pass); /** * Shorthands for constructing method tables. */ #define DEVMETHOD KOBJMETHOD #define DEVMETHOD_END KOBJMETHOD_END /* * Some common device interfaces. */ #include "device_if.h" #include "bus_if.h" struct module; int driver_module_handler(struct module *, int, void *); /** * Module support for automatically adding drivers to busses. */ struct driver_module_data { int (*dmd_chainevh)(struct module *, int, void *); void *dmd_chainarg; const char *dmd_busname; kobj_class_t dmd_driver; devclass_t *dmd_devclass; int dmd_pass; }; #define EARLY_DRIVER_MODULE_ORDERED(name, busname, driver, devclass, \ evh, arg, order, pass) \ \ static struct driver_module_data name##_##busname##_driver_mod = { \ evh, arg, \ #busname, \ (kobj_class_t) &driver, \ &devclass, \ pass \ }; \ \ static moduledata_t name##_##busname##_mod = { \ #busname "/" #name, \ driver_module_handler, \ &name##_##busname##_driver_mod \ }; \ DECLARE_MODULE(name##_##busname, name##_##busname##_mod, \ SI_SUB_DRIVERS, order) #define EARLY_DRIVER_MODULE(name, busname, driver, devclass, evh, arg, pass) \ EARLY_DRIVER_MODULE_ORDERED(name, busname, driver, devclass, \ evh, arg, SI_ORDER_MIDDLE, pass) #define DRIVER_MODULE_ORDERED(name, busname, driver, devclass, evh, arg,\ order) \ EARLY_DRIVER_MODULE_ORDERED(name, busname, driver, devclass, \ evh, arg, order, BUS_PASS_DEFAULT) #define DRIVER_MODULE(name, busname, driver, devclass, evh, arg) \ EARLY_DRIVER_MODULE(name, busname, driver, devclass, evh, arg, \ BUS_PASS_DEFAULT) /** * Generic ivar accessor generation macros for bus drivers */ #define __BUS_ACCESSOR(varp, var, ivarp, ivar, type) \ \ static __inline type varp ## _get_ ## var(device_t dev) \ { \ uintptr_t v; \ BUS_READ_IVAR(device_get_parent(dev), dev, \ ivarp ## _IVAR_ ## ivar, &v); \ return ((type) v); \ } \ \ static __inline void varp ## _set_ ## var(device_t dev, type t) \ { \ uintptr_t v = (uintptr_t) t; \ BUS_WRITE_IVAR(device_get_parent(dev), dev, \ ivarp ## _IVAR_ ## ivar, v); \ } /** * Shorthand macros, taking resource argument * Generated with sys/tools/bus_macro.sh */ #define bus_barrier(r, o, l, f) \ bus_space_barrier((r)->r_bustag, (r)->r_bushandle, (o), (l), (f)) #define bus_read_1(r, o) \ bus_space_read_1((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_1(r, o, d, c) \ bus_space_read_multi_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_1(r, o, d, c) \ bus_space_read_region_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_1(r, o, v, c) \ bus_space_set_multi_1((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_1(r, o, v, c) \ bus_space_set_region_1((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_1(r, o, v) \ bus_space_write_1((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_1(r, o, d, c) \ bus_space_write_multi_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_1(r, o, d, c) \ bus_space_write_region_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_stream_1(r, o) \ bus_space_read_stream_1((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_stream_1(r, o, d, c) \ bus_space_read_multi_stream_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_stream_1(r, o, d, c) \ bus_space_read_region_stream_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_stream_1(r, o, v, c) \ bus_space_set_multi_stream_1((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_stream_1(r, o, v, c) \ bus_space_set_region_stream_1((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_stream_1(r, o, v) \ bus_space_write_stream_1((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_stream_1(r, o, d, c) \ bus_space_write_multi_stream_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_stream_1(r, o, d, c) \ bus_space_write_region_stream_1((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_2(r, o) \ bus_space_read_2((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_2(r, o, d, c) \ bus_space_read_multi_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_2(r, o, d, c) \ bus_space_read_region_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_2(r, o, v, c) \ bus_space_set_multi_2((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_2(r, o, v, c) \ bus_space_set_region_2((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_2(r, o, v) \ bus_space_write_2((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_2(r, o, d, c) \ bus_space_write_multi_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_2(r, o, d, c) \ bus_space_write_region_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_stream_2(r, o) \ bus_space_read_stream_2((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_stream_2(r, o, d, c) \ bus_space_read_multi_stream_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_stream_2(r, o, d, c) \ bus_space_read_region_stream_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_stream_2(r, o, v, c) \ bus_space_set_multi_stream_2((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_stream_2(r, o, v, c) \ bus_space_set_region_stream_2((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_stream_2(r, o, v) \ bus_space_write_stream_2((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_stream_2(r, o, d, c) \ bus_space_write_multi_stream_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_stream_2(r, o, d, c) \ bus_space_write_region_stream_2((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_4(r, o) \ bus_space_read_4((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_4(r, o, d, c) \ bus_space_read_multi_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_4(r, o, d, c) \ bus_space_read_region_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_4(r, o, v, c) \ bus_space_set_multi_4((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_4(r, o, v, c) \ bus_space_set_region_4((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_4(r, o, v) \ bus_space_write_4((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_4(r, o, d, c) \ bus_space_write_multi_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_4(r, o, d, c) \ bus_space_write_region_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_stream_4(r, o) \ bus_space_read_stream_4((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_stream_4(r, o, d, c) \ bus_space_read_multi_stream_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_stream_4(r, o, d, c) \ bus_space_read_region_stream_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_stream_4(r, o, v, c) \ bus_space_set_multi_stream_4((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_stream_4(r, o, v, c) \ bus_space_set_region_stream_4((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_stream_4(r, o, v) \ bus_space_write_stream_4((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_stream_4(r, o, d, c) \ bus_space_write_multi_stream_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_stream_4(r, o, d, c) \ bus_space_write_region_stream_4((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_8(r, o) \ bus_space_read_8((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_8(r, o, d, c) \ bus_space_read_multi_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_8(r, o, d, c) \ bus_space_read_region_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_8(r, o, v, c) \ bus_space_set_multi_8((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_8(r, o, v, c) \ bus_space_set_region_8((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_8(r, o, v) \ bus_space_write_8((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_8(r, o, d, c) \ bus_space_write_multi_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_8(r, o, d, c) \ bus_space_write_region_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_stream_8(r, o) \ bus_space_read_stream_8((r)->r_bustag, (r)->r_bushandle, (o)) #define bus_read_multi_stream_8(r, o, d, c) \ bus_space_read_multi_stream_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_read_region_stream_8(r, o, d, c) \ bus_space_read_region_stream_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_set_multi_stream_8(r, o, v, c) \ bus_space_set_multi_stream_8((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_set_region_stream_8(r, o, v, c) \ bus_space_set_region_stream_8((r)->r_bustag, (r)->r_bushandle, (o), (v), (c)) #define bus_write_stream_8(r, o, v) \ bus_space_write_stream_8((r)->r_bustag, (r)->r_bushandle, (o), (v)) #define bus_write_multi_stream_8(r, o, d, c) \ bus_space_write_multi_stream_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #define bus_write_region_stream_8(r, o, d, c) \ bus_space_write_region_stream_8((r)->r_bustag, (r)->r_bushandle, (o), (d), (c)) #endif /* _KERNEL */ #endif /* !_SYS_BUS_H_ */ Index: projects/clang390-import/sys/sys/mbuf.h =================================================================== --- projects/clang390-import/sys/sys/mbuf.h (revision 305080) +++ projects/clang390-import/sys/sys/mbuf.h (revision 305081) @@ -1,1316 +1,1316 @@ /*- * Copyright (c) 1982, 1986, 1988, 1993 * The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)mbuf.h 8.5 (Berkeley) 2/19/95 * $FreeBSD$ */ #ifndef _SYS_MBUF_H_ #define _SYS_MBUF_H_ /* XXX: These includes suck. Sorry! */ #include #ifdef _KERNEL #include #include #ifdef WITNESS #include #endif #endif #ifdef _KERNEL #include #define MBUF_PROBE1(probe, arg0) \ SDT_PROBE1(sdt, , , probe, arg0) #define MBUF_PROBE2(probe, arg0, arg1) \ SDT_PROBE2(sdt, , , probe, arg0, arg1) #define MBUF_PROBE3(probe, arg0, arg1, arg2) \ SDT_PROBE3(sdt, , , probe, arg0, arg1, arg2) #define MBUF_PROBE4(probe, arg0, arg1, arg2, arg3) \ SDT_PROBE4(sdt, , , probe, arg0, arg1, arg2, arg3) #define MBUF_PROBE5(probe, arg0, arg1, arg2, arg3, arg4) \ SDT_PROBE5(sdt, , , probe, arg0, arg1, arg2, arg3, arg4) SDT_PROBE_DECLARE(sdt, , , m__init); SDT_PROBE_DECLARE(sdt, , , m__gethdr); SDT_PROBE_DECLARE(sdt, , , m__get); SDT_PROBE_DECLARE(sdt, , , m__getcl); SDT_PROBE_DECLARE(sdt, , , m__clget); SDT_PROBE_DECLARE(sdt, , , m__cljget); SDT_PROBE_DECLARE(sdt, , , m__cljset); SDT_PROBE_DECLARE(sdt, , , m__free); SDT_PROBE_DECLARE(sdt, , , m__freem); #endif /* _KERNEL */ /* * Mbufs are of a single size, MSIZE (sys/param.h), which includes overhead. * An mbuf may add a single "mbuf cluster" of size MCLBYTES (also in * sys/param.h), which has no additional overhead and is used instead of the * internal data area; this is done when at least MINCLSIZE of data must be * stored. Additionally, it is possible to allocate a separate buffer * externally and attach it to the mbuf in a way similar to that of mbuf * clusters. * * NB: These calculation do not take actual compiler-induced alignment and * padding inside the complete struct mbuf into account. Appropriate * attention is required when changing members of struct mbuf. * * MLEN is data length in a normal mbuf. * MHLEN is data length in an mbuf with pktheader. * MINCLSIZE is a smallest amount of data that should be put into cluster. * * Compile-time assertions in uipc_mbuf.c test these values to ensure that * they are sensible. */ struct mbuf; #define MHSIZE offsetof(struct mbuf, m_dat) #define MPKTHSIZE offsetof(struct mbuf, m_pktdat) #define MLEN ((int)(MSIZE - MHSIZE)) #define MHLEN ((int)(MSIZE - MPKTHSIZE)) #define MINCLSIZE (MHLEN + 1) #ifdef _KERNEL /*- * Macro for type conversion: convert mbuf pointer to data pointer of correct * type: * * mtod(m, t) -- Convert mbuf pointer to data pointer of correct type. * mtodo(m, o) -- Same as above but with offset 'o' into data. */ #define mtod(m, t) ((t)((m)->m_data)) #define mtodo(m, o) ((void *)(((m)->m_data) + (o))) /* * Argument structure passed to UMA routines during mbuf and packet * allocations. */ struct mb_args { int flags; /* Flags for mbuf being allocated */ short type; /* Type of mbuf being allocated */ }; #endif /* _KERNEL */ /* * Packet tag structure (see below for details). */ struct m_tag { SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ u_int16_t m_tag_id; /* Tag ID */ u_int16_t m_tag_len; /* Length of data */ u_int32_t m_tag_cookie; /* ABI/Module ID */ void (*m_tag_free)(struct m_tag *); }; /* * Record/packet header in first mbuf of chain; valid only if M_PKTHDR is set. * Size ILP32: 48 * LP64: 56 * Compile-time assertions in uipc_mbuf.c test these values to ensure that * they are correct. */ struct pkthdr { struct ifnet *rcvif; /* rcv interface */ SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */ int32_t len; /* total packet length */ /* Layer crossing persistent information. */ uint32_t flowid; /* packet's 4-tuple system */ uint64_t csum_flags; /* checksum and offload features */ uint16_t fibnum; /* this packet should use this fib */ uint8_t cosqos; /* class/quality of service */ uint8_t rsstype; /* hash type */ uint8_t l2hlen; /* layer 2 header length */ uint8_t l3hlen; /* layer 3 header length */ uint8_t l4hlen; /* layer 4 header length */ uint8_t l5hlen; /* layer 5 header length */ union { uint8_t eight[8]; uint16_t sixteen[4]; uint32_t thirtytwo[2]; uint64_t sixtyfour[1]; uintptr_t unintptr[1]; void *ptr; } PH_per; /* Layer specific non-persistent local storage for reassembly, etc. */ union { uint8_t eight[8]; uint16_t sixteen[4]; uint32_t thirtytwo[2]; uint64_t sixtyfour[1]; uintptr_t unintptr[1]; void *ptr; } PH_loc; }; #define ether_vtag PH_per.sixteen[0] #define PH_vt PH_per #define vt_nrecs sixteen[0] #define tso_segsz PH_per.sixteen[1] #define lro_nsegs tso_segsz #define csum_phsum PH_per.sixteen[2] #define csum_data PH_per.thirtytwo[1] /* * Description of external storage mapped into mbuf; valid only if M_EXT is * set. * Size ILP32: 28 * LP64: 48 * Compile-time assertions in uipc_mbuf.c test these values to ensure that * they are correct. */ struct m_ext { union { volatile u_int ext_count; /* value of ref count info */ volatile u_int *ext_cnt; /* pointer to ref count info */ }; caddr_t ext_buf; /* start of buffer */ uint32_t ext_size; /* size of buffer, for ext_free */ uint32_t ext_type:8, /* type of external storage */ ext_flags:24; /* external storage mbuf flags */ void (*ext_free) /* free routine if not the usual */ (struct mbuf *, void *, void *); void *ext_arg1; /* optional argument pointer */ void *ext_arg2; /* optional argument pointer */ }; /* * The core of the mbuf object along with some shortcut defines for practical * purposes. */ struct mbuf { /* * Header present at the beginning of every mbuf. * Size ILP32: 24 * LP64: 32 * Compile-time assertions in uipc_mbuf.c test these values to ensure * that they are correct. */ union { /* next buffer in chain */ struct mbuf *m_next; SLIST_ENTRY(mbuf) m_slist; STAILQ_ENTRY(mbuf) m_stailq; }; union { /* next chain in queue/record */ struct mbuf *m_nextpkt; SLIST_ENTRY(mbuf) m_slistpkt; STAILQ_ENTRY(mbuf) m_stailqpkt; }; caddr_t m_data; /* location of data */ int32_t m_len; /* amount of data in this mbuf */ uint32_t m_type:8, /* type of data in this mbuf */ m_flags:24; /* flags; see below */ #if !defined(__LP64__) uint32_t m_pad; /* pad for 64bit alignment */ #endif /* * A set of optional headers (packet header, external storage header) * and internal data storage. Historically, these arrays were sized * to MHLEN (space left after a packet header) and MLEN (space left * after only a regular mbuf header); they are now variable size in * order to support future work on variable-size mbufs. */ union { struct { struct pkthdr m_pkthdr; /* M_PKTHDR set */ union { struct m_ext m_ext; /* M_EXT set */ char m_pktdat[0]; }; }; char m_dat[0]; /* !M_PKTHDR, !M_EXT */ }; }; /* * mbuf flags of global significance and layer crossing. * Those of only protocol/layer specific significance are to be mapped * to M_PROTO[1-12] and cleared at layer handoff boundaries. * NB: Limited to the lower 24 bits. */ #define M_EXT 0x00000001 /* has associated external storage */ #define M_PKTHDR 0x00000002 /* start of record */ #define M_EOR 0x00000004 /* end of record */ #define M_RDONLY 0x00000008 /* associated data is marked read-only */ #define M_BCAST 0x00000010 /* send/received as link-level broadcast */ #define M_MCAST 0x00000020 /* send/received as link-level multicast */ #define M_PROMISC 0x00000040 /* packet was not for us */ #define M_VLANTAG 0x00000080 /* ether_vtag is valid */ #define M_UNUSED_8 0x00000100 /* --available-- */ #define M_NOFREE 0x00000200 /* do not free mbuf, embedded in cluster */ #define M_PROTO1 0x00001000 /* protocol-specific */ #define M_PROTO2 0x00002000 /* protocol-specific */ #define M_PROTO3 0x00004000 /* protocol-specific */ #define M_PROTO4 0x00008000 /* protocol-specific */ #define M_PROTO5 0x00010000 /* protocol-specific */ #define M_PROTO6 0x00020000 /* protocol-specific */ #define M_PROTO7 0x00040000 /* protocol-specific */ #define M_PROTO8 0x00080000 /* protocol-specific */ #define M_PROTO9 0x00100000 /* protocol-specific */ #define M_PROTO10 0x00200000 /* protocol-specific */ #define M_PROTO11 0x00400000 /* protocol-specific */ #define M_PROTO12 0x00800000 /* protocol-specific */ #define MB_DTOR_SKIP 0x1 /* don't pollute the cache by touching a freed mbuf */ /* * Flags to purge when crossing layers. */ #define M_PROTOFLAGS \ (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO4|M_PROTO5|M_PROTO6|M_PROTO7|M_PROTO8|\ M_PROTO9|M_PROTO10|M_PROTO11|M_PROTO12) /* * Flags preserved when copying m_pkthdr. */ #define M_COPYFLAGS \ (M_PKTHDR|M_EOR|M_RDONLY|M_BCAST|M_MCAST|M_PROMISC|M_VLANTAG| \ M_PROTOFLAGS) /* * Mbuf flag description for use with printf(9) %b identifier. */ #define M_FLAG_BITS \ "\20\1M_EXT\2M_PKTHDR\3M_EOR\4M_RDONLY\5M_BCAST\6M_MCAST" \ "\7M_PROMISC\10M_VLANTAG" #define M_FLAG_PROTOBITS \ "\15M_PROTO1\16M_PROTO2\17M_PROTO3\20M_PROTO4\21M_PROTO5" \ "\22M_PROTO6\23M_PROTO7\24M_PROTO8\25M_PROTO9\26M_PROTO10" \ "\27M_PROTO11\30M_PROTO12" #define M_FLAG_PRINTF (M_FLAG_BITS M_FLAG_PROTOBITS) /* * Network interface cards are able to hash protocol fields (such as IPv4 * addresses and TCP port numbers) classify packets into flows. These flows * can then be used to maintain ordering while delivering packets to the OS * via parallel input queues, as well as to provide a stateless affinity * model. NIC drivers can pass up the hash via m->m_pkthdr.flowid, and set * m_flag fields to indicate how the hash should be interpreted by the * network stack. * * Most NICs support RSS, which provides ordering and explicit affinity, and * use the hash m_flag bits to indicate what header fields were covered by * the hash. M_HASHTYPE_OPAQUE and M_HASHTYPE_OPAQUE_HASH can be set by non- * RSS cards or configurations that provide an opaque flow identifier, allowing * for ordering and distribution without explicit affinity. Additionally, * M_HASHTYPE_OPAQUE_HASH indicates that the flow identifier has hash * properties. */ #define M_HASHTYPE_HASHPROP 0x80 /* has hash properties */ #define M_HASHTYPE_HASH(t) (M_HASHTYPE_HASHPROP | (t)) /* Microsoft RSS standard hash types */ #define M_HASHTYPE_NONE 0 #define M_HASHTYPE_RSS_IPV4 M_HASHTYPE_HASH(1) /* IPv4 2-tuple */ #define M_HASHTYPE_RSS_TCP_IPV4 M_HASHTYPE_HASH(2) /* TCPv4 4-tuple */ #define M_HASHTYPE_RSS_IPV6 M_HASHTYPE_HASH(3) /* IPv6 2-tuple */ #define M_HASHTYPE_RSS_TCP_IPV6 M_HASHTYPE_HASH(4) /* TCPv6 4-tuple */ #define M_HASHTYPE_RSS_IPV6_EX M_HASHTYPE_HASH(5) /* IPv6 2-tuple + * ext hdrs */ #define M_HASHTYPE_RSS_TCP_IPV6_EX M_HASHTYPE_HASH(6) /* TCPv6 4-tiple + * ext hdrs */ /* Non-standard RSS hash types */ #define M_HASHTYPE_RSS_UDP_IPV4 M_HASHTYPE_HASH(7) /* IPv4 UDP 4-tuple*/ #define M_HASHTYPE_RSS_UDP_IPV4_EX M_HASHTYPE_HASH(8) /* IPv4 UDP 4-tuple + * ext hdrs */ #define M_HASHTYPE_RSS_UDP_IPV6 M_HASHTYPE_HASH(9) /* IPv6 UDP 4-tuple*/ #define M_HASHTYPE_RSS_UDP_IPV6_EX M_HASHTYPE_HASH(10)/* IPv6 UDP 4-tuple + * ext hdrs */ #define M_HASHTYPE_OPAQUE 63 /* ordering, not affinity */ #define M_HASHTYPE_OPAQUE_HASH M_HASHTYPE_HASH(M_HASHTYPE_OPAQUE) /* ordering+hash, not affinity*/ #define M_HASHTYPE_CLEAR(m) ((m)->m_pkthdr.rsstype = 0) #define M_HASHTYPE_GET(m) ((m)->m_pkthdr.rsstype) #define M_HASHTYPE_SET(m, v) ((m)->m_pkthdr.rsstype = (v)) #define M_HASHTYPE_TEST(m, v) (M_HASHTYPE_GET(m) == (v)) #define M_HASHTYPE_ISHASH(m) (M_HASHTYPE_GET(m) & M_HASHTYPE_HASHPROP) /* * COS/QOS class and quality of service tags. * It uses DSCP code points as base. */ #define QOS_DSCP_CS0 0x00 #define QOS_DSCP_DEF QOS_DSCP_CS0 #define QOS_DSCP_CS1 0x20 #define QOS_DSCP_AF11 0x28 #define QOS_DSCP_AF12 0x30 #define QOS_DSCP_AF13 0x38 #define QOS_DSCP_CS2 0x40 #define QOS_DSCP_AF21 0x48 #define QOS_DSCP_AF22 0x50 #define QOS_DSCP_AF23 0x58 #define QOS_DSCP_CS3 0x60 #define QOS_DSCP_AF31 0x68 #define QOS_DSCP_AF32 0x70 #define QOS_DSCP_AF33 0x78 #define QOS_DSCP_CS4 0x80 #define QOS_DSCP_AF41 0x88 #define QOS_DSCP_AF42 0x90 #define QOS_DSCP_AF43 0x98 #define QOS_DSCP_CS5 0xa0 #define QOS_DSCP_EF 0xb8 #define QOS_DSCP_CS6 0xc0 #define QOS_DSCP_CS7 0xe0 /* * External mbuf storage buffer types. */ #define EXT_CLUSTER 1 /* mbuf cluster */ #define EXT_SFBUF 2 /* sendfile(2)'s sf_buf */ #define EXT_JUMBOP 3 /* jumbo cluster page sized */ #define EXT_JUMBO9 4 /* jumbo cluster 9216 bytes */ #define EXT_JUMBO16 5 /* jumbo cluster 16184 bytes */ #define EXT_PACKET 6 /* mbuf+cluster from packet zone */ -#define EXT_MBUF 7 /* external mbuf reference (M_IOVEC) */ +#define EXT_MBUF 7 /* external mbuf reference */ #define EXT_SFBUF_NOCACHE 8 /* sendfile(2)'s sf_buf not to be cached */ #define EXT_VENDOR1 224 /* for vendor-internal use */ #define EXT_VENDOR2 225 /* for vendor-internal use */ #define EXT_VENDOR3 226 /* for vendor-internal use */ #define EXT_VENDOR4 227 /* for vendor-internal use */ #define EXT_EXP1 244 /* for experimental use */ #define EXT_EXP2 245 /* for experimental use */ #define EXT_EXP3 246 /* for experimental use */ #define EXT_EXP4 247 /* for experimental use */ #define EXT_NET_DRV 252 /* custom ext_buf provided by net driver(s) */ #define EXT_MOD_TYPE 253 /* custom module's ext_buf type */ #define EXT_DISPOSABLE 254 /* can throw this buffer away w/page flipping */ #define EXT_EXTREF 255 /* has externally maintained ext_cnt ptr */ /* * Flags for external mbuf buffer types. * NB: limited to the lower 24 bits. */ #define EXT_FLAG_EMBREF 0x000001 /* embedded ext_count */ #define EXT_FLAG_EXTREF 0x000002 /* external ext_cnt, notyet */ #define EXT_FLAG_NOFREE 0x000010 /* don't free mbuf to pool, notyet */ #define EXT_FLAG_VENDOR1 0x010000 /* for vendor-internal use */ #define EXT_FLAG_VENDOR2 0x020000 /* for vendor-internal use */ #define EXT_FLAG_VENDOR3 0x040000 /* for vendor-internal use */ #define EXT_FLAG_VENDOR4 0x080000 /* for vendor-internal use */ #define EXT_FLAG_EXP1 0x100000 /* for experimental use */ #define EXT_FLAG_EXP2 0x200000 /* for experimental use */ #define EXT_FLAG_EXP3 0x400000 /* for experimental use */ #define EXT_FLAG_EXP4 0x800000 /* for experimental use */ /* * EXT flag description for use with printf(9) %b identifier. */ #define EXT_FLAG_BITS \ "\20\1EXT_FLAG_EMBREF\2EXT_FLAG_EXTREF\5EXT_FLAG_NOFREE" \ "\21EXT_FLAG_VENDOR1\22EXT_FLAG_VENDOR2\23EXT_FLAG_VENDOR3" \ "\24EXT_FLAG_VENDOR4\25EXT_FLAG_EXP1\26EXT_FLAG_EXP2\27EXT_FLAG_EXP3" \ "\30EXT_FLAG_EXP4" /* * External reference/free functions. */ void sf_ext_free(void *, void *); void sf_ext_free_nocache(void *, void *); /* * Flags indicating checksum, segmentation and other offload work to be * done, or already done, by hardware or lower layers. It is split into * separate inbound and outbound flags. * * Outbound flags that are set by upper protocol layers requesting lower * layers, or ideally the hardware, to perform these offloading tasks. * For outbound packets this field and its flags can be directly tested * against ifnet if_hwassist. */ #define CSUM_IP 0x00000001 /* IP header checksum offload */ #define CSUM_IP_UDP 0x00000002 /* UDP checksum offload */ #define CSUM_IP_TCP 0x00000004 /* TCP checksum offload */ #define CSUM_IP_SCTP 0x00000008 /* SCTP checksum offload */ #define CSUM_IP_TSO 0x00000010 /* TCP segmentation offload */ #define CSUM_IP_ISCSI 0x00000020 /* iSCSI checksum offload */ #define CSUM_IP6_UDP 0x00000200 /* UDP checksum offload */ #define CSUM_IP6_TCP 0x00000400 /* TCP checksum offload */ #define CSUM_IP6_SCTP 0x00000800 /* SCTP checksum offload */ #define CSUM_IP6_TSO 0x00001000 /* TCP segmentation offload */ #define CSUM_IP6_ISCSI 0x00002000 /* iSCSI checksum offload */ /* Inbound checksum support where the checksum was verified by hardware. */ #define CSUM_L3_CALC 0x01000000 /* calculated layer 3 csum */ #define CSUM_L3_VALID 0x02000000 /* checksum is correct */ #define CSUM_L4_CALC 0x04000000 /* calculated layer 4 csum */ #define CSUM_L4_VALID 0x08000000 /* checksum is correct */ #define CSUM_L5_CALC 0x10000000 /* calculated layer 5 csum */ #define CSUM_L5_VALID 0x20000000 /* checksum is correct */ #define CSUM_COALESED 0x40000000 /* contains merged segments */ /* * CSUM flag description for use with printf(9) %b identifier. */ #define CSUM_BITS \ "\20\1CSUM_IP\2CSUM_IP_UDP\3CSUM_IP_TCP\4CSUM_IP_SCTP\5CSUM_IP_TSO" \ "\6CSUM_IP_ISCSI" \ "\12CSUM_IP6_UDP\13CSUM_IP6_TCP\14CSUM_IP6_SCTP\15CSUM_IP6_TSO" \ "\16CSUM_IP6_ISCSI" \ "\31CSUM_L3_CALC\32CSUM_L3_VALID\33CSUM_L4_CALC\34CSUM_L4_VALID" \ "\35CSUM_L5_CALC\36CSUM_L5_VALID\37CSUM_COALESED" /* CSUM flags compatibility mappings. */ #define CSUM_IP_CHECKED CSUM_L3_CALC #define CSUM_IP_VALID CSUM_L3_VALID #define CSUM_DATA_VALID CSUM_L4_VALID #define CSUM_PSEUDO_HDR CSUM_L4_CALC #define CSUM_SCTP_VALID CSUM_L4_VALID #define CSUM_DELAY_DATA (CSUM_TCP|CSUM_UDP) #define CSUM_DELAY_IP CSUM_IP /* Only v4, no v6 IP hdr csum */ #define CSUM_DELAY_DATA_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6) #define CSUM_DATA_VALID_IPV6 CSUM_DATA_VALID #define CSUM_TCP CSUM_IP_TCP #define CSUM_UDP CSUM_IP_UDP #define CSUM_SCTP CSUM_IP_SCTP #define CSUM_TSO (CSUM_IP_TSO|CSUM_IP6_TSO) #define CSUM_UDP_IPV6 CSUM_IP6_UDP #define CSUM_TCP_IPV6 CSUM_IP6_TCP #define CSUM_SCTP_IPV6 CSUM_IP6_SCTP /* * mbuf types describing the content of the mbuf (including external storage). */ #define MT_NOTMBUF 0 /* USED INTERNALLY ONLY! Object is not mbuf */ #define MT_DATA 1 /* dynamic (data) allocation */ #define MT_HEADER MT_DATA /* packet header, use M_PKTHDR instead */ #define MT_VENDOR1 4 /* for vendor-internal use */ #define MT_VENDOR2 5 /* for vendor-internal use */ #define MT_VENDOR3 6 /* for vendor-internal use */ #define MT_VENDOR4 7 /* for vendor-internal use */ #define MT_SONAME 8 /* socket name */ #define MT_EXP1 9 /* for experimental use */ #define MT_EXP2 10 /* for experimental use */ #define MT_EXP3 11 /* for experimental use */ #define MT_EXP4 12 /* for experimental use */ #define MT_CONTROL 14 /* extra-data protocol message */ #define MT_OOBDATA 15 /* expedited data */ #define MT_NTYPES 16 /* number of mbuf types for mbtypes[] */ #define MT_NOINIT 255 /* Not a type but a flag to allocate a non-initialized mbuf */ /* * String names of mbuf-related UMA(9) and malloc(9) types. Exposed to * !_KERNEL so that monitoring tools can look up the zones with * libmemstat(3). */ #define MBUF_MEM_NAME "mbuf" #define MBUF_CLUSTER_MEM_NAME "mbuf_cluster" #define MBUF_PACKET_MEM_NAME "mbuf_packet" #define MBUF_JUMBOP_MEM_NAME "mbuf_jumbo_page" #define MBUF_JUMBO9_MEM_NAME "mbuf_jumbo_9k" #define MBUF_JUMBO16_MEM_NAME "mbuf_jumbo_16k" #define MBUF_TAG_MEM_NAME "mbuf_tag" #define MBUF_EXTREFCNT_MEM_NAME "mbuf_ext_refcnt" #ifdef _KERNEL #ifdef WITNESS #define MBUF_CHECKSLEEP(how) do { \ if (how == M_WAITOK) \ WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, \ "Sleeping in \"%s\"", __func__); \ } while (0) #else #define MBUF_CHECKSLEEP(how) #endif /* * Network buffer allocation API * * The rest of it is defined in kern/kern_mbuf.c */ extern uma_zone_t zone_mbuf; extern uma_zone_t zone_clust; extern uma_zone_t zone_pack; extern uma_zone_t zone_jumbop; extern uma_zone_t zone_jumbo9; extern uma_zone_t zone_jumbo16; void mb_dupcl(struct mbuf *, struct mbuf *); void mb_free_ext(struct mbuf *); void m_adj(struct mbuf *, int); int m_apply(struct mbuf *, int, int, int (*)(void *, void *, u_int), void *); int m_append(struct mbuf *, int, c_caddr_t); void m_cat(struct mbuf *, struct mbuf *); void m_catpkt(struct mbuf *, struct mbuf *); int m_clget(struct mbuf *m, int how); void *m_cljget(struct mbuf *m, int how, int size); struct mbuf *m_collapse(struct mbuf *, int, int); void m_copyback(struct mbuf *, int, int, c_caddr_t); void m_copydata(const struct mbuf *, int, int, caddr_t); struct mbuf *m_copym(struct mbuf *, int, int, int); struct mbuf *m_copypacket(struct mbuf *, int); void m_copy_pkthdr(struct mbuf *, struct mbuf *); struct mbuf *m_copyup(struct mbuf *, int, int); struct mbuf *m_defrag(struct mbuf *, int); void m_demote_pkthdr(struct mbuf *); void m_demote(struct mbuf *, int, int); struct mbuf *m_devget(char *, int, int, struct ifnet *, void (*)(char *, caddr_t, u_int)); struct mbuf *m_dup(const struct mbuf *, int); int m_dup_pkthdr(struct mbuf *, const struct mbuf *, int); void m_extadd(struct mbuf *, caddr_t, u_int, void (*)(struct mbuf *, void *, void *), void *, void *, int, int); u_int m_fixhdr(struct mbuf *); struct mbuf *m_fragment(struct mbuf *, int, int); void m_freem(struct mbuf *); struct mbuf *m_get2(int, int, short, int); struct mbuf *m_getjcl(int, short, int, int); struct mbuf *m_getm2(struct mbuf *, int, int, short, int); struct mbuf *m_getptr(struct mbuf *, int, int *); u_int m_length(struct mbuf *, struct mbuf **); int m_mbuftouio(struct uio *, struct mbuf *, int); void m_move_pkthdr(struct mbuf *, struct mbuf *); int m_pkthdr_init(struct mbuf *, int); struct mbuf *m_prepend(struct mbuf *, int, int); void m_print(const struct mbuf *, int); struct mbuf *m_pulldown(struct mbuf *, int, int, int *); struct mbuf *m_pullup(struct mbuf *, int); int m_sanity(struct mbuf *, int); struct mbuf *m_split(struct mbuf *, int, int); struct mbuf *m_uiotombuf(struct uio *, int, int, int, int); struct mbuf *m_unshare(struct mbuf *, int); static __inline int m_gettype(int size) { int type; switch (size) { case MSIZE: type = EXT_MBUF; break; case MCLBYTES: type = EXT_CLUSTER; break; #if MJUMPAGESIZE != MCLBYTES case MJUMPAGESIZE: type = EXT_JUMBOP; break; #endif case MJUM9BYTES: type = EXT_JUMBO9; break; case MJUM16BYTES: type = EXT_JUMBO16; break; default: panic("%s: invalid cluster size %d", __func__, size); } return (type); } /* * Associated an external reference counted buffer with an mbuf. */ static __inline void m_extaddref(struct mbuf *m, caddr_t buf, u_int size, u_int *ref_cnt, void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2) { KASSERT(ref_cnt != NULL, ("%s: ref_cnt not provided", __func__)); atomic_add_int(ref_cnt, 1); m->m_flags |= M_EXT; m->m_ext.ext_buf = buf; m->m_ext.ext_cnt = ref_cnt; m->m_data = m->m_ext.ext_buf; m->m_ext.ext_size = size; m->m_ext.ext_free = freef; m->m_ext.ext_arg1 = arg1; m->m_ext.ext_arg2 = arg2; m->m_ext.ext_type = EXT_EXTREF; m->m_ext.ext_flags = 0; } static __inline uma_zone_t m_getzone(int size) { uma_zone_t zone; switch (size) { case MCLBYTES: zone = zone_clust; break; #if MJUMPAGESIZE != MCLBYTES case MJUMPAGESIZE: zone = zone_jumbop; break; #endif case MJUM9BYTES: zone = zone_jumbo9; break; case MJUM16BYTES: zone = zone_jumbo16; break; default: panic("%s: invalid cluster size %d", __func__, size); } return (zone); } /* * Initialize an mbuf with linear storage. * * Inline because the consumer text overhead will be roughly the same to * initialize or call a function with this many parameters and M_PKTHDR * should go away with constant propagation for !MGETHDR. */ static __inline int m_init(struct mbuf *m, int how, short type, int flags) { int error; m->m_next = NULL; m->m_nextpkt = NULL; m->m_data = m->m_dat; m->m_len = 0; m->m_flags = flags; m->m_type = type; if (flags & M_PKTHDR) error = m_pkthdr_init(m, how); else error = 0; MBUF_PROBE5(m__init, m, how, type, flags, error); return (error); } static __inline struct mbuf * m_get(int how, short type) { struct mbuf *m; struct mb_args args; args.flags = 0; args.type = type; m = uma_zalloc_arg(zone_mbuf, &args, how); MBUF_PROBE3(m__get, how, type, m); return (m); } static __inline struct mbuf * m_gethdr(int how, short type) { struct mbuf *m; struct mb_args args; args.flags = M_PKTHDR; args.type = type; m = uma_zalloc_arg(zone_mbuf, &args, how); MBUF_PROBE3(m__gethdr, how, type, m); return (m); } static __inline struct mbuf * m_getcl(int how, short type, int flags) { struct mbuf *m; struct mb_args args; args.flags = flags; args.type = type; m = uma_zalloc_arg(zone_pack, &args, how); MBUF_PROBE4(m__getcl, how, type, flags, m); return (m); } /* * XXX: m_cljset() is a dangerous API. One must attach only a new, * unreferenced cluster to an mbuf(9). It is not possible to assert * that, so care can be taken only by users of the API. */ static __inline void m_cljset(struct mbuf *m, void *cl, int type) { int size; switch (type) { case EXT_CLUSTER: size = MCLBYTES; break; #if MJUMPAGESIZE != MCLBYTES case EXT_JUMBOP: size = MJUMPAGESIZE; break; #endif case EXT_JUMBO9: size = MJUM9BYTES; break; case EXT_JUMBO16: size = MJUM16BYTES; break; default: panic("%s: unknown cluster type %d", __func__, type); break; } m->m_data = m->m_ext.ext_buf = cl; m->m_ext.ext_free = m->m_ext.ext_arg1 = m->m_ext.ext_arg2 = NULL; m->m_ext.ext_size = size; m->m_ext.ext_type = type; m->m_ext.ext_flags = EXT_FLAG_EMBREF; m->m_ext.ext_count = 1; m->m_flags |= M_EXT; MBUF_PROBE3(m__cljset, m, cl, type); } static __inline void m_chtype(struct mbuf *m, short new_type) { m->m_type = new_type; } static __inline void m_clrprotoflags(struct mbuf *m) { while (m) { m->m_flags &= ~M_PROTOFLAGS; m = m->m_next; } } static __inline struct mbuf * m_last(struct mbuf *m) { while (m->m_next) m = m->m_next; return (m); } static inline u_int m_extrefcnt(struct mbuf *m) { KASSERT(m->m_flags & M_EXT, ("%s: M_EXT missing", __func__)); return ((m->m_ext.ext_flags & EXT_FLAG_EMBREF) ? m->m_ext.ext_count : *m->m_ext.ext_cnt); } /* * mbuf, cluster, and external object allocation macros (for compatibility * purposes). */ #define M_MOVE_PKTHDR(to, from) m_move_pkthdr((to), (from)) #define MGET(m, how, type) ((m) = m_get((how), (type))) #define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) #define MCLGET(m, how) m_clget((m), (how)) #define MEXTADD(m, buf, size, free, arg1, arg2, flags, type) \ m_extadd((m), (caddr_t)(buf), (size), (free), (arg1), (arg2), \ (flags), (type)) #define m_getm(m, len, how, type) \ m_getm2((m), (len), (how), (type), M_PKTHDR) /* * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can * be both the local data payload, or an external buffer area, depending on * whether M_EXT is set). */ #define M_WRITABLE(m) (!((m)->m_flags & M_RDONLY) && \ (!(((m)->m_flags & M_EXT)) || \ (m_extrefcnt(m) == 1))) /* Check if the supplied mbuf has a packet header, or else panic. */ #define M_ASSERTPKTHDR(m) \ KASSERT((m) != NULL && (m)->m_flags & M_PKTHDR, \ ("%s: no mbuf packet header!", __func__)) /* * Ensure that the supplied mbuf is a valid, non-free mbuf. * * XXX: Broken at the moment. Need some UMA magic to make it work again. */ #define M_ASSERTVALID(m) \ KASSERT((((struct mbuf *)m)->m_flags & 0) == 0, \ ("%s: attempted use of a free mbuf!", __func__)) /* * Return the address of the start of the buffer associated with an mbuf, * handling external storage, packet-header mbufs, and regular data mbufs. */ #define M_START(m) \ (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf : \ ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \ &(m)->m_dat[0]) /* * Return the size of the buffer associated with an mbuf, handling external * storage, packet-header mbufs, and regular data mbufs. */ #define M_SIZE(m) \ (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \ ((m)->m_flags & M_PKTHDR) ? MHLEN : \ MLEN) /* * Set the m_data pointer of a newly allocated mbuf to place an object of the * specified size at the end of the mbuf, longword aligned. * * NB: Historically, we had M_ALIGN(), MH_ALIGN(), and MEXT_ALIGN() as * separate macros, each asserting that it was called at the proper moment. * This required callers to themselves test the storage type and call the * right one. Rather than require callers to be aware of those layout * decisions, we centralize here. */ static __inline void m_align(struct mbuf *m, int len) { #ifdef INVARIANTS const char *msg = "%s: not a virgin mbuf"; #endif int adjust; KASSERT(m->m_data == M_START(m), (msg, __func__)); adjust = M_SIZE(m) - len; m->m_data += adjust &~ (sizeof(long)-1); } #define M_ALIGN(m, len) m_align(m, len) #define MH_ALIGN(m, len) m_align(m, len) #define MEXT_ALIGN(m, len) m_align(m, len) /* * Compute the amount of space available before the current start of data in * an mbuf. * * The M_WRITABLE() is a temporary, conservative safety measure: the burden * of checking writability of the mbuf data area rests solely with the caller. * * NB: In previous versions, M_LEADINGSPACE() would only check M_WRITABLE() * for mbufs with external storage. We now allow mbuf-embedded data to be * read-only as well. */ #define M_LEADINGSPACE(m) \ (M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0) /* * Compute the amount of space available after the end of data in an mbuf. * * The M_WRITABLE() is a temporary, conservative safety measure: the burden * of checking writability of the mbuf data area rests solely with the caller. * * NB: In previous versions, M_TRAILINGSPACE() would only check M_WRITABLE() * for mbufs with external storage. We now allow mbuf-embedded data to be * read-only as well. */ #define M_TRAILINGSPACE(m) \ (M_WRITABLE(m) ? \ ((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0) /* * Arrange to prepend space of size plen to mbuf m. If a new mbuf must be * allocated, how specifies whether to wait. If the allocation fails, the * original mbuf chain is freed and m is set to NULL. */ #define M_PREPEND(m, plen, how) do { \ struct mbuf **_mmp = &(m); \ struct mbuf *_mm = *_mmp; \ int _mplen = (plen); \ int __mhow = (how); \ \ MBUF_CHECKSLEEP(how); \ if (M_LEADINGSPACE(_mm) >= _mplen) { \ _mm->m_data -= _mplen; \ _mm->m_len += _mplen; \ } else \ _mm = m_prepend(_mm, _mplen, __mhow); \ if (_mm != NULL && _mm->m_flags & M_PKTHDR) \ _mm->m_pkthdr.len += _mplen; \ *_mmp = _mm; \ } while (0) /* * Change mbuf to new type. This is a relatively expensive operation and * should be avoided. */ #define MCHTYPE(m, t) m_chtype((m), (t)) /* Length to m_copy to copy all. */ #define M_COPYALL 1000000000 /* Compatibility with 4.3. */ #define m_copy(m, o, l) m_copym((m), (o), (l), M_NOWAIT) extern int max_datalen; /* MHLEN - max_hdr */ extern int max_hdr; /* Largest link + protocol header */ extern int max_linkhdr; /* Largest link-level header */ extern int max_protohdr; /* Largest protocol header */ extern int nmbclusters; /* Maximum number of clusters */ /*- * Network packets may have annotations attached by affixing a list of * "packet tags" to the pkthdr structure. Packet tags are dynamically * allocated semi-opaque data structures that have a fixed header * (struct m_tag) that specifies the size of the memory block and a * pair that identifies it. The cookie is a 32-bit unique * unsigned value used to identify a module or ABI. By convention this value * is chosen as the date+time that the module is created, expressed as the * number of seconds since the epoch (e.g., using date -u +'%s'). The type * value is an ABI/module-specific value that identifies a particular * annotation and is private to the module. For compatibility with systems * like OpenBSD that define packet tags w/o an ABI/module cookie, the value * PACKET_ABI_COMPAT is used to implement m_tag_get and m_tag_find * compatibility shim functions and several tag types are defined below. * Users that do not require compatibility should use a private cookie value * so that packet tag-related definitions can be maintained privately. * * Note that the packet tag returned by m_tag_alloc has the default memory * alignment implemented by malloc. To reference private data one can use a * construct like: * * struct m_tag *mtag = m_tag_alloc(...); * struct foo *p = (struct foo *)(mtag+1); * * if the alignment of struct m_tag is sufficient for referencing members of * struct foo. Otherwise it is necessary to embed struct m_tag within the * private data structure to insure proper alignment; e.g., * * struct foo { * struct m_tag tag; * ... * }; * struct foo *p = (struct foo *) m_tag_alloc(...); * struct m_tag *mtag = &p->tag; */ /* * Persistent tags stay with an mbuf until the mbuf is reclaimed. Otherwise * tags are expected to ``vanish'' when they pass through a network * interface. For most interfaces this happens normally as the tags are * reclaimed when the mbuf is free'd. However in some special cases * reclaiming must be done manually. An example is packets that pass through * the loopback interface. Also, one must be careful to do this when * ``turning around'' packets (e.g., icmp_reflect). * * To mark a tag persistent bit-or this flag in when defining the tag id. * The tag will then be treated as described above. */ #define MTAG_PERSISTENT 0x800 #define PACKET_TAG_NONE 0 /* Nadda */ /* Packet tags for use with PACKET_ABI_COMPAT. */ #define PACKET_TAG_IPSEC_IN_DONE 1 /* IPsec applied, in */ #define PACKET_TAG_IPSEC_OUT_DONE 2 /* IPsec applied, out */ #define PACKET_TAG_IPSEC_IN_CRYPTO_DONE 3 /* NIC IPsec crypto done */ #define PACKET_TAG_IPSEC_OUT_CRYPTO_NEEDED 4 /* NIC IPsec crypto req'ed */ #define PACKET_TAG_IPSEC_IN_COULD_DO_CRYPTO 5 /* NIC notifies IPsec */ #define PACKET_TAG_IPSEC_PENDING_TDB 6 /* Reminder to do IPsec */ #define PACKET_TAG_BRIDGE 7 /* Bridge processing done */ #define PACKET_TAG_GIF 8 /* GIF processing done */ #define PACKET_TAG_GRE 9 /* GRE processing done */ #define PACKET_TAG_IN_PACKET_CHECKSUM 10 /* NIC checksumming done */ #define PACKET_TAG_ENCAP 11 /* Encap. processing */ #define PACKET_TAG_IPSEC_SOCKET 12 /* IPSEC socket ref */ #define PACKET_TAG_IPSEC_HISTORY 13 /* IPSEC history */ #define PACKET_TAG_IPV6_INPUT 14 /* IPV6 input processing */ #define PACKET_TAG_DUMMYNET 15 /* dummynet info */ #define PACKET_TAG_DIVERT 17 /* divert info */ #define PACKET_TAG_IPFORWARD 18 /* ipforward info */ #define PACKET_TAG_MACLABEL (19 | MTAG_PERSISTENT) /* MAC label */ #define PACKET_TAG_PF (21 | MTAG_PERSISTENT) /* PF/ALTQ information */ #define PACKET_TAG_RTSOCKFAM 25 /* rtsock sa family */ #define PACKET_TAG_IPOPTIONS 27 /* Saved IP options */ #define PACKET_TAG_CARP 28 /* CARP info */ #define PACKET_TAG_IPSEC_NAT_T_PORTS 29 /* two uint16_t */ #define PACKET_TAG_ND_OUTGOING 30 /* ND outgoing */ /* Specific cookies and tags. */ /* Packet tag routines. */ struct m_tag *m_tag_alloc(u_int32_t, int, int, int); void m_tag_delete(struct mbuf *, struct m_tag *); void m_tag_delete_chain(struct mbuf *, struct m_tag *); void m_tag_free_default(struct m_tag *); struct m_tag *m_tag_locate(struct mbuf *, u_int32_t, int, struct m_tag *); struct m_tag *m_tag_copy(struct m_tag *, int); int m_tag_copy_chain(struct mbuf *, const struct mbuf *, int); void m_tag_delete_nonpersistent(struct mbuf *); /* * Initialize the list of tags associated with an mbuf. */ static __inline void m_tag_init(struct mbuf *m) { SLIST_INIT(&m->m_pkthdr.tags); } /* * Set up the contents of a tag. Note that this does not fill in the free * method; the caller is expected to do that. * * XXX probably should be called m_tag_init, but that was already taken. */ static __inline void m_tag_setup(struct m_tag *t, u_int32_t cookie, int type, int len) { t->m_tag_id = type; t->m_tag_len = len; t->m_tag_cookie = cookie; } /* * Reclaim resources associated with a tag. */ static __inline void m_tag_free(struct m_tag *t) { (*t->m_tag_free)(t); } /* * Return the first tag associated with an mbuf. */ static __inline struct m_tag * m_tag_first(struct mbuf *m) { return (SLIST_FIRST(&m->m_pkthdr.tags)); } /* * Return the next tag in the list of tags associated with an mbuf. */ static __inline struct m_tag * m_tag_next(struct mbuf *m __unused, struct m_tag *t) { return (SLIST_NEXT(t, m_tag_link)); } /* * Prepend a tag to the list of tags associated with an mbuf. */ static __inline void m_tag_prepend(struct mbuf *m, struct m_tag *t) { SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link); } /* * Unlink a tag from the list of tags associated with an mbuf. */ static __inline void m_tag_unlink(struct mbuf *m, struct m_tag *t) { SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link); } /* These are for OpenBSD compatibility. */ #define MTAG_ABI_COMPAT 0 /* compatibility ABI */ static __inline struct m_tag * m_tag_get(int type, int length, int wait) { return (m_tag_alloc(MTAG_ABI_COMPAT, type, length, wait)); } static __inline struct m_tag * m_tag_find(struct mbuf *m, int type, struct m_tag *start) { return (SLIST_EMPTY(&m->m_pkthdr.tags) ? (struct m_tag *)NULL : m_tag_locate(m, MTAG_ABI_COMPAT, type, start)); } static __inline struct mbuf * m_free(struct mbuf *m) { struct mbuf *n = m->m_next; MBUF_PROBE1(m__free, m); if ((m->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE)) m_tag_delete_chain(m, NULL); if (m->m_flags & M_EXT) mb_free_ext(m); else if ((m->m_flags & M_NOFREE) == 0) uma_zfree(zone_mbuf, m); return (n); } static __inline int rt_m_getfib(struct mbuf *m) { KASSERT(m->m_flags & M_PKTHDR , ("Attempt to get FIB from non header mbuf.")); return (m->m_pkthdr.fibnum); } #define M_GETFIB(_m) rt_m_getfib(_m) #define M_SETFIB(_m, _fib) do { \ KASSERT((_m)->m_flags & M_PKTHDR, ("Attempt to set FIB on non header mbuf.")); \ ((_m)->m_pkthdr.fibnum) = (_fib); \ } while (0) /* flags passed as first argument for "m_ether_tcpip_hash()" */ #define MBUF_HASHFLAG_L2 (1 << 2) #define MBUF_HASHFLAG_L3 (1 << 3) #define MBUF_HASHFLAG_L4 (1 << 4) /* mbuf hashing helper routines */ uint32_t m_ether_tcpip_hash_init(void); uint32_t m_ether_tcpip_hash(const uint32_t, const struct mbuf *, const uint32_t); #ifdef MBUF_PROFILING void m_profile(struct mbuf *m); #define M_PROFILE(m) m_profile(m) #else #define M_PROFILE(m) #endif struct mbufq { STAILQ_HEAD(, mbuf) mq_head; int mq_len; int mq_maxlen; }; static inline void mbufq_init(struct mbufq *mq, int maxlen) { STAILQ_INIT(&mq->mq_head); mq->mq_maxlen = maxlen; mq->mq_len = 0; } static inline struct mbuf * mbufq_flush(struct mbufq *mq) { struct mbuf *m; m = STAILQ_FIRST(&mq->mq_head); STAILQ_INIT(&mq->mq_head); mq->mq_len = 0; return (m); } static inline void mbufq_drain(struct mbufq *mq) { struct mbuf *m, *n; n = mbufq_flush(mq); while ((m = n) != NULL) { n = STAILQ_NEXT(m, m_stailqpkt); m_freem(m); } } static inline struct mbuf * mbufq_first(const struct mbufq *mq) { return (STAILQ_FIRST(&mq->mq_head)); } static inline struct mbuf * mbufq_last(const struct mbufq *mq) { return (STAILQ_LAST(&mq->mq_head, mbuf, m_stailqpkt)); } static inline int mbufq_full(const struct mbufq *mq) { return (mq->mq_len >= mq->mq_maxlen); } static inline int mbufq_len(const struct mbufq *mq) { return (mq->mq_len); } static inline int mbufq_enqueue(struct mbufq *mq, struct mbuf *m) { if (mbufq_full(mq)) return (ENOBUFS); STAILQ_INSERT_TAIL(&mq->mq_head, m, m_stailqpkt); mq->mq_len++; return (0); } static inline struct mbuf * mbufq_dequeue(struct mbufq *mq) { struct mbuf *m; m = STAILQ_FIRST(&mq->mq_head); if (m) { STAILQ_REMOVE_HEAD(&mq->mq_head, m_stailqpkt); m->m_nextpkt = NULL; mq->mq_len--; } return (m); } static inline void mbufq_prepend(struct mbufq *mq, struct mbuf *m) { STAILQ_INSERT_HEAD(&mq->mq_head, m, m_stailqpkt); mq->mq_len++; } #endif /* _KERNEL */ #endif /* !_SYS_MBUF_H_ */ Index: projects/clang390-import/sys/vm/swap_pager.c =================================================================== --- projects/clang390-import/sys/vm/swap_pager.c (revision 305080) +++ projects/clang390-import/sys/vm/swap_pager.c (revision 305081) @@ -1,2739 +1,2763 @@ /*- * Copyright (c) 1998 Matthew Dillon, * Copyright (c) 1994 John S. Dyson * Copyright (c) 1990 University of Utah. * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * New Swap System * Matthew Dillon * * Radix Bitmap 'blists'. * * - The new swapper uses the new radix bitmap code. This should scale * to arbitrarily small or arbitrarily large swap spaces and an almost * arbitrary degree of fragmentation. * * Features: * * - on the fly reallocation of swap during putpages. The new system * does not try to keep previously allocated swap blocks for dirty * pages. * * - on the fly deallocation of swap * * - No more garbage collection required. Unnecessarily allocated swap * blocks only exist for dirty vm_page_t's now and these are already * cycled (in a high-load system) by the pager. We also do on-the-fly * removal of invalidated swap blocks when a page is destroyed * or renamed. * * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ * * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_swap.h" #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * SWB_NPAGES must be a power of 2. It may be set to 1, 2, 4, 8, 16 * or 32 pages per allocation. * The 32-page limit is due to the radix code (kern/subr_blist.c). */ #ifndef MAX_PAGEOUT_CLUSTER #define MAX_PAGEOUT_CLUSTER 16 #endif #if !defined(SWB_NPAGES) #define SWB_NPAGES MAX_PAGEOUT_CLUSTER #endif /* * The swblock structure maps an object and a small, fixed-size range * of page indices to disk addresses within a swap area. * The collection of these mappings is implemented as a hash table. * Unused disk addresses within a swap area are allocated and managed * using a blist. */ #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t)) #define SWAP_META_PAGES (SWB_NPAGES * 2) #define SWAP_META_MASK (SWAP_META_PAGES - 1) struct swblock { struct swblock *swb_hnext; vm_object_t swb_object; vm_pindex_t swb_index; int swb_count; daddr_t swb_pages[SWAP_META_PAGES]; }; static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data"); static struct mtx sw_dev_mtx; static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq); static struct swdevt *swdevhd; /* Allocate from here next */ static int nswapdev; /* Number of swap devices */ int swap_pager_avail; static struct sx swdev_syscall_lock; /* serialize swap(on|off) */ static vm_ooffset_t swap_total; SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0, "Total amount of available swap storage."); static vm_ooffset_t swap_reserved; SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0, "Amount of swap storage needed to back all allocated anonymous memory."); static int overcommit = 0; SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0, "Configure virtual memory overcommit behavior. See tuning(7) " "for details."); static unsigned long swzone; SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0, "Actual size of swap metadata zone"); static unsigned long swap_maxpages; SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0, "Maximum amount of swap supported"); /* bits from overcommit */ #define SWAP_RESERVE_FORCE_ON (1 << 0) #define SWAP_RESERVE_RLIMIT_ON (1 << 1) #define SWAP_RESERVE_ALLOW_NONWIRED (1 << 2) int swap_reserve(vm_ooffset_t incr) { return (swap_reserve_by_cred(incr, curthread->td_ucred)); } int swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred) { vm_ooffset_t r, s; int res, error; static int curfail; static struct timeval lastfail; struct uidinfo *uip; uip = cred->cr_ruidinfo; if (incr & PAGE_MASK) panic("swap_reserve: & PAGE_MASK"); #ifdef RACCT if (racct_enable) { PROC_LOCK(curproc); error = racct_add(curproc, RACCT_SWAP, incr); PROC_UNLOCK(curproc); if (error != 0) return (0); } #endif res = 0; mtx_lock(&sw_dev_mtx); r = swap_reserved + incr; if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) { s = vm_cnt.v_page_count - vm_cnt.v_free_reserved - vm_cnt.v_wire_count; s *= PAGE_SIZE; } else s = 0; s += swap_total; if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s || (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) { res = 1; swap_reserved = r; } mtx_unlock(&sw_dev_mtx); if (res) { UIDINFO_VMSIZE_LOCK(uip); if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 && uip->ui_vmsize + incr > lim_cur(curthread, RLIMIT_SWAP) && priv_check(curthread, PRIV_VM_SWAP_NORLIMIT)) res = 0; else uip->ui_vmsize += incr; UIDINFO_VMSIZE_UNLOCK(uip); if (!res) { mtx_lock(&sw_dev_mtx); swap_reserved -= incr; mtx_unlock(&sw_dev_mtx); } } if (!res && ppsratecheck(&lastfail, &curfail, 1)) { printf("uid %d, pid %d: swap reservation for %jd bytes failed\n", uip->ui_uid, curproc->p_pid, incr); } #ifdef RACCT if (!res) { PROC_LOCK(curproc); racct_sub(curproc, RACCT_SWAP, incr); PROC_UNLOCK(curproc); } #endif return (res); } void swap_reserve_force(vm_ooffset_t incr) { struct uidinfo *uip; mtx_lock(&sw_dev_mtx); swap_reserved += incr; mtx_unlock(&sw_dev_mtx); #ifdef RACCT PROC_LOCK(curproc); racct_add_force(curproc, RACCT_SWAP, incr); PROC_UNLOCK(curproc); #endif uip = curthread->td_ucred->cr_ruidinfo; PROC_LOCK(curproc); UIDINFO_VMSIZE_LOCK(uip); uip->ui_vmsize += incr; UIDINFO_VMSIZE_UNLOCK(uip); PROC_UNLOCK(curproc); } void swap_release(vm_ooffset_t decr) { struct ucred *cred; PROC_LOCK(curproc); cred = curthread->td_ucred; swap_release_by_cred(decr, cred); PROC_UNLOCK(curproc); } void swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred) { struct uidinfo *uip; uip = cred->cr_ruidinfo; if (decr & PAGE_MASK) panic("swap_release: & PAGE_MASK"); mtx_lock(&sw_dev_mtx); if (swap_reserved < decr) panic("swap_reserved < decr"); swap_reserved -= decr; mtx_unlock(&sw_dev_mtx); UIDINFO_VMSIZE_LOCK(uip); if (uip->ui_vmsize < decr) printf("negative vmsize for uid = %d\n", uip->ui_uid); uip->ui_vmsize -= decr; UIDINFO_VMSIZE_UNLOCK(uip); racct_sub_cred(cred, RACCT_SWAP, decr); } #define SWM_FREE 0x02 /* free, period */ #define SWM_POP 0x04 /* pop out */ int swap_pager_full = 2; /* swap space exhaustion (task killing) */ static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/ static int nsw_rcount; /* free read buffers */ static int nsw_wcount_sync; /* limit write buffers / synchronous */ static int nsw_wcount_async; /* limit write buffers / asynchronous */ static int nsw_wcount_async_max;/* assigned maximum */ static int nsw_cluster_max; /* maximum VOP I/O allowed */ static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_async_max, "I", "Maximum running async swap ops"); static struct swblock **swhash; static int swhash_mask; static struct mtx swhash_mtx; static struct sx sw_alloc_sx; /* * "named" and "unnamed" anon region objects. Try to reduce the overhead * of searching a named list by hashing it just a little. */ #define NOBJLISTS 8 #define NOBJLIST(handle) \ (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)]) static struct pagerlst swap_pager_object_list[NOBJLISTS]; static uma_zone_t swap_zone; /* * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure * calls hooked from other parts of the VM system and do not appear here. * (see vm/swap_pager.h). */ static vm_object_t swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t offset, struct ucred *); static void swap_pager_dealloc(vm_object_t object); static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int *, int *); static int swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int *, int *, pgo_getpages_iodone_t, void *); static void swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); static boolean_t swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); static void swap_pager_init(void); static void swap_pager_unswapped(vm_page_t); static void swap_pager_swapoff(struct swdevt *sp); struct pagerops swappagerops = { .pgo_init = swap_pager_init, /* early system initialization of pager */ .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ .pgo_getpages = swap_pager_getpages, /* pagein */ .pgo_getpages_async = swap_pager_getpages_async, /* pagein (async) */ .pgo_putpages = swap_pager_putpages, /* pageout */ .pgo_haspage = swap_pager_haspage, /* get backing store status for page */ .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ }; /* * dmmax is in page-sized chunks with the new swap system. It was * dev-bsized chunks in the old. dmmax is always a power of 2. * * swap_*() routines are externally accessible. swp_*() routines are * internal. */ static int dmmax; static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */ static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block"); static void swp_sizecheck(void); static void swp_pager_async_iodone(struct buf *bp); static int swapongeom(struct vnode *); static int swaponvp(struct thread *, struct vnode *, u_long); static int swapoff_one(struct swdevt *sp, struct ucred *cred); /* * Swap bitmap functions */ static void swp_pager_freeswapspace(daddr_t blk, int npages); static daddr_t swp_pager_getswapspace(int npages); /* * Metadata functions */ static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index); static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); /* * SWP_SIZECHECK() - update swap_pager_full indication * * update the swap_pager_almost_full indication and warn when we are * about to run out of swap space, using lowat/hiwat hysteresis. * * Clear swap_pager_full ( task killing ) indication when lowat is met. * * No restrictions on call * This routine may not block. */ static void swp_sizecheck(void) { if (swap_pager_avail < nswap_lowat) { if (swap_pager_almost_full == 0) { printf("swap_pager: out of swap space\n"); swap_pager_almost_full = 1; } } else { swap_pager_full = 0; if (swap_pager_avail > nswap_hiwat) swap_pager_almost_full = 0; } } /* * SWP_PAGER_HASH() - hash swap meta data * * This is an helper function which hashes the swapblk given * the object and page index. It returns a pointer to a pointer * to the object, or a pointer to a NULL pointer if it could not * find a swapblk. */ static struct swblock ** swp_pager_hash(vm_object_t object, vm_pindex_t index) { struct swblock **pswap; struct swblock *swap; index &= ~(vm_pindex_t)SWAP_META_MASK; pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask]; while ((swap = *pswap) != NULL) { if (swap->swb_object == object && swap->swb_index == index ) { break; } pswap = &swap->swb_hnext; } return (pswap); } /* * SWAP_PAGER_INIT() - initialize the swap pager! * * Expected to be started from system init. NOTE: This code is run * before much else so be careful what you depend on. Most of the VM * system has yet to be initialized at this point. */ static void swap_pager_init(void) { /* * Initialize object lists */ int i; for (i = 0; i < NOBJLISTS; ++i) TAILQ_INIT(&swap_pager_object_list[i]); mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF); sx_init(&sw_alloc_sx, "swspsx"); sx_init(&swdev_syscall_lock, "swsysc"); /* * Device Stripe, in PAGE_SIZE'd blocks */ dmmax = SWB_NPAGES * 2; } /* * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process * * Expected to be started from pageout process once, prior to entering * its main loop. */ void swap_pager_swap_init(void) { unsigned long n, n2; /* * Number of in-transit swap bp operations. Don't * exhaust the pbufs completely. Make sure we * initialize workable values (0 will work for hysteresis * but it isn't very efficient). * * The nsw_cluster_max is constrained by the bp->b_pages[] * array (MAXPHYS/PAGE_SIZE) and our locally defined * MAX_PAGEOUT_CLUSTER. Also be aware that swap ops are * constrained by the swap device interleave stripe size. * * Currently we hardwire nsw_wcount_async to 4. This limit is * designed to prevent other I/O from having high latencies due to * our pageout I/O. The value 4 works well for one or two active swap * devices but is probably a little low if you have more. Even so, * a higher value would probably generate only a limited improvement * with three or four active swap devices since the system does not * typically have to pageout at extreme bandwidths. We will want * at least 2 per swap devices, and 4 is a pretty good value if you * have one NFS swap device due to the command/ack latency over NFS. * So it all works out pretty well. */ nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER); mtx_lock(&pbuf_mtx); nsw_rcount = (nswbuf + 1) / 2; nsw_wcount_sync = (nswbuf + 3) / 4; nsw_wcount_async = 4; nsw_wcount_async_max = nsw_wcount_async; mtx_unlock(&pbuf_mtx); /* * Initialize our zone. Right now I'm just guessing on the number * we need based on the number of pages in the system. Each swblock * can hold 32 pages, so this is probably overkill. This reservation * is typically limited to around 32MB by default. */ n = vm_cnt.v_page_count / 2; if (maxswzone && n > maxswzone / sizeof(struct swblock)) n = maxswzone / sizeof(struct swblock); n2 = n; swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM); if (swap_zone == NULL) panic("failed to create swap_zone."); do { if (uma_zone_reserve_kva(swap_zone, n)) break; /* * if the allocation failed, try a zone two thirds the * size of the previous attempt. */ n -= ((n + 2) / 3); } while (n > 0); if (n2 != n) printf("Swap zone entries reduced from %lu to %lu.\n", n2, n); swap_maxpages = n * SWAP_META_PAGES; swzone = n * sizeof(struct swblock); n2 = n; /* * Initialize our meta-data hash table. The swapper does not need to * be quite as efficient as the VM system, so we do not use an * oversized hash table. * * n: size of hash table, must be power of 2 * swhash_mask: hash table index mask */ for (n = 1; n < n2 / 8; n *= 2) ; swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO); swhash_mask = n - 1; mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF); } static vm_object_t swap_pager_alloc_init(void *handle, struct ucred *cred, vm_ooffset_t size, vm_ooffset_t offset) { vm_object_t object; if (cred != NULL) { if (!swap_reserve_by_cred(size, cred)) return (NULL); crhold(cred); } object = vm_object_allocate(OBJT_SWAP, OFF_TO_IDX(offset + PAGE_MASK + size)); object->handle = handle; if (cred != NULL) { object->cred = cred; object->charge = size; } object->un_pager.swp.swp_bcount = 0; return (object); } /* * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate * its metadata structures. * * This routine is called from the mmap and fork code to create a new * OBJT_SWAP object. * * This routine must ensure that no live duplicate is created for * the named object request, which is protected against by * holding the sw_alloc_sx lock in case handle != NULL. */ static vm_object_t swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t offset, struct ucred *cred) { vm_object_t object; if (handle != NULL) { /* * Reference existing named region or allocate new one. There * should not be a race here against swp_pager_meta_build() * as called from vm_page_remove() in regards to the lookup * of the handle. */ sx_xlock(&sw_alloc_sx); object = vm_pager_object_lookup(NOBJLIST(handle), handle); if (object == NULL) { object = swap_pager_alloc_init(handle, cred, size, offset); if (object != NULL) { TAILQ_INSERT_TAIL(NOBJLIST(object->handle), object, pager_object_list); } } sx_xunlock(&sw_alloc_sx); } else { object = swap_pager_alloc_init(handle, cred, size, offset); } return (object); } /* * SWAP_PAGER_DEALLOC() - remove swap metadata from object * * The swap backing for the object is destroyed. The code is * designed such that we can reinstantiate it later, but this * routine is typically called only when the entire object is * about to be destroyed. * * The object must be locked. */ static void swap_pager_dealloc(vm_object_t object) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT((object->flags & OBJ_DEAD) != 0, ("dealloc of reachable obj")); /* * Remove from list right away so lookups will fail if we block for * pageout completion. */ if (object->handle != NULL) { VM_OBJECT_WUNLOCK(object); sx_xlock(&sw_alloc_sx); TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list); sx_xunlock(&sw_alloc_sx); VM_OBJECT_WLOCK(object); } vm_object_pip_wait(object, "swpdea"); /* * Free all remaining metadata. We only bother to free it from * the swap meta data. We do not attempt to free swapblk's still * associated with vm_page_t's for this object. We do not care * if paging is still in progress on some objects. */ swp_pager_meta_free_all(object); object->handle = NULL; object->type = OBJT_DEAD; } /************************************************************************ * SWAP PAGER BITMAP ROUTINES * ************************************************************************/ /* * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space * * Allocate swap for the requested number of pages. The starting * swap block number (a page index) is returned or SWAPBLK_NONE * if the allocation failed. * * Also has the side effect of advising that somebody made a mistake * when they configured swap and didn't configure enough. * * This routine may not sleep. * * We allocate in round-robin fashion from the configured devices. */ static daddr_t swp_pager_getswapspace(int npages) { daddr_t blk; struct swdevt *sp; int i; blk = SWAPBLK_NONE; mtx_lock(&sw_dev_mtx); sp = swdevhd; for (i = 0; i < nswapdev; i++) { if (sp == NULL) sp = TAILQ_FIRST(&swtailq); if (!(sp->sw_flags & SW_CLOSING)) { blk = blist_alloc(sp->sw_blist, npages); if (blk != SWAPBLK_NONE) { blk += sp->sw_first; sp->sw_used += npages; swap_pager_avail -= npages; swp_sizecheck(); swdevhd = TAILQ_NEXT(sp, sw_list); goto done; } } sp = TAILQ_NEXT(sp, sw_list); } if (swap_pager_full != 2) { printf("swap_pager_getswapspace(%d): failed\n", npages); swap_pager_full = 2; swap_pager_almost_full = 1; } swdevhd = NULL; done: mtx_unlock(&sw_dev_mtx); return (blk); } static int swp_pager_isondev(daddr_t blk, struct swdevt *sp) { return (blk >= sp->sw_first && blk < sp->sw_end); } static void swp_pager_strategy(struct buf *bp) { struct swdevt *sp; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) { mtx_unlock(&sw_dev_mtx); if ((sp->sw_flags & SW_UNMAPPED) != 0 && unmapped_buf_allowed) { bp->b_data = unmapped_buf; bp->b_offset = 0; } else { pmap_qenter((vm_offset_t)bp->b_data, &bp->b_pages[0], bp->b_bcount / PAGE_SIZE); } sp->sw_strategy(bp, sp); return; } } panic("Swapdev not found"); } /* * SWP_PAGER_FREESWAPSPACE() - free raw swap space * * This routine returns the specified swap blocks back to the bitmap. * * This routine may not sleep. */ static void swp_pager_freeswapspace(daddr_t blk, int npages) { struct swdevt *sp; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (blk >= sp->sw_first && blk < sp->sw_end) { sp->sw_used -= npages; /* * If we are attempting to stop swapping on * this device, we don't want to mark any * blocks free lest they be reused. */ if ((sp->sw_flags & SW_CLOSING) == 0) { blist_free(sp->sw_blist, blk - sp->sw_first, npages); swap_pager_avail += npages; swp_sizecheck(); } mtx_unlock(&sw_dev_mtx); return; } } panic("Swapdev not found"); } /* * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page * range within an object. * * This is a globally accessible routine. * * This routine removes swapblk assignments from swap metadata. * * The external callers of this routine typically have already destroyed * or renamed vm_page_t's associated with this range in the object so * we should be ok. * * The object must be locked. */ void swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size) { swp_pager_meta_free(object, start, size); } /* * SWAP_PAGER_RESERVE() - reserve swap blocks in object * * Assigns swap blocks to the specified range within the object. The * swap blocks are not zeroed. Any previous swap assignment is destroyed. * * Returns 0 on success, -1 on failure. */ int swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size) { int n = 0; daddr_t blk = SWAPBLK_NONE; vm_pindex_t beg = start; /* save start index */ VM_OBJECT_WLOCK(object); while (size) { if (n == 0) { n = BLIST_MAX_ALLOC; while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) { n >>= 1; if (n == 0) { swp_pager_meta_free(object, beg, start - beg); VM_OBJECT_WUNLOCK(object); return (-1); } } } swp_pager_meta_build(object, start, blk); --size; ++start; ++blk; --n; } swp_pager_meta_free(object, start, n); VM_OBJECT_WUNLOCK(object); return (0); } /* * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager * and destroy the source. * * Copy any valid swapblks from the source to the destination. In * cases where both the source and destination have a valid swapblk, * we keep the destination's. * * This routine is allowed to sleep. It may sleep allocating metadata * indirectly through swp_pager_meta_build() or if paging is still in * progress on the source. * * The source object contains no vm_page_t's (which is just as well) * * The source object is of type OBJT_SWAP. * * The source and destination objects must be locked. * Both object locks may temporarily be released. */ void swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, vm_pindex_t offset, int destroysource) { vm_pindex_t i; VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); /* * If destroysource is set, we remove the source object from the * swap_pager internal queue now. */ if (destroysource && srcobject->handle != NULL) { vm_object_pip_add(srcobject, 1); VM_OBJECT_WUNLOCK(srcobject); vm_object_pip_add(dstobject, 1); VM_OBJECT_WUNLOCK(dstobject); sx_xlock(&sw_alloc_sx); TAILQ_REMOVE(NOBJLIST(srcobject->handle), srcobject, pager_object_list); sx_xunlock(&sw_alloc_sx); VM_OBJECT_WLOCK(dstobject); vm_object_pip_wakeup(dstobject); VM_OBJECT_WLOCK(srcobject); vm_object_pip_wakeup(srcobject); } /* * transfer source to destination. */ for (i = 0; i < dstobject->size; ++i) { daddr_t dstaddr; /* * Locate (without changing) the swapblk on the destination, * unless it is invalid in which case free it silently, or * if the destination is a resident page, in which case the * source is thrown away. */ dstaddr = swp_pager_meta_ctl(dstobject, i, 0); if (dstaddr == SWAPBLK_NONE) { /* * Destination has no swapblk and is not resident, * copy source. */ daddr_t srcaddr; srcaddr = swp_pager_meta_ctl( srcobject, i + offset, SWM_POP ); if (srcaddr != SWAPBLK_NONE) { /* * swp_pager_meta_build() can sleep. */ vm_object_pip_add(srcobject, 1); VM_OBJECT_WUNLOCK(srcobject); vm_object_pip_add(dstobject, 1); swp_pager_meta_build(dstobject, i, srcaddr); vm_object_pip_wakeup(dstobject); VM_OBJECT_WLOCK(srcobject); vm_object_pip_wakeup(srcobject); } } else { /* * Destination has valid swapblk or it is represented * by a resident page. We destroy the sourceblock. */ swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE); } } /* * Free left over swap blocks in source. * * We have to revert the type to OBJT_DEFAULT so we do not accidentally * double-remove the object from the swap queues. */ if (destroysource) { swp_pager_meta_free_all(srcobject); /* * Reverting the type is not necessary, the caller is going * to destroy srcobject directly, but I'm doing it here * for consistency since we've removed the object from its * queues. */ srcobject->type = OBJT_DEFAULT; } } /* * SWAP_PAGER_HASPAGE() - determine if we have good backing store for * the requested page. * * We determine whether good backing store exists for the requested * page and return TRUE if it does, FALSE if it doesn't. * * If TRUE, we also try to determine how much valid, contiguous backing - * store exists before and after the requested page within a reasonable - * distance. We do not try to restrict it to the swap device stripe - * (that is handled in getpages/putpages). It probably isn't worth - * doing here. + * store exists before and after the requested page. */ static boolean_t -swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) +swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, + int *after) { - daddr_t blk0; + daddr_t blk, blk0; + int i; VM_OBJECT_ASSERT_LOCKED(object); + /* * do we have good backing store at the requested index ? */ blk0 = swp_pager_meta_ctl(object, pindex, 0); - if (blk0 == SWAPBLK_NONE) { if (before) *before = 0; if (after) *after = 0; return (FALSE); } /* * find backwards-looking contiguous good backing store */ if (before != NULL) { - int i; - - for (i = 1; i < (SWB_NPAGES/2); ++i) { - daddr_t blk; - + for (i = 1; i < SWB_NPAGES; i++) { if (i > pindex) break; blk = swp_pager_meta_ctl(object, pindex - i, 0); if (blk != blk0 - i) break; } - *before = (i - 1); + *before = i - 1; } /* * find forward-looking contiguous good backing store */ if (after != NULL) { - int i; - - for (i = 1; i < (SWB_NPAGES/2); ++i) { - daddr_t blk; - + for (i = 1; i < SWB_NPAGES; i++) { blk = swp_pager_meta_ctl(object, pindex + i, 0); if (blk != blk0 + i) break; } - *after = (i - 1); + *after = i - 1; } return (TRUE); } /* * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page * * This removes any associated swap backing store, whether valid or * not, from the page. * * This routine is typically called when a page is made dirty, at * which point any associated swap can be freed. MADV_FREE also * calls us in a special-case situation * * NOTE!!! If the page is clean and the swap was valid, the caller * should make the page dirty before calling this routine. This routine * does NOT change the m->dirty status of the page. Also: MADV_FREE * depends on it. * * This routine may not sleep. * * The object containing the page must be locked. */ static void swap_pager_unswapped(vm_page_t m) { swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE); } /* - * SWAP_PAGER_GETPAGES() - bring pages in from swap + * swap_pager_getpages() - bring pages in from swap * - * Attempt to retrieve (m, count) pages from backing store, but make - * sure we retrieve at least m[reqpage]. We try to load in as large - * a chunk surrounding m[reqpage] as is contiguous in swap and which - * belongs to the same object. + * Attempt to page in the pages in array "m" of length "count". The caller + * may optionally specify that additional pages preceding and succeeding + * the specified range be paged in. The number of such pages is returned + * in the "rbehind" and "rahead" parameters, and they will be in the + * inactive queue upon return. * - * The code is designed for asynchronous operation and - * immediate-notification of 'reqpage' but tends not to be - * used that way. Please do not optimize-out this algorithmic - * feature, I intend to improve on it in the future. - * - * The parent has a single vm_object_pip_add() reference prior to - * calling us and we should return with the same. - * - * The parent has BUSY'd the pages. We should return with 'm' - * left busy, but the others adjusted. + * The pages in "m" must be busied and will remain busied upon return. */ static int swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, int *rahead) { struct buf *bp; + vm_page_t mpred, msucc, p; + vm_pindex_t pindex; daddr_t blk; + int i, j, reqcount, shift; - /* - * Calculate range to retrieve. The pages have already been assigned - * their swapblks. We require a *contiguous* range but we know it to - * not span devices. If we do not supply it, bad things - * happen. Note that blk, iblk & jblk can be SWAPBLK_NONE, but the - * loops are set up such that the case(s) are handled implicitly. - * - * The swp_*() calls must be made with the object locked. - */ - blk = swp_pager_meta_ctl(m[0]->object, m[0]->pindex, 0); + reqcount = count; - if (blk == SWAPBLK_NONE) + VM_OBJECT_WUNLOCK(object); + bp = getpbuf(&nsw_rcount); + VM_OBJECT_WLOCK(object); + + if (!swap_pager_haspage(object, m[0]->pindex, rbehind, rahead)) { + relpbuf(bp, &nsw_rcount); return (VM_PAGER_FAIL); + } -#ifdef INVARIANTS - for (int i = 0; i < count; i++) - KASSERT(blk + i == - swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0), - ("%s: range is not contiguous", __func__)); -#endif - /* - * Getpbuf() can sleep. + * Clip the readahead and readbehind ranges to exclude resident pages. */ - VM_OBJECT_WUNLOCK(object); + if (rahead != NULL) { + KASSERT(reqcount - 1 <= *rahead, + ("page count %d extends beyond swap block", reqcount)); + *rahead -= reqcount - 1; + pindex = m[reqcount - 1]->pindex; + msucc = TAILQ_NEXT(m[reqcount - 1], listq); + if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead) + *rahead = msucc->pindex - pindex - 1; + } + if (rbehind != NULL) { + pindex = m[0]->pindex; + mpred = TAILQ_PREV(m[0], pglist, listq); + if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind) + *rbehind = pindex - mpred->pindex - 1; + } + /* - * Get a swap buffer header to perform the IO + * Allocate readahead and readbehind pages. */ - bp = getpbuf(&nsw_rcount); - bp->b_flags |= B_PAGING; + shift = rbehind != NULL ? *rbehind : 0; + if (shift != 0) { + for (i = 1; i <= shift; i++) { + p = vm_page_alloc(object, m[0]->pindex - i, + VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED); + if (p == NULL) { + /* Shift allocated pages to the left. */ + for (j = 0; j < i - 1; j++) + bp->b_pages[j] = + bp->b_pages[j + shift - i + 1]; + break; + } + bp->b_pages[shift - i] = p; + } + shift = i - 1; + *rbehind = shift; + } + for (i = 0; i < reqcount; i++) + bp->b_pages[i + shift] = m[i]; + if (rahead != NULL) { + for (i = 0; i < *rahead; i++) { + p = vm_page_alloc(object, + m[reqcount - 1]->pindex + i + 1, + VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED); + if (p == NULL) + break; + bp->b_pages[shift + reqcount + i] = p; + } + *rahead = i; + } + if (rbehind != NULL) + count += *rbehind; + if (rahead != NULL) + count += *rahead; + vm_object_pip_add(object, count); + + for (i = 0; i < count; i++) + bp->b_pages[i]->oflags |= VPO_SWAPINPROG; + + pindex = bp->b_pages[0]->pindex; + blk = swp_pager_meta_ctl(object, pindex, 0); + KASSERT(blk != SWAPBLK_NONE, + ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); + + VM_OBJECT_WUNLOCK(object); + + bp->b_flags |= B_PAGING; bp->b_iocmd = BIO_READ; bp->b_iodone = swp_pager_async_iodone; bp->b_rcred = crhold(thread0.td_ucred); bp->b_wcred = crhold(thread0.td_ucred); bp->b_blkno = blk; bp->b_bcount = PAGE_SIZE * count; bp->b_bufsize = PAGE_SIZE * count; bp->b_npages = count; + bp->b_pgbefore = rbehind != NULL ? *rbehind : 0; + bp->b_pgafter = rahead != NULL ? *rahead : 0; - VM_OBJECT_WLOCK(object); - for (int i = 0; i < count; i++) { - bp->b_pages[i] = m[i]; - m[i]->oflags |= VPO_SWAPINPROG; - } - PCPU_INC(cnt.v_swapin); - PCPU_ADD(cnt.v_swappgsin, bp->b_npages); + PCPU_ADD(cnt.v_swappgsin, count); /* - * We still hold the lock on mreq, and our automatic completion routine - * does not remove it. - */ - vm_object_pip_add(object, bp->b_npages); - VM_OBJECT_WUNLOCK(object); - - /* * perform the I/O. NOTE!!! bp cannot be considered valid after * this point because we automatically release it on completion. * Instead, we look at the one page we are interested in which we * still hold a lock on even through the I/O completion. * * The other pages in our m[] array are also released on completion, * so we cannot assume they are valid anymore either. * * NOTE: b_blkno is destroyed by the call to swapdev_strategy */ BUF_KERNPROC(bp); swp_pager_strategy(bp); /* - * wait for the page we want to complete. VPO_SWAPINPROG is always + * Wait for the pages we want to complete. VPO_SWAPINPROG is always * cleared on completion. If an I/O error occurs, SWAPBLK_NONE - * is set in the meta-data. + * is set in the metadata for each page in the request. */ VM_OBJECT_WLOCK(object); while ((m[0]->oflags & VPO_SWAPINPROG) != 0) { m[0]->oflags |= VPO_SWAPSLEEP; PCPU_INC(cnt.v_intrans); if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP, "swread", hz * 20)) { printf( "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount); } } /* * If we had an unrecoverable read error pages will not be valid. */ - for (int i = 0; i < count; i++) + for (i = 0; i < reqcount; i++) if (m[i]->valid != VM_PAGE_BITS_ALL) return (VM_PAGER_ERROR); - if (rbehind) - *rbehind = 0; - if (rahead) - *rahead = 0; - return (VM_PAGER_OK); /* * A final note: in a low swap situation, we cannot deallocate swap * and mark a page dirty here because the caller is likely to mark * the page clean when we return, causing the page to possibly revert * to all-zero's later. */ } /* * swap_pager_getpages_async(): * * Right now this is emulation of asynchronous operation on top of * swap_pager_getpages(). */ static int swap_pager_getpages_async(vm_object_t object, vm_page_t *m, int count, int *rbehind, int *rahead, pgo_getpages_iodone_t iodone, void *arg) { int r, error; r = swap_pager_getpages(object, m, count, rbehind, rahead); VM_OBJECT_WUNLOCK(object); switch (r) { case VM_PAGER_OK: error = 0; break; case VM_PAGER_ERROR: error = EIO; break; case VM_PAGER_FAIL: error = EINVAL; break; default: panic("unhandled swap_pager_getpages() error %d", r); } (iodone)(arg, m, count, error); VM_OBJECT_WLOCK(object); return (r); } /* * swap_pager_putpages: * * Assign swap (if necessary) and initiate I/O on the specified pages. * * We support both OBJT_DEFAULT and OBJT_SWAP objects. DEFAULT objects * are automatically converted to SWAP objects. * * In a low memory situation we may block in VOP_STRATEGY(), but the new * vm_page reservation system coupled with properly written VFS devices * should ensure that no low-memory deadlock occurs. This is an area * which needs work. * * The parent has N vm_object_pip_add() references prior to * calling us and will remove references for rtvals[] that are * not set to VM_PAGER_PEND. We need to remove the rest on I/O * completion. * * The parent has soft-busy'd the pages it passes us and will unbusy * those whos rtvals[] entry is not set to VM_PAGER_PEND on return. * We need to unbusy the rest on I/O completion. */ static void swap_pager_putpages(vm_object_t object, vm_page_t *m, int count, int flags, int *rtvals) { int i, n; boolean_t sync; if (count && m[0]->object != object) { panic("swap_pager_putpages: object mismatch %p/%p", object, m[0]->object ); } /* * Step 1 * * Turn object into OBJT_SWAP * check for bogus sysops * force sync if not pageout process */ if (object->type != OBJT_SWAP) swp_pager_meta_build(object, 0, SWAPBLK_NONE); VM_OBJECT_WUNLOCK(object); n = 0; if (curproc != pageproc) sync = TRUE; else sync = (flags & VM_PAGER_PUT_SYNC) != 0; /* * Step 2 * * Assign swap blocks and issue I/O. We reallocate swap on the fly. * The page is left dirty until the pageout operation completes * successfully. */ for (i = 0; i < count; i += n) { int j; struct buf *bp; daddr_t blk; /* * Maximum I/O size is limited by a number of factors. */ n = min(BLIST_MAX_ALLOC, count - i); n = min(n, nsw_cluster_max); /* * Get biggest block of swap we can. If we fail, fall * back and try to allocate a smaller block. Don't go * overboard trying to allocate space if it would overly * fragment swap. */ while ( (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE && n > 4 ) { n >>= 1; } if (blk == SWAPBLK_NONE) { for (j = 0; j < n; ++j) rtvals[i+j] = VM_PAGER_FAIL; continue; } /* * All I/O parameters have been satisfied, build the I/O * request and assign the swap space. */ if (sync == TRUE) { bp = getpbuf(&nsw_wcount_sync); } else { bp = getpbuf(&nsw_wcount_async); bp->b_flags = B_ASYNC; } bp->b_flags |= B_PAGING; bp->b_iocmd = BIO_WRITE; bp->b_rcred = crhold(thread0.td_ucred); bp->b_wcred = crhold(thread0.td_ucred); bp->b_bcount = PAGE_SIZE * n; bp->b_bufsize = PAGE_SIZE * n; bp->b_blkno = blk; VM_OBJECT_WLOCK(object); for (j = 0; j < n; ++j) { vm_page_t mreq = m[i+j]; swp_pager_meta_build( mreq->object, mreq->pindex, blk + j ); vm_page_dirty(mreq); mreq->oflags |= VPO_SWAPINPROG; bp->b_pages[j] = mreq; } VM_OBJECT_WUNLOCK(object); bp->b_npages = n; /* * Must set dirty range for NFS to work. */ bp->b_dirtyoff = 0; bp->b_dirtyend = bp->b_bcount; PCPU_INC(cnt.v_swapout); PCPU_ADD(cnt.v_swappgsout, bp->b_npages); /* * We unconditionally set rtvals[] to VM_PAGER_PEND so that we * can call the async completion routine at the end of a * synchronous I/O operation. Otherwise, our caller would * perform duplicate unbusy and wakeup operations on the page * and object, respectively. */ for (j = 0; j < n; j++) rtvals[i + j] = VM_PAGER_PEND; /* * asynchronous * * NOTE: b_blkno is destroyed by the call to swapdev_strategy */ if (sync == FALSE) { bp->b_iodone = swp_pager_async_iodone; BUF_KERNPROC(bp); swp_pager_strategy(bp); continue; } /* * synchronous * * NOTE: b_blkno is destroyed by the call to swapdev_strategy */ bp->b_iodone = bdone; swp_pager_strategy(bp); /* * Wait for the sync I/O to complete. */ bwait(bp, PVM, "swwrt"); /* * Now that we are through with the bp, we can call the * normal async completion, which frees everything up. */ swp_pager_async_iodone(bp); } VM_OBJECT_WLOCK(object); } /* * swp_pager_async_iodone: * * Completion routine for asynchronous reads and writes from/to swap. * Also called manually by synchronous code to finish up a bp. * * This routine may not sleep. */ static void swp_pager_async_iodone(struct buf *bp) { int i; vm_object_t object = NULL; /* * report error */ if (bp->b_ioflags & BIO_ERROR) { printf( "swap_pager: I/O error - %s failed; blkno %ld," "size %ld, error %d\n", ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"), (long)bp->b_blkno, (long)bp->b_bcount, bp->b_error ); } /* * remove the mapping for kernel virtual */ if (buf_mapped(bp)) pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages); else bp->b_data = bp->b_kvabase; if (bp->b_npages) { object = bp->b_pages[0]->object; VM_OBJECT_WLOCK(object); } /* * cleanup pages. If an error occurs writing to swap, we are in * very serious trouble. If it happens to be a disk error, though, * we may be able to recover by reassigning the swap later on. So * in this case we remove the m->swapblk assignment for the page * but do not free it in the rlist. The errornous block(s) are thus * never reallocated as swap. Redirty the page and continue. */ for (i = 0; i < bp->b_npages; ++i) { vm_page_t m = bp->b_pages[i]; m->oflags &= ~VPO_SWAPINPROG; if (m->oflags & VPO_SWAPSLEEP) { m->oflags &= ~VPO_SWAPSLEEP; wakeup(&object->paging_in_progress); } if (bp->b_ioflags & BIO_ERROR) { /* * If an error occurs I'd love to throw the swapblk * away without freeing it back to swapspace, so it * can never be used again. But I can't from an * interrupt. */ if (bp->b_iocmd == BIO_READ) { /* * NOTE: for reads, m->dirty will probably * be overridden by the original caller of * getpages so don't play cute tricks here. */ m->valid = 0; } else { /* * If a write error occurs, reactivate page * so it doesn't clog the inactive list, * then finish the I/O. */ vm_page_dirty(m); vm_page_lock(m); vm_page_activate(m); vm_page_unlock(m); vm_page_sunbusy(m); } } else if (bp->b_iocmd == BIO_READ) { /* * NOTE: for reads, m->dirty will probably be * overridden by the original caller of getpages so * we cannot set them in order to free the underlying * swap in a low-swap situation. I don't think we'd * want to do that anyway, but it was an optimization * that existed in the old swapper for a time before * it got ripped out due to precisely this problem. */ KASSERT(!pmap_page_is_mapped(m), ("swp_pager_async_iodone: page %p is mapped", m)); KASSERT(m->dirty == 0, ("swp_pager_async_iodone: page %p is dirty", m)); + m->valid = VM_PAGE_BITS_ALL; + if (i < bp->b_pgbefore || + i >= bp->b_npages - bp->b_pgafter) + vm_page_readahead_finish(m); } else { /* * For write success, clear the dirty * status, then finish the I/O ( which decrements the * busy count and possibly wakes waiter's up ). */ KASSERT(!pmap_page_is_write_mapped(m), ("swp_pager_async_iodone: page %p is not write" " protected", m)); vm_page_undirty(m); vm_page_sunbusy(m); if (vm_page_count_severe()) { vm_page_lock(m); vm_page_try_to_cache(m); vm_page_unlock(m); } } } /* * adjust pip. NOTE: the original parent may still have its own * pip refs on the object. */ if (object != NULL) { vm_object_pip_wakeupn(object, bp->b_npages); VM_OBJECT_WUNLOCK(object); } /* * swapdev_strategy() manually sets b_vp and b_bufobj before calling * bstrategy(). Set them back to NULL now we're done with it, or we'll * trigger a KASSERT in relpbuf(). */ if (bp->b_vp) { bp->b_vp = NULL; bp->b_bufobj = NULL; } /* * release the physical I/O buffer */ relpbuf( bp, ((bp->b_iocmd == BIO_READ) ? &nsw_rcount : ((bp->b_flags & B_ASYNC) ? &nsw_wcount_async : &nsw_wcount_sync ) ) ); } /* * swap_pager_isswapped: * * Return 1 if at least one page in the given object is paged * out to the given swap device. * * This routine may not sleep. */ int swap_pager_isswapped(vm_object_t object, struct swdevt *sp) { daddr_t index = 0; int bcount; int i; VM_OBJECT_ASSERT_WLOCKED(object); if (object->type != OBJT_SWAP) return (0); mtx_lock(&swhash_mtx); for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) { struct swblock *swap; if ((swap = *swp_pager_hash(object, index)) != NULL) { for (i = 0; i < SWAP_META_PAGES; ++i) { if (swp_pager_isondev(swap->swb_pages[i], sp)) { mtx_unlock(&swhash_mtx); return (1); } } } index += SWAP_META_PAGES; } mtx_unlock(&swhash_mtx); return (0); } /* * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in * * This routine dissociates the page at the given index within a * swap block from its backing store, paging it in if necessary. * If the page is paged in, it is placed in the inactive queue, * since it had its backing store ripped out from under it. * We also attempt to swap in all other pages in the swap block, * we only guarantee that the one at the specified index is * paged in. * * XXX - The code to page the whole block in doesn't work, so we * revert to the one-by-one behavior for now. Sigh. */ static inline void swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex) { vm_page_t m; vm_object_pip_add(object, 1); m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); if (m->valid == VM_PAGE_BITS_ALL) { vm_object_pip_wakeup(object); vm_page_dirty(m); vm_page_lock(m); vm_page_activate(m); vm_page_unlock(m); vm_page_xunbusy(m); vm_pager_page_unswapped(m); return; } if (swap_pager_getpages(object, &m, 1, NULL, NULL) != VM_PAGER_OK) panic("swap_pager_force_pagein: read from swap failed");/*XXX*/ vm_object_pip_wakeup(object); vm_page_dirty(m); vm_page_lock(m); vm_page_deactivate(m); vm_page_unlock(m); vm_page_xunbusy(m); vm_pager_page_unswapped(m); } /* * swap_pager_swapoff: * * Page in all of the pages that have been paged out to the * given device. The corresponding blocks in the bitmap must be * marked as allocated and the device must be flagged SW_CLOSING. * There may be no processes swapped out to the device. * * This routine may block. */ static void swap_pager_swapoff(struct swdevt *sp) { struct swblock *swap; int i, j, retries; sx_assert(&swdev_syscall_lock, SA_XLOCKED); retries = 0; full_rescan: mtx_lock(&swhash_mtx); for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */ restart: for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) { vm_object_t object = swap->swb_object; vm_pindex_t pindex = swap->swb_index; for (j = 0; j < SWAP_META_PAGES; ++j) { if (swp_pager_isondev(swap->swb_pages[j], sp)) { /* avoid deadlock */ if (!VM_OBJECT_TRYWLOCK(object)) { break; } else { mtx_unlock(&swhash_mtx); swp_pager_force_pagein(object, pindex + j); VM_OBJECT_WUNLOCK(object); mtx_lock(&swhash_mtx); goto restart; } } } } } mtx_unlock(&swhash_mtx); if (sp->sw_used) { /* * Objects may be locked or paging to the device being * removed, so we will miss their pages and need to * make another pass. We have marked this device as * SW_CLOSING, so the activity should finish soon. */ retries++; if (retries > 100) { panic("swapoff: failed to locate %d swap blocks", sp->sw_used); } pause("swpoff", hz / 20); goto full_rescan; } } /************************************************************************ * SWAP META DATA * ************************************************************************ * * These routines manipulate the swap metadata stored in the * OBJT_SWAP object. * * Swap metadata is implemented with a global hash and not directly * linked into the object. Instead the object simply contains * appropriate tracking counters. */ /* * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object * * We first convert the object to a swap object if it is a default * object. * * The specified swapblk is added to the object's swap metadata. If * the swapblk is not valid, it is freed instead. Any previously * assigned swapblk is freed. */ static void swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) { static volatile int exhausted; struct swblock *swap; struct swblock **pswap; int idx; VM_OBJECT_ASSERT_WLOCKED(object); /* * Convert default object to swap object if necessary */ if (object->type != OBJT_SWAP) { object->type = OBJT_SWAP; object->un_pager.swp.swp_bcount = 0; KASSERT(object->handle == NULL, ("default pager with handle")); } /* * Locate hash entry. If not found create, but if we aren't adding * anything just return. If we run out of space in the map we wait * and, since the hash table may have changed, retry. */ retry: mtx_lock(&swhash_mtx); pswap = swp_pager_hash(object, pindex); if ((swap = *pswap) == NULL) { int i; if (swapblk == SWAPBLK_NONE) goto done; swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT | (curproc == pageproc ? M_USE_RESERVE : 0)); if (swap == NULL) { mtx_unlock(&swhash_mtx); VM_OBJECT_WUNLOCK(object); if (uma_zone_exhausted(swap_zone)) { if (atomic_cmpset_int(&exhausted, 0, 1)) printf("swap zone exhausted, " "increase kern.maxswzone\n"); vm_pageout_oom(VM_OOM_SWAPZ); pause("swzonex", 10); } else VM_WAIT; VM_OBJECT_WLOCK(object); goto retry; } if (atomic_cmpset_int(&exhausted, 1, 0)) printf("swap zone ok\n"); swap->swb_hnext = NULL; swap->swb_object = object; swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK; swap->swb_count = 0; ++object->un_pager.swp.swp_bcount; for (i = 0; i < SWAP_META_PAGES; ++i) swap->swb_pages[i] = SWAPBLK_NONE; } /* * Delete prior contents of metadata */ idx = pindex & SWAP_META_MASK; if (swap->swb_pages[idx] != SWAPBLK_NONE) { swp_pager_freeswapspace(swap->swb_pages[idx], 1); --swap->swb_count; } /* * Enter block into metadata */ swap->swb_pages[idx] = swapblk; if (swapblk != SWAPBLK_NONE) ++swap->swb_count; done: mtx_unlock(&swhash_mtx); } /* * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata * * The requested range of blocks is freed, with any associated swap * returned to the swap bitmap. * * This routine will free swap metadata structures as they are cleaned * out. This routine does *NOT* operate on swap metadata associated * with resident pages. */ static void swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count) { VM_OBJECT_ASSERT_LOCKED(object); if (object->type != OBJT_SWAP) return; while (count > 0) { struct swblock **pswap; struct swblock *swap; mtx_lock(&swhash_mtx); pswap = swp_pager_hash(object, index); if ((swap = *pswap) != NULL) { daddr_t v = swap->swb_pages[index & SWAP_META_MASK]; if (v != SWAPBLK_NONE) { swp_pager_freeswapspace(v, 1); swap->swb_pages[index & SWAP_META_MASK] = SWAPBLK_NONE; if (--swap->swb_count == 0) { *pswap = swap->swb_hnext; uma_zfree(swap_zone, swap); --object->un_pager.swp.swp_bcount; } } --count; ++index; } else { int n = SWAP_META_PAGES - (index & SWAP_META_MASK); count -= n; index += n; } mtx_unlock(&swhash_mtx); } } /* * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object * * This routine locates and destroys all swap metadata associated with * an object. */ static void swp_pager_meta_free_all(vm_object_t object) { daddr_t index = 0; VM_OBJECT_ASSERT_WLOCKED(object); if (object->type != OBJT_SWAP) return; while (object->un_pager.swp.swp_bcount) { struct swblock **pswap; struct swblock *swap; mtx_lock(&swhash_mtx); pswap = swp_pager_hash(object, index); if ((swap = *pswap) != NULL) { int i; for (i = 0; i < SWAP_META_PAGES; ++i) { daddr_t v = swap->swb_pages[i]; if (v != SWAPBLK_NONE) { --swap->swb_count; swp_pager_freeswapspace(v, 1); } } if (swap->swb_count != 0) panic("swap_pager_meta_free_all: swb_count != 0"); *pswap = swap->swb_hnext; uma_zfree(swap_zone, swap); --object->un_pager.swp.swp_bcount; } mtx_unlock(&swhash_mtx); index += SWAP_META_PAGES; } } /* * SWP_PAGER_METACTL() - misc control of swap and vm_page_t meta data. * * This routine is capable of looking up, popping, or freeing * swapblk assignments in the swap meta data or in the vm_page_t. * The routine typically returns the swapblk being looked-up, or popped, * or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block * was invalid. This routine will automatically free any invalid * meta-data swapblks. * * It is not possible to store invalid swapblks in the swap meta data * (other then a literal 'SWAPBLK_NONE'), so we don't bother checking. * * When acting on a busy resident page and paging is in progress, we * have to wait until paging is complete but otherwise can act on the * busy page. * * SWM_FREE remove and free swap block from metadata * SWM_POP remove from meta data but do not free.. pop it out */ static daddr_t swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags) { struct swblock **pswap; struct swblock *swap; daddr_t r1; int idx; VM_OBJECT_ASSERT_LOCKED(object); /* * The meta data only exists of the object is OBJT_SWAP * and even then might not be allocated yet. */ if (object->type != OBJT_SWAP) return (SWAPBLK_NONE); r1 = SWAPBLK_NONE; mtx_lock(&swhash_mtx); pswap = swp_pager_hash(object, pindex); if ((swap = *pswap) != NULL) { idx = pindex & SWAP_META_MASK; r1 = swap->swb_pages[idx]; if (r1 != SWAPBLK_NONE) { if (flags & SWM_FREE) { swp_pager_freeswapspace(r1, 1); r1 = SWAPBLK_NONE; } if (flags & (SWM_FREE|SWM_POP)) { swap->swb_pages[idx] = SWAPBLK_NONE; if (--swap->swb_count == 0) { *pswap = swap->swb_hnext; uma_zfree(swap_zone, swap); --object->un_pager.swp.swp_bcount; } } } } mtx_unlock(&swhash_mtx); return (r1); } /* * System call swapon(name) enables swapping on device name, * which must be in the swdevsw. Return EBUSY * if already swapping on this device. */ #ifndef _SYS_SYSPROTO_H_ struct swapon_args { char *name; }; #endif /* * MPSAFE */ /* ARGSUSED */ int sys_swapon(struct thread *td, struct swapon_args *uap) { struct vattr attr; struct vnode *vp; struct nameidata nd; int error; error = priv_check(td, PRIV_SWAPON); if (error) return (error); sx_xlock(&swdev_syscall_lock); /* * Swap metadata may not fit in the KVM if we have physical * memory of >1GB. */ if (swap_zone == NULL) { error = ENOMEM; goto done; } NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name, td); error = namei(&nd); if (error) goto done; NDFREE(&nd, NDF_ONLY_PNBUF); vp = nd.ni_vp; if (vn_isdisk(vp, &error)) { error = swapongeom(vp); } else if (vp->v_type == VREG && (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 && (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) { /* * Allow direct swapping to NFS regular files in the same * way that nfs_mountroot() sets up diskless swapping. */ error = swaponvp(td, vp, attr.va_size / DEV_BSIZE); } if (error) vrele(vp); done: sx_xunlock(&swdev_syscall_lock); return (error); } /* * Check that the total amount of swap currently configured does not * exceed half the theoretical maximum. If it does, print a warning * message and return -1; otherwise, return 0. */ static int swapon_check_swzone(unsigned long npages) { unsigned long maxpages; /* absolute maximum we can handle assuming 100% efficiency */ maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES; /* recommend using no more than half that amount */ if (npages > maxpages / 2) { printf("warning: total configured swap (%lu pages) " "exceeds maximum recommended amount (%lu pages).\n", npages, maxpages / 2); printf("warning: increase kern.maxswzone " "or reduce amount of swap.\n"); return (-1); } return (0); } static void swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags) { struct swdevt *sp, *tsp; swblk_t dvbase; u_long mblocks; /* * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks. * First chop nblks off to page-align it, then convert. * * sw->sw_nblks is in page-sized chunks now too. */ nblks &= ~(ctodb(1) - 1); nblks = dbtoc(nblks); /* * If we go beyond this, we get overflows in the radix * tree bitmap code. */ mblocks = 0x40000000 / BLIST_META_RADIX; if (nblks > mblocks) { printf( "WARNING: reducing swap size to maximum of %luMB per unit\n", mblocks / 1024 / 1024 * PAGE_SIZE); nblks = mblocks; } sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO); sp->sw_vp = vp; sp->sw_id = id; sp->sw_dev = dev; sp->sw_flags = 0; sp->sw_nblks = nblks; sp->sw_used = 0; sp->sw_strategy = strategy; sp->sw_close = close; sp->sw_flags = flags; sp->sw_blist = blist_create(nblks, M_WAITOK); /* * Do not free the first two block in order to avoid overwriting * any bsd label at the front of the partition */ blist_free(sp->sw_blist, 2, nblks - 2); dvbase = 0; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(tsp, &swtailq, sw_list) { if (tsp->sw_end >= dvbase) { /* * We put one uncovered page between the devices * in order to definitively prevent any cross-device * I/O requests */ dvbase = tsp->sw_end + 1; } } sp->sw_first = dvbase; sp->sw_end = dvbase + nblks; TAILQ_INSERT_TAIL(&swtailq, sp, sw_list); nswapdev++; swap_pager_avail += nblks; swap_total += (vm_ooffset_t)nblks * PAGE_SIZE; swapon_check_swzone(swap_total / PAGE_SIZE); swp_sizecheck(); mtx_unlock(&sw_dev_mtx); } /* * SYSCALL: swapoff(devname) * * Disable swapping on the given device. * * XXX: Badly designed system call: it should use a device index * rather than filename as specification. We keep sw_vp around * only to make this work. */ #ifndef _SYS_SYSPROTO_H_ struct swapoff_args { char *name; }; #endif /* * MPSAFE */ /* ARGSUSED */ int sys_swapoff(struct thread *td, struct swapoff_args *uap) { struct vnode *vp; struct nameidata nd; struct swdevt *sp; int error; error = priv_check(td, PRIV_SWAPOFF); if (error) return (error); sx_xlock(&swdev_syscall_lock); NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name, td); error = namei(&nd); if (error) goto done; NDFREE(&nd, NDF_ONLY_PNBUF); vp = nd.ni_vp; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (sp->sw_vp == vp) break; } mtx_unlock(&sw_dev_mtx); if (sp == NULL) { error = EINVAL; goto done; } error = swapoff_one(sp, td->td_ucred); done: sx_xunlock(&swdev_syscall_lock); return (error); } static int swapoff_one(struct swdevt *sp, struct ucred *cred) { u_long nblks, dvbase; #ifdef MAC int error; #endif sx_assert(&swdev_syscall_lock, SA_XLOCKED); #ifdef MAC (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY); error = mac_system_check_swapoff(cred, sp->sw_vp); (void) VOP_UNLOCK(sp->sw_vp, 0); if (error != 0) return (error); #endif nblks = sp->sw_nblks; /* * We can turn off this swap device safely only if the * available virtual memory in the system will fit the amount * of data we will have to page back in, plus an epsilon so * the system doesn't become critically low on swap space. */ if (vm_cnt.v_free_count + vm_cnt.v_cache_count + swap_pager_avail < nblks + nswap_lowat) { return (ENOMEM); } /* * Prevent further allocations on this device. */ mtx_lock(&sw_dev_mtx); sp->sw_flags |= SW_CLOSING; for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) { swap_pager_avail -= blist_fill(sp->sw_blist, dvbase, dmmax); } swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE; mtx_unlock(&sw_dev_mtx); /* * Page in the contents of the device and close it. */ swap_pager_swapoff(sp); sp->sw_close(curthread, sp); mtx_lock(&sw_dev_mtx); sp->sw_id = NULL; TAILQ_REMOVE(&swtailq, sp, sw_list); nswapdev--; if (nswapdev == 0) { swap_pager_full = 2; swap_pager_almost_full = 1; } if (swdevhd == sp) swdevhd = NULL; mtx_unlock(&sw_dev_mtx); blist_destroy(sp->sw_blist); free(sp, M_VMPGDATA); return (0); } void swapoff_all(void) { struct swdevt *sp, *spt; const char *devname; int error; sx_xlock(&swdev_syscall_lock); mtx_lock(&sw_dev_mtx); TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) { mtx_unlock(&sw_dev_mtx); if (vn_isdisk(sp->sw_vp, NULL)) devname = devtoname(sp->sw_vp->v_rdev); else devname = "[file]"; error = swapoff_one(sp, thread0.td_ucred); if (error != 0) { printf("Cannot remove swap device %s (error=%d), " "skipping.\n", devname, error); } else if (bootverbose) { printf("Swap device %s removed.\n", devname); } mtx_lock(&sw_dev_mtx); } mtx_unlock(&sw_dev_mtx); sx_xunlock(&swdev_syscall_lock); } void swap_pager_status(int *total, int *used) { struct swdevt *sp; *total = 0; *used = 0; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { *total += sp->sw_nblks; *used += sp->sw_used; } mtx_unlock(&sw_dev_mtx); } int swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len) { struct swdevt *sp; const char *tmp_devname; int error, n; n = 0; error = ENOENT; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (n != name) { n++; continue; } xs->xsw_version = XSWDEV_VERSION; xs->xsw_dev = sp->sw_dev; xs->xsw_flags = sp->sw_flags; xs->xsw_nblks = sp->sw_nblks; xs->xsw_used = sp->sw_used; if (devname != NULL) { if (vn_isdisk(sp->sw_vp, NULL)) tmp_devname = devtoname(sp->sw_vp->v_rdev); else tmp_devname = "[file]"; strncpy(devname, tmp_devname, len); } error = 0; break; } mtx_unlock(&sw_dev_mtx); return (error); } static int sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS) { struct xswdev xs; int error; if (arg2 != 1) /* name length */ return (EINVAL); error = swap_dev_info(*(int *)arg1, &xs, NULL, 0); if (error != 0) return (error); error = SYSCTL_OUT(req, &xs, sizeof(xs)); return (error); } SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0, "Number of swap devices"); SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_vm_swap_info, "Swap statistics by device"); /* * vmspace_swap_count() - count the approximate swap usage in pages for a * vmspace. * * The map must be locked. * * Swap usage is determined by taking the proportional swap used by * VM objects backing the VM map. To make up for fractional losses, * if the VM object has any swap use at all the associated map entries * count for at least 1 swap page. */ long vmspace_swap_count(struct vmspace *vmspace) { vm_map_t map; vm_map_entry_t cur; vm_object_t object; long count, n; map = &vmspace->vm_map; count = 0; for (cur = map->header.next; cur != &map->header; cur = cur->next) { if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 && (object = cur->object.vm_object) != NULL) { VM_OBJECT_WLOCK(object); if (object->type == OBJT_SWAP && object->un_pager.swp.swp_bcount != 0) { n = (cur->end - cur->start) / PAGE_SIZE; count += object->un_pager.swp.swp_bcount * SWAP_META_PAGES * n / object->size + 1; } VM_OBJECT_WUNLOCK(object); } } return (count); } /* * GEOM backend * * Swapping onto disk devices. * */ static g_orphan_t swapgeom_orphan; static struct g_class g_swap_class = { .name = "SWAP", .version = G_VERSION, .orphan = swapgeom_orphan, }; DECLARE_GEOM_CLASS(g_swap_class, g_class); static void swapgeom_close_ev(void *arg, int flags) { struct g_consumer *cp; cp = arg; g_access(cp, -1, -1, 0); g_detach(cp); g_destroy_consumer(cp); } /* * Add a reference to the g_consumer for an inflight transaction. */ static void swapgeom_acquire(struct g_consumer *cp) { mtx_assert(&sw_dev_mtx, MA_OWNED); cp->index++; } /* * Remove a reference from the g_consumer. Post a close event if all * references go away, since the function might be called from the * biodone context. */ static void swapgeom_release(struct g_consumer *cp, struct swdevt *sp) { mtx_assert(&sw_dev_mtx, MA_OWNED); cp->index--; if (cp->index == 0) { if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0) sp->sw_id = NULL; } } static void swapgeom_done(struct bio *bp2) { struct swdevt *sp; struct buf *bp; struct g_consumer *cp; bp = bp2->bio_caller2; cp = bp2->bio_from; bp->b_ioflags = bp2->bio_flags; if (bp2->bio_error) bp->b_ioflags |= BIO_ERROR; bp->b_resid = bp->b_bcount - bp2->bio_completed; bp->b_error = bp2->bio_error; bufdone(bp); sp = bp2->bio_caller1; mtx_lock(&sw_dev_mtx); swapgeom_release(cp, sp); mtx_unlock(&sw_dev_mtx); g_destroy_bio(bp2); } static void swapgeom_strategy(struct buf *bp, struct swdevt *sp) { struct bio *bio; struct g_consumer *cp; mtx_lock(&sw_dev_mtx); cp = sp->sw_id; if (cp == NULL) { mtx_unlock(&sw_dev_mtx); bp->b_error = ENXIO; bp->b_ioflags |= BIO_ERROR; bufdone(bp); return; } swapgeom_acquire(cp); mtx_unlock(&sw_dev_mtx); if (bp->b_iocmd == BIO_WRITE) bio = g_new_bio(); else bio = g_alloc_bio(); if (bio == NULL) { mtx_lock(&sw_dev_mtx); swapgeom_release(cp, sp); mtx_unlock(&sw_dev_mtx); bp->b_error = ENOMEM; bp->b_ioflags |= BIO_ERROR; bufdone(bp); return; } bio->bio_caller1 = sp; bio->bio_caller2 = bp; bio->bio_cmd = bp->b_iocmd; bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE; bio->bio_length = bp->b_bcount; bio->bio_done = swapgeom_done; if (!buf_mapped(bp)) { bio->bio_ma = bp->b_pages; bio->bio_data = unmapped_buf; bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK; bio->bio_ma_n = bp->b_npages; bio->bio_flags |= BIO_UNMAPPED; } else { bio->bio_data = bp->b_data; bio->bio_ma = NULL; } g_io_request(bio, cp); return; } static void swapgeom_orphan(struct g_consumer *cp) { struct swdevt *sp; int destroy; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (sp->sw_id == cp) { sp->sw_flags |= SW_CLOSING; break; } } /* * Drop reference we were created with. Do directly since we're in a * special context where we don't have to queue the call to * swapgeom_close_ev(). */ cp->index--; destroy = ((sp != NULL) && (cp->index == 0)); if (destroy) sp->sw_id = NULL; mtx_unlock(&sw_dev_mtx); if (destroy) swapgeom_close_ev(cp, 0); } static void swapgeom_close(struct thread *td, struct swdevt *sw) { struct g_consumer *cp; mtx_lock(&sw_dev_mtx); cp = sw->sw_id; sw->sw_id = NULL; mtx_unlock(&sw_dev_mtx); /* * swapgeom_close() may be called from the biodone context, * where we cannot perform topology changes. Delegate the * work to the events thread. */ if (cp != NULL) g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL); } static int swapongeom_locked(struct cdev *dev, struct vnode *vp) { struct g_provider *pp; struct g_consumer *cp; static struct g_geom *gp; struct swdevt *sp; u_long nblks; int error; pp = g_dev_getprovider(dev); if (pp == NULL) return (ENODEV); mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { cp = sp->sw_id; if (cp != NULL && cp->provider == pp) { mtx_unlock(&sw_dev_mtx); return (EBUSY); } } mtx_unlock(&sw_dev_mtx); if (gp == NULL) gp = g_new_geomf(&g_swap_class, "swap"); cp = g_new_consumer(gp); cp->index = 1; /* Number of active I/Os, plus one for being active. */ cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; g_attach(cp, pp); /* * XXX: Every time you think you can improve the margin for * footshooting, somebody depends on the ability to do so: * savecore(8) wants to write to our swapdev so we cannot * set an exclusive count :-( */ error = g_access(cp, 1, 1, 0); if (error != 0) { g_detach(cp); g_destroy_consumer(cp); return (error); } nblks = pp->mediasize / DEV_BSIZE; swaponsomething(vp, cp, nblks, swapgeom_strategy, swapgeom_close, dev2udev(dev), (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0); return (0); } static int swapongeom(struct vnode *vp) { int error; vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); if (vp->v_type != VCHR || (vp->v_iflag & VI_DOOMED) != 0) { error = ENOENT; } else { g_topology_lock(); error = swapongeom_locked(vp->v_rdev, vp); g_topology_unlock(); } VOP_UNLOCK(vp, 0); return (error); } /* * VNODE backend * * This is used mainly for network filesystem (read: probably only tested * with NFS) swapfiles. * */ static void swapdev_strategy(struct buf *bp, struct swdevt *sp) { struct vnode *vp2; bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first); vp2 = sp->sw_id; vhold(vp2); if (bp->b_iocmd == BIO_WRITE) { if (bp->b_bufobj) bufobj_wdrop(bp->b_bufobj); bufobj_wref(&vp2->v_bufobj); } if (bp->b_bufobj != &vp2->v_bufobj) bp->b_bufobj = &vp2->v_bufobj; bp->b_vp = vp2; bp->b_iooffset = dbtob(bp->b_blkno); bstrategy(bp); return; } static void swapdev_close(struct thread *td, struct swdevt *sp) { VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td); vrele(sp->sw_vp); } static int swaponvp(struct thread *td, struct vnode *vp, u_long nblks) { struct swdevt *sp; int error; if (nblks == 0) return (ENXIO); mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (sp->sw_id == vp) { mtx_unlock(&sw_dev_mtx); return (EBUSY); } } mtx_unlock(&sw_dev_mtx); (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); #ifdef MAC error = mac_system_check_swapon(td->td_ucred, vp); if (error == 0) #endif error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL); (void) VOP_UNLOCK(vp, 0); if (error) return (error); swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close, NODEV, 0); return (0); } static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS) { int error, new, n; new = nsw_wcount_async_max; error = sysctl_handle_int(oidp, &new, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (new > nswbuf / 2 || new < 1) return (EINVAL); mtx_lock(&pbuf_mtx); while (nsw_wcount_async_max != new) { /* * Adjust difference. If the current async count is too low, * we will need to sqeeze our update slowly in. Sleep with a * higher priority than getpbuf() to finish faster. */ n = new - nsw_wcount_async_max; if (nsw_wcount_async + n >= 0) { nsw_wcount_async += n; nsw_wcount_async_max += n; wakeup(&nsw_wcount_async); } else { nsw_wcount_async_max -= nsw_wcount_async; nsw_wcount_async = 0; msleep(&nsw_wcount_async, &pbuf_mtx, PSWP, "swpsysctl", 0); } } mtx_unlock(&pbuf_mtx); return (0); } Index: projects/clang390-import/sys/vm/vm_page.c =================================================================== --- projects/clang390-import/sys/vm/vm_page.c (revision 305080) +++ projects/clang390-import/sys/vm/vm_page.c (revision 305081) @@ -1,3974 +1,3974 @@ /*- * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1998 Matthew Dillon. All Rights Reserved. * * This code is derived from software contributed to Berkeley by * The Mach Operating System project at Carnegie-Mellon University. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91 */ /*- * Copyright (c) 1987, 1990 Carnegie-Mellon University. * All rights reserved. * * Authors: Avadis Tevanian, Jr., Michael Wayne Young * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ /* * GENERAL RULES ON VM_PAGE MANIPULATION * * - A page queue lock is required when adding or removing a page from a * page queue regardless of other locks or the busy state of a page. * * * In general, no thread besides the page daemon can acquire or * hold more than one page queue lock at a time. * * * The page daemon can acquire and hold any pair of page queue * locks in any order. * * - The object lock is required when inserting or removing * pages from an object (vm_page_insert() or vm_page_remove()). * */ /* * Resident memory management module. */ #include __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Associated with page of user-allocatable memory is a * page structure. */ struct vm_domain vm_dom[MAXMEMDOM]; struct mtx_padalign vm_page_queue_free_mtx; struct mtx_padalign pa_lock[PA_LOCK_COUNT]; vm_page_t vm_page_array; long vm_page_array_size; long first_page; int vm_page_zero_count; static int boot_pages = UMA_BOOT_PAGES; SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &boot_pages, 0, "number of pages allocated for bootstrapping the VM system"); static int pa_tryrelock_restart; SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD, &pa_tryrelock_restart, 0, "Number of tryrelock restarts"); static TAILQ_HEAD(, vm_page) blacklist_head; static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages"); /* Is the page daemon waiting for free pages? */ static int vm_pageout_pages_needed; static uma_zone_t fakepg_zone; static struct vnode *vm_page_alloc_init(vm_page_t m); static void vm_page_cache_turn_free(vm_page_t m); static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); static void vm_page_enqueue(uint8_t queue, vm_page_t m); static void vm_page_free_wakeup(void); static void vm_page_init_fakepg(void *dummy); static int vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex, vm_page_t mpred); static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, vm_page_t mpred); static int vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run, vm_paddr_t high); SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init_fakepg, NULL); static void vm_page_init_fakepg(void *dummy) { fakepg_zone = uma_zcreate("fakepg", sizeof(struct vm_page), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM); } /* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */ #if PAGE_SIZE == 32768 #ifdef CTASSERT CTASSERT(sizeof(u_long) >= 8); #endif #endif /* * Try to acquire a physical address lock while a pmap is locked. If we * fail to trylock we unlock and lock the pmap directly and cache the * locked pa in *locked. The caller should then restart their loop in case * the virtual to physical mapping has changed. */ int vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) { vm_paddr_t lockpa; lockpa = *locked; *locked = pa; if (lockpa) { PA_LOCK_ASSERT(lockpa, MA_OWNED); if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa)) return (0); PA_UNLOCK(lockpa); } if (PA_TRYLOCK(pa)) return (0); PMAP_UNLOCK(pmap); atomic_add_int(&pa_tryrelock_restart, 1); PA_LOCK(pa); PMAP_LOCK(pmap); return (EAGAIN); } /* * vm_set_page_size: * * Sets the page size, perhaps based upon the memory * size. Must be called before any use of page-size * dependent functions. */ void vm_set_page_size(void) { if (vm_cnt.v_page_size == 0) vm_cnt.v_page_size = PAGE_SIZE; if (((vm_cnt.v_page_size - 1) & vm_cnt.v_page_size) != 0) panic("vm_set_page_size: page size not a power of two"); } /* * vm_page_blacklist_next: * * Find the next entry in the provided string of blacklist * addresses. Entries are separated by space, comma, or newline. * If an invalid integer is encountered then the rest of the * string is skipped. Updates the list pointer to the next * character, or NULL if the string is exhausted or invalid. */ static vm_paddr_t vm_page_blacklist_next(char **list, char *end) { vm_paddr_t bad; char *cp, *pos; if (list == NULL || *list == NULL) return (0); if (**list =='\0') { *list = NULL; return (0); } /* * If there's no end pointer then the buffer is coming from * the kenv and we know it's null-terminated. */ if (end == NULL) end = *list + strlen(*list); /* Ensure that strtoq() won't walk off the end */ if (*end != '\0') { if (*end == '\n' || *end == ' ' || *end == ',') *end = '\0'; else { printf("Blacklist not terminated, skipping\n"); *list = NULL; return (0); } } for (pos = *list; *pos != '\0'; pos = cp) { bad = strtoq(pos, &cp, 0); if (*cp == '\0' || *cp == ' ' || *cp == ',' || *cp == '\n') { if (bad == 0) { if (++cp < end) continue; else break; } } else break; if (*cp == '\0' || ++cp >= end) *list = NULL; else *list = cp; return (trunc_page(bad)); } printf("Garbage in RAM blacklist, skipping\n"); *list = NULL; return (0); } /* * vm_page_blacklist_check: * * Iterate through the provided string of blacklist addresses, pulling * each entry out of the physical allocator free list and putting it * onto a list for reporting via the vm.page_blacklist sysctl. */ static void vm_page_blacklist_check(char *list, char *end) { vm_paddr_t pa; vm_page_t m; char *next; int ret; next = list; while (next != NULL) { if ((pa = vm_page_blacklist_next(&next, end)) == 0) continue; m = vm_phys_paddr_to_vm_page(pa); if (m == NULL) continue; mtx_lock(&vm_page_queue_free_mtx); ret = vm_phys_unfree_page(m); mtx_unlock(&vm_page_queue_free_mtx); if (ret == TRUE) { TAILQ_INSERT_TAIL(&blacklist_head, m, listq); if (bootverbose) printf("Skipping page with pa 0x%jx\n", (uintmax_t)pa); } } } /* * vm_page_blacklist_load: * * Search for a special module named "ram_blacklist". It'll be a * plain text file provided by the user via the loader directive * of the same name. */ static void vm_page_blacklist_load(char **list, char **end) { void *mod; u_char *ptr; u_int len; mod = NULL; ptr = NULL; mod = preload_search_by_type("ram_blacklist"); if (mod != NULL) { ptr = preload_fetch_addr(mod); len = preload_fetch_size(mod); } *list = ptr; if (ptr != NULL) *end = ptr + len; else *end = NULL; return; } static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS) { vm_page_t m; struct sbuf sbuf; int error, first; first = 1; error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); TAILQ_FOREACH(m, &blacklist_head, listq) { sbuf_printf(&sbuf, "%s%#jx", first ? "" : ",", (uintmax_t)m->phys_addr); first = 0; } error = sbuf_finish(&sbuf); sbuf_delete(&sbuf); return (error); } static void vm_page_domain_init(struct vm_domain *vmd) { struct vm_pagequeue *pq; int i; *__DECONST(char **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_name) = "vm inactive pagequeue"; *__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_vcnt) = &vm_cnt.v_inactive_count; *__DECONST(char **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_name) = "vm active pagequeue"; *__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_vcnt) = &vm_cnt.v_active_count; vmd->vmd_page_count = 0; vmd->vmd_free_count = 0; vmd->vmd_segs = 0; vmd->vmd_oom = FALSE; vmd->vmd_pass = 0; for (i = 0; i < PQ_COUNT; i++) { pq = &vmd->vmd_pagequeues[i]; TAILQ_INIT(&pq->pq_pl); mtx_init(&pq->pq_mutex, pq->pq_name, "vm pagequeue", MTX_DEF | MTX_DUPOK); } } /* * vm_page_startup: * * Initializes the resident memory module. * * Allocates memory for the page cells, and * for the object/offset-to-page hash table headers. * Each page cell is initialized and placed on the free list. */ vm_offset_t vm_page_startup(vm_offset_t vaddr) { vm_offset_t mapped; vm_paddr_t page_range; vm_paddr_t new_end; int i; vm_paddr_t pa; vm_paddr_t last_pa; char *list, *listend; vm_paddr_t end; vm_paddr_t biggestsize; vm_paddr_t low_water, high_water; int biggestone; int pages_per_zone; biggestsize = 0; biggestone = 0; vaddr = round_page(vaddr); for (i = 0; phys_avail[i + 1]; i += 2) { phys_avail[i] = round_page(phys_avail[i]); phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); } low_water = phys_avail[0]; high_water = phys_avail[1]; for (i = 0; i < vm_phys_nsegs; i++) { if (vm_phys_segs[i].start < low_water) low_water = vm_phys_segs[i].start; if (vm_phys_segs[i].end > high_water) high_water = vm_phys_segs[i].end; } for (i = 0; phys_avail[i + 1]; i += 2) { vm_paddr_t size = phys_avail[i + 1] - phys_avail[i]; if (size > biggestsize) { biggestone = i; biggestsize = size; } if (phys_avail[i] < low_water) low_water = phys_avail[i]; if (phys_avail[i + 1] > high_water) high_water = phys_avail[i + 1]; } end = phys_avail[biggestone+1]; /* * Initialize the page and queue locks. */ mtx_init(&vm_page_queue_free_mtx, "vm page free queue", NULL, MTX_DEF); for (i = 0; i < PA_LOCK_COUNT; i++) mtx_init(&pa_lock[i], "vm page", NULL, MTX_DEF); for (i = 0; i < vm_ndomains; i++) vm_page_domain_init(&vm_dom[i]); /* * Almost all of the pages needed for boot strapping UMA are used * for zone structures, so if the number of CPUs results in those * structures taking more than one page each, we set aside more pages * in proportion to the zone structure size. */ pages_per_zone = howmany(sizeof(struct uma_zone) + sizeof(struct uma_cache) * (mp_maxid + 1), UMA_SLAB_SIZE); if (pages_per_zone > 1) { /* Reserve more pages so that we don't run out. */ boot_pages = UMA_BOOT_PAGES_ZONES * pages_per_zone; } /* * Allocate memory for use when boot strapping the kernel memory * allocator. * * CTFLAG_RDTUN doesn't work during the early boot process, so we must * manually fetch the value. */ TUNABLE_INT_FETCH("vm.boot_pages", &boot_pages); new_end = end - (boot_pages * UMA_SLAB_SIZE); new_end = trunc_page(new_end); mapped = pmap_map(&vaddr, new_end, end, VM_PROT_READ | VM_PROT_WRITE); bzero((void *)mapped, end - new_end); uma_startup((void *)mapped, boot_pages); #if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ defined(__i386__) || defined(__mips__) /* * Allocate a bitmap to indicate that a random physical page * needs to be included in a minidump. * * The amd64 port needs this to indicate which direct map pages * need to be dumped, via calls to dump_add_page()/dump_drop_page(). * * However, i386 still needs this workspace internally within the * minidump code. In theory, they are not needed on i386, but are * included should the sf_buf code decide to use them. */ last_pa = 0; for (i = 0; dump_avail[i + 1] != 0; i += 2) if (dump_avail[i + 1] > last_pa) last_pa = dump_avail[i + 1]; page_range = last_pa / PAGE_SIZE; vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY); new_end -= vm_page_dump_size; vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end, new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE); bzero((void *)vm_page_dump, vm_page_dump_size); #endif #ifdef __amd64__ /* * Request that the physical pages underlying the message buffer be * included in a crash dump. Since the message buffer is accessed * through the direct map, they are not automatically included. */ pa = DMAP_TO_PHYS((vm_offset_t)msgbufp->msg_ptr); last_pa = pa + round_page(msgbufsize); while (pa < last_pa) { dump_add_page(pa); pa += PAGE_SIZE; } #endif /* * Compute the number of pages of memory that will be available for * use (taking into account the overhead of a page structure per * page). */ first_page = low_water / PAGE_SIZE; #ifdef VM_PHYSSEG_SPARSE page_range = 0; for (i = 0; i < vm_phys_nsegs; i++) { page_range += atop(vm_phys_segs[i].end - vm_phys_segs[i].start); } for (i = 0; phys_avail[i + 1] != 0; i += 2) page_range += atop(phys_avail[i + 1] - phys_avail[i]); #elif defined(VM_PHYSSEG_DENSE) page_range = high_water / PAGE_SIZE - first_page; #else #error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined." #endif end = new_end; /* * Reserve an unmapped guard page to trap access to vm_page_array[-1]. */ vaddr += PAGE_SIZE; /* * Initialize the mem entry structures now, and put them in the free * queue. */ new_end = trunc_page(end - page_range * sizeof(struct vm_page)); mapped = pmap_map(&vaddr, new_end, end, VM_PROT_READ | VM_PROT_WRITE); vm_page_array = (vm_page_t) mapped; #if VM_NRESERVLEVEL > 0 /* * Allocate memory for the reservation management system's data * structures. */ new_end = vm_reserv_startup(&vaddr, new_end, high_water); #endif #if defined(__aarch64__) || defined(__amd64__) || defined(__mips__) /* * pmap_map on arm64, amd64, and mips can come out of the direct-map, * not kvm like i386, so the pages must be tracked for a crashdump to * include this data. This includes the vm_page_array and the early * UMA bootstrap pages. */ for (pa = new_end; pa < phys_avail[biggestone + 1]; pa += PAGE_SIZE) dump_add_page(pa); #endif phys_avail[biggestone + 1] = new_end; /* * Add physical memory segments corresponding to the available * physical pages. */ for (i = 0; phys_avail[i + 1] != 0; i += 2) vm_phys_add_seg(phys_avail[i], phys_avail[i + 1]); /* * Clear all of the page structures */ bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page)); for (i = 0; i < page_range; i++) vm_page_array[i].order = VM_NFREEORDER; vm_page_array_size = page_range; /* * Initialize the physical memory allocator. */ vm_phys_init(); /* * Add every available physical page that is not blacklisted to * the free lists. */ vm_cnt.v_page_count = 0; vm_cnt.v_free_count = 0; for (i = 0; phys_avail[i + 1] != 0; i += 2) { pa = phys_avail[i]; last_pa = phys_avail[i + 1]; while (pa < last_pa) { vm_phys_add_page(pa); pa += PAGE_SIZE; } } TAILQ_INIT(&blacklist_head); vm_page_blacklist_load(&list, &listend); vm_page_blacklist_check(list, listend); list = kern_getenv("vm.blacklist"); vm_page_blacklist_check(list, NULL); freeenv(list); #if VM_NRESERVLEVEL > 0 /* * Initialize the reservation management system. */ vm_reserv_init(); #endif return (vaddr); } void vm_page_reference(vm_page_t m) { vm_page_aflag_set(m, PGA_REFERENCED); } /* * vm_page_busy_downgrade: * * Downgrade an exclusive busy page into a single shared busy page. */ void vm_page_busy_downgrade(vm_page_t m) { u_int x; vm_page_assert_xbusied(m); for (;;) { x = m->busy_lock; x &= VPB_BIT_WAITERS; if (atomic_cmpset_rel_int(&m->busy_lock, VPB_SINGLE_EXCLUSIVER | x, VPB_SHARERS_WORD(1) | x)) break; } } /* * vm_page_sbusied: * * Return a positive value if the page is shared busied, 0 otherwise. */ int vm_page_sbusied(vm_page_t m) { u_int x; x = m->busy_lock; return ((x & VPB_BIT_SHARED) != 0 && x != VPB_UNBUSIED); } /* * vm_page_sunbusy: * * Shared unbusy a page. */ void vm_page_sunbusy(vm_page_t m) { u_int x; vm_page_assert_sbusied(m); for (;;) { x = m->busy_lock; if (VPB_SHARERS(x) > 1) { if (atomic_cmpset_int(&m->busy_lock, x, x - VPB_ONE_SHARER)) break; continue; } if ((x & VPB_BIT_WAITERS) == 0) { KASSERT(x == VPB_SHARERS_WORD(1), ("vm_page_sunbusy: invalid lock state")); if (atomic_cmpset_int(&m->busy_lock, VPB_SHARERS_WORD(1), VPB_UNBUSIED)) break; continue; } KASSERT(x == (VPB_SHARERS_WORD(1) | VPB_BIT_WAITERS), ("vm_page_sunbusy: invalid lock state for waiters")); vm_page_lock(m); if (!atomic_cmpset_int(&m->busy_lock, x, VPB_UNBUSIED)) { vm_page_unlock(m); continue; } wakeup(m); vm_page_unlock(m); break; } } /* * vm_page_busy_sleep: * * Sleep and release the page lock, using the page pointer as wchan. * This is used to implement the hard-path of busying mechanism. * * The given page must be locked. */ void vm_page_busy_sleep(vm_page_t m, const char *wmesg) { u_int x; vm_page_lock_assert(m, MA_OWNED); x = m->busy_lock; if (x == VPB_UNBUSIED) { vm_page_unlock(m); return; } if ((x & VPB_BIT_WAITERS) == 0 && !atomic_cmpset_int(&m->busy_lock, x, x | VPB_BIT_WAITERS)) { vm_page_unlock(m); return; } msleep(m, vm_page_lockptr(m), PVM | PDROP, wmesg, 0); } /* * vm_page_trysbusy: * * Try to shared busy a page. * If the operation succeeds 1 is returned otherwise 0. * The operation never sleeps. */ int vm_page_trysbusy(vm_page_t m) { u_int x; for (;;) { x = m->busy_lock; if ((x & VPB_BIT_SHARED) == 0) return (0); if (atomic_cmpset_acq_int(&m->busy_lock, x, x + VPB_ONE_SHARER)) return (1); } } static void vm_page_xunbusy_locked(vm_page_t m) { vm_page_assert_xbusied(m); vm_page_assert_locked(m); atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED); /* There is a waiter, do wakeup() instead of vm_page_flash(). */ wakeup(m); } static void vm_page_xunbusy_maybelocked(vm_page_t m) { bool lockacq; vm_page_assert_xbusied(m); /* * Fast path for unbusy. If it succeeds, we know that there * are no waiters, so we do not need a wakeup. */ if (atomic_cmpset_rel_int(&m->busy_lock, VPB_SINGLE_EXCLUSIVER, VPB_UNBUSIED)) return; lockacq = !mtx_owned(vm_page_lockptr(m)); if (lockacq) vm_page_lock(m); vm_page_xunbusy_locked(m); if (lockacq) vm_page_unlock(m); } /* * vm_page_xunbusy_hard: * * Called after the first try the exclusive unbusy of a page failed. * It is assumed that the waiters bit is on. */ void vm_page_xunbusy_hard(vm_page_t m) { vm_page_assert_xbusied(m); vm_page_lock(m); vm_page_xunbusy_locked(m); vm_page_unlock(m); } /* * vm_page_flash: * * Wakeup anyone waiting for the page. * The ownership bits do not change. * * The given page must be locked. */ void vm_page_flash(vm_page_t m) { u_int x; vm_page_lock_assert(m, MA_OWNED); for (;;) { x = m->busy_lock; if ((x & VPB_BIT_WAITERS) == 0) return; if (atomic_cmpset_int(&m->busy_lock, x, x & (~VPB_BIT_WAITERS))) break; } wakeup(m); } /* * Keep page from being freed by the page daemon * much of the same effect as wiring, except much lower * overhead and should be used only for *very* temporary * holding ("wiring"). */ void vm_page_hold(vm_page_t mem) { vm_page_lock_assert(mem, MA_OWNED); mem->hold_count++; } void vm_page_unhold(vm_page_t mem) { vm_page_lock_assert(mem, MA_OWNED); KASSERT(mem->hold_count >= 1, ("vm_page_unhold: hold count < 0!!!")); --mem->hold_count; if (mem->hold_count == 0 && (mem->flags & PG_UNHOLDFREE) != 0) vm_page_free_toq(mem); } /* * vm_page_unhold_pages: * * Unhold each of the pages that is referenced by the given array. */ void vm_page_unhold_pages(vm_page_t *ma, int count) { struct mtx *mtx, *new_mtx; mtx = NULL; for (; count != 0; count--) { /* * Avoid releasing and reacquiring the same page lock. */ new_mtx = vm_page_lockptr(*ma); if (mtx != new_mtx) { if (mtx != NULL) mtx_unlock(mtx); mtx = new_mtx; mtx_lock(mtx); } vm_page_unhold(*ma); ma++; } if (mtx != NULL) mtx_unlock(mtx); } vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa) { vm_page_t m; #ifdef VM_PHYSSEG_SPARSE m = vm_phys_paddr_to_vm_page(pa); if (m == NULL) m = vm_phys_fictitious_to_vm_page(pa); return (m); #elif defined(VM_PHYSSEG_DENSE) long pi; pi = atop(pa); if (pi >= first_page && (pi - first_page) < vm_page_array_size) { m = &vm_page_array[pi - first_page]; return (m); } return (vm_phys_fictitious_to_vm_page(pa)); #else #error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined." #endif } /* * vm_page_getfake: * * Create a fictitious page with the specified physical address and * memory attribute. The memory attribute is the only the machine- * dependent aspect of a fictitious page that must be initialized. */ vm_page_t vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr) { vm_page_t m; m = uma_zalloc(fakepg_zone, M_WAITOK | M_ZERO); vm_page_initfake(m, paddr, memattr); return (m); } void vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr) { if ((m->flags & PG_FICTITIOUS) != 0) { /* * The page's memattr might have changed since the * previous initialization. Update the pmap to the * new memattr. */ goto memattr; } m->phys_addr = paddr; m->queue = PQ_NONE; /* Fictitious pages don't use "segind". */ m->flags = PG_FICTITIOUS; /* Fictitious pages don't use "order" or "pool". */ m->oflags = VPO_UNMANAGED; m->busy_lock = VPB_SINGLE_EXCLUSIVER; m->wire_count = 1; pmap_page_init(m); memattr: pmap_page_set_memattr(m, memattr); } /* * vm_page_putfake: * * Release a fictitious page. */ void vm_page_putfake(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) != 0, ("managed %p", m)); KASSERT((m->flags & PG_FICTITIOUS) != 0, ("vm_page_putfake: bad page %p", m)); uma_zfree(fakepg_zone, m); } /* * vm_page_updatefake: * * Update the given fictitious page to the specified physical address and * memory attribute. */ void vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr) { KASSERT((m->flags & PG_FICTITIOUS) != 0, ("vm_page_updatefake: bad page %p", m)); m->phys_addr = paddr; pmap_page_set_memattr(m, memattr); } /* * vm_page_free: * * Free a page. */ void vm_page_free(vm_page_t m) { m->flags &= ~PG_ZERO; vm_page_free_toq(m); } /* * vm_page_free_zero: * * Free a page to the zerod-pages queue */ void vm_page_free_zero(vm_page_t m) { m->flags |= PG_ZERO; vm_page_free_toq(m); } /* - * Unbusy and handle the page queueing for a page from the VOP_GETPAGES() - * array which was optionally read ahead or behind. + * Unbusy and handle the page queueing for a page from a getpages request that + * was optionally read ahead or behind. */ void vm_page_readahead_finish(vm_page_t m) { /* We shouldn't put invalid pages on queues. */ KASSERT(m->valid != 0, ("%s: %p is invalid", __func__, m)); /* * Since the page is not the actually needed one, whether it should * be activated or deactivated is not obvious. Empirical results * have shown that deactivating the page is usually the best choice, * unless the page is wanted by another thread. */ vm_page_lock(m); if ((m->busy_lock & VPB_BIT_WAITERS) != 0) vm_page_activate(m); else vm_page_deactivate(m); vm_page_unlock(m); vm_page_xunbusy(m); } /* * vm_page_sleep_if_busy: * * Sleep and release the page queues lock if the page is busied. * Returns TRUE if the thread slept. * * The given page must be unlocked and object containing it must * be locked. */ int vm_page_sleep_if_busy(vm_page_t m, const char *msg) { vm_object_t obj; vm_page_lock_assert(m, MA_NOTOWNED); VM_OBJECT_ASSERT_WLOCKED(m->object); if (vm_page_busied(m)) { /* * The page-specific object must be cached because page * identity can change during the sleep, causing the * re-lock of a different object. * It is assumed that a reference to the object is already * held by the callers. */ obj = m->object; vm_page_lock(m); VM_OBJECT_WUNLOCK(obj); vm_page_busy_sleep(m, msg); VM_OBJECT_WLOCK(obj); return (TRUE); } return (FALSE); } /* * vm_page_dirty_KBI: [ internal use only ] * * Set all bits in the page's dirty field. * * The object containing the specified page must be locked if the * call is made from the machine-independent layer. * * See vm_page_clear_dirty_mask(). * * This function should only be called by vm_page_dirty(). */ void vm_page_dirty_KBI(vm_page_t m) { /* These assertions refer to this operation by its public name. */ KASSERT((m->flags & PG_CACHED) == 0, ("vm_page_dirty: page in cache!")); KASSERT(m->valid == VM_PAGE_BITS_ALL, ("vm_page_dirty: page is invalid!")); m->dirty = VM_PAGE_BITS_ALL; } /* * vm_page_insert: [ internal use only ] * * Inserts the given mem entry into the object and object list. * * The object must be locked. */ int vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex) { vm_page_t mpred; VM_OBJECT_ASSERT_WLOCKED(object); mpred = vm_radix_lookup_le(&object->rtree, pindex); return (vm_page_insert_after(m, object, pindex, mpred)); } /* * vm_page_insert_after: * * Inserts the page "m" into the specified object at offset "pindex". * * The page "mpred" must immediately precede the offset "pindex" within * the specified object. * * The object must be locked. */ static int vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex, vm_page_t mpred) { vm_page_t msucc; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(m->object == NULL, ("vm_page_insert_after: page already inserted")); if (mpred != NULL) { KASSERT(mpred->object == object, ("vm_page_insert_after: object doesn't contain mpred")); KASSERT(mpred->pindex < pindex, ("vm_page_insert_after: mpred doesn't precede pindex")); msucc = TAILQ_NEXT(mpred, listq); } else msucc = TAILQ_FIRST(&object->memq); if (msucc != NULL) KASSERT(msucc->pindex > pindex, ("vm_page_insert_after: msucc doesn't succeed pindex")); /* * Record the object/offset pair in this page */ m->object = object; m->pindex = pindex; /* * Now link into the object's ordered list of backed pages. */ if (vm_radix_insert(&object->rtree, m)) { m->object = NULL; m->pindex = 0; return (1); } vm_page_insert_radixdone(m, object, mpred); return (0); } /* * vm_page_insert_radixdone: * * Complete page "m" insertion into the specified object after the * radix trie hooking. * * The page "mpred" must precede the offset "m->pindex" within the * specified object. * * The object must be locked. */ static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, vm_page_t mpred) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object != NULL && m->object == object, ("vm_page_insert_radixdone: page %p has inconsistent object", m)); if (mpred != NULL) { KASSERT(mpred->object == object, ("vm_page_insert_after: object doesn't contain mpred")); KASSERT(mpred->pindex < m->pindex, ("vm_page_insert_after: mpred doesn't precede pindex")); } if (mpred != NULL) TAILQ_INSERT_AFTER(&object->memq, mpred, m, listq); else TAILQ_INSERT_HEAD(&object->memq, m, listq); /* * Show that the object has one more resident page. */ object->resident_page_count++; /* * Hold the vnode until the last page is released. */ if (object->resident_page_count == 1 && object->type == OBJT_VNODE) vhold(object->handle); /* * Since we are inserting a new and possibly dirty page, * update the object's OBJ_MIGHTBEDIRTY flag. */ if (pmap_page_is_write_mapped(m)) vm_object_set_writeable_dirty(object); } /* * vm_page_remove: * * Removes the given mem entry from the object/offset-page * table and the object page list, but do not invalidate/terminate * the backing store. * * The object must be locked. The page must be locked if it is managed. */ void vm_page_remove(vm_page_t m) { vm_object_t object; if ((m->oflags & VPO_UNMANAGED) == 0) vm_page_assert_locked(m); if ((object = m->object) == NULL) return; VM_OBJECT_ASSERT_WLOCKED(object); if (vm_page_xbusied(m)) vm_page_xunbusy_maybelocked(m); /* * Now remove from the object's list of backed pages. */ vm_radix_remove(&object->rtree, m->pindex); TAILQ_REMOVE(&object->memq, m, listq); /* * And show that the object has one fewer resident page. */ object->resident_page_count--; /* * The vnode may now be recycled. */ if (object->resident_page_count == 0 && object->type == OBJT_VNODE) vdrop(object->handle); m->object = NULL; } /* * vm_page_lookup: * * Returns the page associated with the object/offset * pair specified; if none is found, NULL is returned. * * The object must be locked. */ vm_page_t vm_page_lookup(vm_object_t object, vm_pindex_t pindex) { VM_OBJECT_ASSERT_LOCKED(object); return (vm_radix_lookup(&object->rtree, pindex)); } /* * vm_page_find_least: * * Returns the page associated with the object with least pindex * greater than or equal to the parameter pindex, or NULL. * * The object must be locked. */ vm_page_t vm_page_find_least(vm_object_t object, vm_pindex_t pindex) { vm_page_t m; VM_OBJECT_ASSERT_LOCKED(object); if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex) m = vm_radix_lookup_ge(&object->rtree, pindex); return (m); } /* * Returns the given page's successor (by pindex) within the object if it is * resident; if none is found, NULL is returned. * * The object must be locked. */ vm_page_t vm_page_next(vm_page_t m) { vm_page_t next; VM_OBJECT_ASSERT_LOCKED(m->object); if ((next = TAILQ_NEXT(m, listq)) != NULL && next->pindex != m->pindex + 1) next = NULL; return (next); } /* * Returns the given page's predecessor (by pindex) within the object if it is * resident; if none is found, NULL is returned. * * The object must be locked. */ vm_page_t vm_page_prev(vm_page_t m) { vm_page_t prev; VM_OBJECT_ASSERT_LOCKED(m->object); if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL && prev->pindex != m->pindex - 1) prev = NULL; return (prev); } /* * Uses the page mnew as a replacement for an existing page at index * pindex which must be already present in the object. * * The existing page must not be on a paging queue. */ vm_page_t vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex) { vm_page_t mold; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(mnew->object == NULL, ("vm_page_replace: page already in object")); /* * This function mostly follows vm_page_insert() and * vm_page_remove() without the radix, object count and vnode * dance. Double check such functions for more comments. */ mnew->object = object; mnew->pindex = pindex; mold = vm_radix_replace(&object->rtree, mnew); KASSERT(mold->queue == PQ_NONE, ("vm_page_replace: mold is on a paging queue")); /* Keep the resident page list in sorted order. */ TAILQ_INSERT_AFTER(&object->memq, mold, mnew, listq); TAILQ_REMOVE(&object->memq, mold, listq); mold->object = NULL; vm_page_xunbusy_maybelocked(mold); /* * The object's resident_page_count does not change because we have * swapped one page for another, but OBJ_MIGHTBEDIRTY. */ if (pmap_page_is_write_mapped(mnew)) vm_object_set_writeable_dirty(object); return (mold); } /* * vm_page_rename: * * Move the given memory entry from its * current object to the specified target object/offset. * * Note: swap associated with the page must be invalidated by the move. We * have to do this for several reasons: (1) we aren't freeing the * page, (2) we are dirtying the page, (3) the VM system is probably * moving the page from object A to B, and will then later move * the backing store from A to B and we can't have a conflict. * * Note: we *always* dirty the page. It is necessary both for the * fact that we moved it, and because we may be invalidating * swap. If the page is on the cache, we have to deactivate it * or vm_page_dirty() will panic. Dirty pages are not allowed * on the cache. * * The objects must be locked. */ int vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex) { vm_page_t mpred; vm_pindex_t opidx; VM_OBJECT_ASSERT_WLOCKED(new_object); mpred = vm_radix_lookup_le(&new_object->rtree, new_pindex); KASSERT(mpred == NULL || mpred->pindex != new_pindex, ("vm_page_rename: pindex already renamed")); /* * Create a custom version of vm_page_insert() which does not depend * by m_prev and can cheat on the implementation aspects of the * function. */ opidx = m->pindex; m->pindex = new_pindex; if (vm_radix_insert(&new_object->rtree, m)) { m->pindex = opidx; return (1); } /* * The operation cannot fail anymore. The removal must happen before * the listq iterator is tainted. */ m->pindex = opidx; vm_page_lock(m); vm_page_remove(m); /* Return back to the new pindex to complete vm_page_insert(). */ m->pindex = new_pindex; m->object = new_object; vm_page_unlock(m); vm_page_insert_radixdone(m, new_object, mpred); vm_page_dirty(m); return (0); } /* * Convert all of the given object's cached pages that have a * pindex within the given range into free pages. If the value * zero is given for "end", then the range's upper bound is * infinity. If the given object is backed by a vnode and it * transitions from having one or more cached pages to none, the * vnode's hold count is reduced. */ void vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end) { vm_page_t m; boolean_t empty; mtx_lock(&vm_page_queue_free_mtx); if (__predict_false(vm_radix_is_empty(&object->cache))) { mtx_unlock(&vm_page_queue_free_mtx); return; } while ((m = vm_radix_lookup_ge(&object->cache, start)) != NULL) { if (end != 0 && m->pindex >= end) break; vm_radix_remove(&object->cache, m->pindex); vm_page_cache_turn_free(m); } empty = vm_radix_is_empty(&object->cache); mtx_unlock(&vm_page_queue_free_mtx); if (object->type == OBJT_VNODE && empty) vdrop(object->handle); } /* * Returns the cached page that is associated with the given * object and offset. If, however, none exists, returns NULL. * * The free page queue must be locked. */ static inline vm_page_t vm_page_cache_lookup(vm_object_t object, vm_pindex_t pindex) { mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); return (vm_radix_lookup(&object->cache, pindex)); } /* * Remove the given cached page from its containing object's * collection of cached pages. * * The free page queue must be locked. */ static void vm_page_cache_remove(vm_page_t m) { mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); KASSERT((m->flags & PG_CACHED) != 0, ("vm_page_cache_remove: page %p is not cached", m)); vm_radix_remove(&m->object->cache, m->pindex); m->object = NULL; vm_cnt.v_cache_count--; } /* * Transfer all of the cached pages with offset greater than or * equal to 'offidxstart' from the original object's cache to the * new object's cache. However, any cached pages with offset * greater than or equal to the new object's size are kept in the * original object. Initially, the new object's cache must be * empty. Offset 'offidxstart' in the original object must * correspond to offset zero in the new object. * * The new object must be locked. */ void vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart, vm_object_t new_object) { vm_page_t m; /* * Insertion into an object's collection of cached pages * requires the object to be locked. In contrast, removal does * not. */ VM_OBJECT_ASSERT_WLOCKED(new_object); KASSERT(vm_radix_is_empty(&new_object->cache), ("vm_page_cache_transfer: object %p has cached pages", new_object)); mtx_lock(&vm_page_queue_free_mtx); while ((m = vm_radix_lookup_ge(&orig_object->cache, offidxstart)) != NULL) { /* * Transfer all of the pages with offset greater than or * equal to 'offidxstart' from the original object's * cache to the new object's cache. */ if ((m->pindex - offidxstart) >= new_object->size) break; vm_radix_remove(&orig_object->cache, m->pindex); /* Update the page's object and offset. */ m->object = new_object; m->pindex -= offidxstart; if (vm_radix_insert(&new_object->cache, m)) vm_page_cache_turn_free(m); } mtx_unlock(&vm_page_queue_free_mtx); } /* * Returns TRUE if a cached page is associated with the given object and * offset, and FALSE otherwise. * * The object must be locked. */ boolean_t vm_page_is_cached(vm_object_t object, vm_pindex_t pindex) { vm_page_t m; /* * Insertion into an object's collection of cached pages requires the * object to be locked. Therefore, if the object is locked and the * object's collection is empty, there is no need to acquire the free * page queues lock in order to prove that the specified page doesn't * exist. */ VM_OBJECT_ASSERT_WLOCKED(object); if (__predict_true(vm_object_cache_is_empty(object))) return (FALSE); mtx_lock(&vm_page_queue_free_mtx); m = vm_page_cache_lookup(object, pindex); mtx_unlock(&vm_page_queue_free_mtx); return (m != NULL); } /* * vm_page_alloc: * * Allocate and return a page that is associated with the specified * object and offset pair. By default, this page is exclusive busied. * * The caller must always specify an allocation class. * * allocation classes: * VM_ALLOC_NORMAL normal process request * VM_ALLOC_SYSTEM system *really* needs a page * VM_ALLOC_INTERRUPT interrupt time request * * optional allocation flags: * VM_ALLOC_COUNT(number) the number of additional pages that the caller * intends to allocate * VM_ALLOC_IFCACHED return page only if it is cached * VM_ALLOC_IFNOTCACHED return NULL, do not reactivate if the page * is cached * VM_ALLOC_NOBUSY do not exclusive busy the page * VM_ALLOC_NODUMP do not include the page in a kernel core dump * VM_ALLOC_NOOBJ page is not associated with an object and * should not be exclusive busy * VM_ALLOC_SBUSY shared busy the allocated page * VM_ALLOC_WIRED wire the allocated page * VM_ALLOC_ZERO prefer a zeroed page * * This routine may not sleep. */ vm_page_t vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) { struct vnode *vp = NULL; vm_object_t m_object; vm_page_t m, mpred; int flags, req_class; mpred = 0; /* XXX: pacify gcc */ KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), ("vm_page_alloc: inconsistent object(%p)/req(%x)", (void *)object, req)); if (object != NULL) VM_OBJECT_ASSERT_WLOCKED(object); req_class = req & VM_ALLOC_CLASS_MASK; /* * The page daemon is allowed to dig deeper into the free page list. */ if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) req_class = VM_ALLOC_SYSTEM; if (object != NULL) { mpred = vm_radix_lookup_le(&object->rtree, pindex); KASSERT(mpred == NULL || mpred->pindex != pindex, ("vm_page_alloc: pindex already allocated")); } /* * The page allocation request can came from consumers which already * hold the free page queue mutex, like vm_page_insert() in * vm_page_cache(). */ mtx_lock_flags(&vm_page_queue_free_mtx, MTX_RECURSE); if (vm_cnt.v_free_count + vm_cnt.v_cache_count > vm_cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM && vm_cnt.v_free_count + vm_cnt.v_cache_count > vm_cnt.v_interrupt_free_min) || (req_class == VM_ALLOC_INTERRUPT && vm_cnt.v_free_count + vm_cnt.v_cache_count > 0)) { /* * Allocate from the free queue if the number of free pages * exceeds the minimum for the request class. */ if (object != NULL && (m = vm_page_cache_lookup(object, pindex)) != NULL) { if ((req & VM_ALLOC_IFNOTCACHED) != 0) { mtx_unlock(&vm_page_queue_free_mtx); return (NULL); } if (vm_phys_unfree_page(m)) vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, 0); #if VM_NRESERVLEVEL > 0 else if (!vm_reserv_reactivate_page(m)) #else else #endif panic("vm_page_alloc: cache page %p is missing" " from the free queue", m); } else if ((req & VM_ALLOC_IFCACHED) != 0) { mtx_unlock(&vm_page_queue_free_mtx); return (NULL); #if VM_NRESERVLEVEL > 0 } else if (object == NULL || (object->flags & (OBJ_COLORED | OBJ_FICTITIOUS)) != OBJ_COLORED || (m = vm_reserv_alloc_page(object, pindex, mpred)) == NULL) { #else } else { #endif m = vm_phys_alloc_pages(object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); #if VM_NRESERVLEVEL > 0 if (m == NULL && vm_reserv_reclaim_inactive()) { m = vm_phys_alloc_pages(object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); } #endif } } else { /* * Not allocatable, give up. */ mtx_unlock(&vm_page_queue_free_mtx); atomic_add_int(&vm_pageout_deficit, max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); pagedaemon_wakeup(); return (NULL); } /* * At this point we had better have found a good page. */ KASSERT(m != NULL, ("vm_page_alloc: missing page")); KASSERT(m->queue == PQ_NONE, ("vm_page_alloc: page %p has unexpected queue %d", m, m->queue)); KASSERT(m->wire_count == 0, ("vm_page_alloc: page %p is wired", m)); KASSERT(m->hold_count == 0, ("vm_page_alloc: page %p is held", m)); KASSERT(!vm_page_busied(m), ("vm_page_alloc: page %p is busy", m)); KASSERT(m->dirty == 0, ("vm_page_alloc: page %p is dirty", m)); KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, ("vm_page_alloc: page %p has unexpected memattr %d", m, pmap_page_get_memattr(m))); if ((m->flags & PG_CACHED) != 0) { KASSERT((m->flags & PG_ZERO) == 0, ("vm_page_alloc: cached page %p is PG_ZERO", m)); KASSERT(m->valid != 0, ("vm_page_alloc: cached page %p is invalid", m)); if (m->object == object && m->pindex == pindex) vm_cnt.v_reactivated++; else m->valid = 0; m_object = m->object; vm_page_cache_remove(m); if (m_object->type == OBJT_VNODE && vm_object_cache_is_empty(m_object)) vp = m_object->handle; } else { KASSERT(m->valid == 0, ("vm_page_alloc: free page %p is valid", m)); vm_phys_freecnt_adj(m, -1); if ((m->flags & PG_ZERO) != 0) vm_page_zero_count--; } mtx_unlock(&vm_page_queue_free_mtx); /* * Initialize the page. Only the PG_ZERO flag is inherited. */ flags = 0; if ((req & VM_ALLOC_ZERO) != 0) flags = PG_ZERO; flags &= m->flags; if ((req & VM_ALLOC_NODUMP) != 0) flags |= PG_NODUMP; m->flags = flags; m->aflags = 0; m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; m->busy_lock = VPB_UNBUSIED; if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) m->busy_lock = VPB_SINGLE_EXCLUSIVER; if ((req & VM_ALLOC_SBUSY) != 0) m->busy_lock = VPB_SHARERS_WORD(1); if (req & VM_ALLOC_WIRED) { /* * The page lock is not required for wiring a page until that * page is inserted into the object. */ atomic_add_int(&vm_cnt.v_wire_count, 1); m->wire_count = 1; } m->act_count = 0; if (object != NULL) { if (vm_page_insert_after(m, object, pindex, mpred)) { /* See the comment below about hold count. */ if (vp != NULL) vdrop(vp); pagedaemon_wakeup(); if (req & VM_ALLOC_WIRED) { atomic_subtract_int(&vm_cnt.v_wire_count, 1); m->wire_count = 0; } m->object = NULL; m->oflags = VPO_UNMANAGED; m->busy_lock = VPB_UNBUSIED; vm_page_free(m); return (NULL); } /* Ignore device objects; the pager sets "memattr" for them. */ if (object->memattr != VM_MEMATTR_DEFAULT && (object->flags & OBJ_FICTITIOUS) == 0) pmap_page_set_memattr(m, object->memattr); } else m->pindex = pindex; /* * The following call to vdrop() must come after the above call * to vm_page_insert() in case both affect the same object and * vnode. Otherwise, the affected vnode's hold count could * temporarily become zero. */ if (vp != NULL) vdrop(vp); /* * Don't wakeup too often - wakeup the pageout daemon when * we would be nearly out of memory. */ if (vm_paging_needed()) pagedaemon_wakeup(); return (m); } static void vm_page_alloc_contig_vdrop(struct spglist *lst) { while (!SLIST_EMPTY(lst)) { vdrop((struct vnode *)SLIST_FIRST(lst)-> plinks.s.pv); SLIST_REMOVE_HEAD(lst, plinks.s.ss); } } /* * vm_page_alloc_contig: * * Allocate a contiguous set of physical pages of the given size "npages" * from the free lists. All of the physical pages must be at or above * the given physical address "low" and below the given physical address * "high". The given value "alignment" determines the alignment of the * first physical page in the set. If the given value "boundary" is * non-zero, then the set of physical pages cannot cross any physical * address boundary that is a multiple of that value. Both "alignment" * and "boundary" must be a power of two. * * If the specified memory attribute, "memattr", is VM_MEMATTR_DEFAULT, * then the memory attribute setting for the physical pages is configured * to the object's memory attribute setting. Otherwise, the memory * attribute setting for the physical pages is configured to "memattr", * overriding the object's memory attribute setting. However, if the * object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the * memory attribute setting for the physical pages cannot be configured * to VM_MEMATTR_DEFAULT. * * The caller must always specify an allocation class. * * allocation classes: * VM_ALLOC_NORMAL normal process request * VM_ALLOC_SYSTEM system *really* needs a page * VM_ALLOC_INTERRUPT interrupt time request * * optional allocation flags: * VM_ALLOC_NOBUSY do not exclusive busy the page * VM_ALLOC_NODUMP do not include the page in a kernel core dump * VM_ALLOC_NOOBJ page is not associated with an object and * should not be exclusive busy * VM_ALLOC_SBUSY shared busy the allocated page * VM_ALLOC_WIRED wire the allocated page * VM_ALLOC_ZERO prefer a zeroed page * * This routine may not sleep. */ vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr) { struct vnode *drop; struct spglist deferred_vdrop_list; vm_page_t m, m_tmp, m_ret; u_int flags; int req_class; KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), ("vm_page_alloc: inconsistent object(%p)/req(%x)", (void *)object, req)); if (object != NULL) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_PHYS, ("vm_page_alloc_contig: object %p isn't OBJT_PHYS", object)); } KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); req_class = req & VM_ALLOC_CLASS_MASK; /* * The page daemon is allowed to dig deeper into the free page list. */ if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) req_class = VM_ALLOC_SYSTEM; SLIST_INIT(&deferred_vdrop_list); mtx_lock(&vm_page_queue_free_mtx); if (vm_cnt.v_free_count + vm_cnt.v_cache_count >= npages + vm_cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM && vm_cnt.v_free_count + vm_cnt.v_cache_count >= npages + vm_cnt.v_interrupt_free_min) || (req_class == VM_ALLOC_INTERRUPT && vm_cnt.v_free_count + vm_cnt.v_cache_count >= npages)) { #if VM_NRESERVLEVEL > 0 retry: if (object == NULL || (object->flags & OBJ_COLORED) == 0 || (m_ret = vm_reserv_alloc_contig(object, pindex, npages, low, high, alignment, boundary)) == NULL) #endif m_ret = vm_phys_alloc_contig(npages, low, high, alignment, boundary); } else { mtx_unlock(&vm_page_queue_free_mtx); atomic_add_int(&vm_pageout_deficit, npages); pagedaemon_wakeup(); return (NULL); } if (m_ret != NULL) for (m = m_ret; m < &m_ret[npages]; m++) { drop = vm_page_alloc_init(m); if (drop != NULL) { /* * Enqueue the vnode for deferred vdrop(). */ m->plinks.s.pv = drop; SLIST_INSERT_HEAD(&deferred_vdrop_list, m, plinks.s.ss); } } else { #if VM_NRESERVLEVEL > 0 if (vm_reserv_reclaim_contig(npages, low, high, alignment, boundary)) goto retry; #endif } mtx_unlock(&vm_page_queue_free_mtx); if (m_ret == NULL) return (NULL); /* * Initialize the pages. Only the PG_ZERO flag is inherited. */ flags = 0; if ((req & VM_ALLOC_ZERO) != 0) flags = PG_ZERO; if ((req & VM_ALLOC_NODUMP) != 0) flags |= PG_NODUMP; if ((req & VM_ALLOC_WIRED) != 0) atomic_add_int(&vm_cnt.v_wire_count, npages); if (object != NULL) { if (object->memattr != VM_MEMATTR_DEFAULT && memattr == VM_MEMATTR_DEFAULT) memattr = object->memattr; } for (m = m_ret; m < &m_ret[npages]; m++) { m->aflags = 0; m->flags = (m->flags | PG_NODUMP) & flags; m->busy_lock = VPB_UNBUSIED; if (object != NULL) { if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) m->busy_lock = VPB_SINGLE_EXCLUSIVER; if ((req & VM_ALLOC_SBUSY) != 0) m->busy_lock = VPB_SHARERS_WORD(1); } if ((req & VM_ALLOC_WIRED) != 0) m->wire_count = 1; /* Unmanaged pages don't use "act_count". */ m->oflags = VPO_UNMANAGED; if (object != NULL) { if (vm_page_insert(m, object, pindex)) { vm_page_alloc_contig_vdrop( &deferred_vdrop_list); if (vm_paging_needed()) pagedaemon_wakeup(); if ((req & VM_ALLOC_WIRED) != 0) atomic_subtract_int(&vm_cnt.v_wire_count, npages); for (m_tmp = m, m = m_ret; m < &m_ret[npages]; m++) { if ((req & VM_ALLOC_WIRED) != 0) m->wire_count = 0; if (m >= m_tmp) { m->object = NULL; m->oflags |= VPO_UNMANAGED; } m->busy_lock = VPB_UNBUSIED; vm_page_free(m); } return (NULL); } } else m->pindex = pindex; if (memattr != VM_MEMATTR_DEFAULT) pmap_page_set_memattr(m, memattr); pindex++; } vm_page_alloc_contig_vdrop(&deferred_vdrop_list); if (vm_paging_needed()) pagedaemon_wakeup(); return (m_ret); } /* * Initialize a page that has been freshly dequeued from a freelist. * The caller has to drop the vnode returned, if it is not NULL. * * This function may only be used to initialize unmanaged pages. * * To be called with vm_page_queue_free_mtx held. */ static struct vnode * vm_page_alloc_init(vm_page_t m) { struct vnode *drop; vm_object_t m_object; KASSERT(m->queue == PQ_NONE, ("vm_page_alloc_init: page %p has unexpected queue %d", m, m->queue)); KASSERT(m->wire_count == 0, ("vm_page_alloc_init: page %p is wired", m)); KASSERT(m->hold_count == 0, ("vm_page_alloc_init: page %p is held", m)); KASSERT(!vm_page_busied(m), ("vm_page_alloc_init: page %p is busy", m)); KASSERT(m->dirty == 0, ("vm_page_alloc_init: page %p is dirty", m)); KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, ("vm_page_alloc_init: page %p has unexpected memattr %d", m, pmap_page_get_memattr(m))); mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); drop = NULL; if ((m->flags & PG_CACHED) != 0) { KASSERT((m->flags & PG_ZERO) == 0, ("vm_page_alloc_init: cached page %p is PG_ZERO", m)); m->valid = 0; m_object = m->object; vm_page_cache_remove(m); if (m_object->type == OBJT_VNODE && vm_object_cache_is_empty(m_object)) drop = m_object->handle; } else { KASSERT(m->valid == 0, ("vm_page_alloc_init: free page %p is valid", m)); vm_phys_freecnt_adj(m, -1); if ((m->flags & PG_ZERO) != 0) vm_page_zero_count--; } return (drop); } /* * vm_page_alloc_freelist: * * Allocate a physical page from the specified free page list. * * The caller must always specify an allocation class. * * allocation classes: * VM_ALLOC_NORMAL normal process request * VM_ALLOC_SYSTEM system *really* needs a page * VM_ALLOC_INTERRUPT interrupt time request * * optional allocation flags: * VM_ALLOC_COUNT(number) the number of additional pages that the caller * intends to allocate * VM_ALLOC_WIRED wire the allocated page * VM_ALLOC_ZERO prefer a zeroed page * * This routine may not sleep. */ vm_page_t vm_page_alloc_freelist(int flind, int req) { struct vnode *drop; vm_page_t m; u_int flags; int req_class; req_class = req & VM_ALLOC_CLASS_MASK; /* * The page daemon is allowed to dig deeper into the free page list. */ if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) req_class = VM_ALLOC_SYSTEM; /* * Do not allocate reserved pages unless the req has asked for it. */ mtx_lock_flags(&vm_page_queue_free_mtx, MTX_RECURSE); if (vm_cnt.v_free_count + vm_cnt.v_cache_count > vm_cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM && vm_cnt.v_free_count + vm_cnt.v_cache_count > vm_cnt.v_interrupt_free_min) || (req_class == VM_ALLOC_INTERRUPT && vm_cnt.v_free_count + vm_cnt.v_cache_count > 0)) m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, 0); else { mtx_unlock(&vm_page_queue_free_mtx); atomic_add_int(&vm_pageout_deficit, max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); pagedaemon_wakeup(); return (NULL); } if (m == NULL) { mtx_unlock(&vm_page_queue_free_mtx); return (NULL); } drop = vm_page_alloc_init(m); mtx_unlock(&vm_page_queue_free_mtx); /* * Initialize the page. Only the PG_ZERO flag is inherited. */ m->aflags = 0; flags = 0; if ((req & VM_ALLOC_ZERO) != 0) flags = PG_ZERO; m->flags &= flags; if ((req & VM_ALLOC_WIRED) != 0) { /* * The page lock is not required for wiring a page that does * not belong to an object. */ atomic_add_int(&vm_cnt.v_wire_count, 1); m->wire_count = 1; } /* Unmanaged pages don't use "act_count". */ m->oflags = VPO_UNMANAGED; if (drop != NULL) vdrop(drop); if (vm_paging_needed()) pagedaemon_wakeup(); return (m); } #define VPSC_ANY 0 /* No restrictions. */ #define VPSC_NORESERV 1 /* Skip reservations; implies VPSC_NOSUPER. */ #define VPSC_NOSUPER 2 /* Skip superpages. */ /* * vm_page_scan_contig: * * Scan vm_page_array[] between the specified entries "m_start" and * "m_end" for a run of contiguous physical pages that satisfy the * specified conditions, and return the lowest page in the run. The * specified "alignment" determines the alignment of the lowest physical * page in the run. If the specified "boundary" is non-zero, then the * run of physical pages cannot span a physical address that is a * multiple of "boundary". * * "m_end" is never dereferenced, so it need not point to a vm_page * structure within vm_page_array[]. * * "npages" must be greater than zero. "m_start" and "m_end" must not * span a hole (or discontiguity) in the physical address space. Both * "alignment" and "boundary" must be a power of two. */ vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start, vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options) { struct mtx *m_mtx, *new_mtx; vm_object_t object; vm_paddr_t pa; vm_page_t m, m_run; #if VM_NRESERVLEVEL > 0 int level; #endif int m_inc, order, run_ext, run_len; KASSERT(npages > 0, ("npages is 0")); KASSERT(powerof2(alignment), ("alignment is not a power of 2")); KASSERT(powerof2(boundary), ("boundary is not a power of 2")); m_run = NULL; run_len = 0; m_mtx = NULL; for (m = m_start; m < m_end && run_len < npages; m += m_inc) { KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0, ("page %p is PG_FICTITIOUS or PG_MARKER", m)); /* * If the current page would be the start of a run, check its * physical address against the end, alignment, and boundary * conditions. If it doesn't satisfy these conditions, either * terminate the scan or advance to the next page that * satisfies the failed condition. */ if (run_len == 0) { KASSERT(m_run == NULL, ("m_run != NULL")); if (m + npages > m_end) break; pa = VM_PAGE_TO_PHYS(m); if ((pa & (alignment - 1)) != 0) { m_inc = atop(roundup2(pa, alignment) - pa); continue; } if (rounddown2(pa ^ (pa + ptoa(npages) - 1), boundary) != 0) { m_inc = atop(roundup2(pa, boundary) - pa); continue; } } else KASSERT(m_run != NULL, ("m_run == NULL")); /* * Avoid releasing and reacquiring the same page lock. */ new_mtx = vm_page_lockptr(m); if (m_mtx != new_mtx) { if (m_mtx != NULL) mtx_unlock(m_mtx); m_mtx = new_mtx; mtx_lock(m_mtx); } m_inc = 1; retry: if (m->wire_count != 0 || m->hold_count != 0) run_ext = 0; #if VM_NRESERVLEVEL > 0 else if ((level = vm_reserv_level(m)) >= 0 && (options & VPSC_NORESERV) != 0) { run_ext = 0; /* Advance to the end of the reservation. */ pa = VM_PAGE_TO_PHYS(m); m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) - pa); } #endif else if ((object = m->object) != NULL) { /* * The page is considered eligible for relocation if * and only if it could be laundered or reclaimed by * the page daemon. */ if (!VM_OBJECT_TRYRLOCK(object)) { mtx_unlock(m_mtx); VM_OBJECT_RLOCK(object); mtx_lock(m_mtx); if (m->object != object) { /* * The page may have been freed. */ VM_OBJECT_RUNLOCK(object); goto retry; } else if (m->wire_count != 0 || m->hold_count != 0) { run_ext = 0; goto unlock; } } KASSERT((m->flags & PG_UNHOLDFREE) == 0, ("page %p is PG_UNHOLDFREE", m)); /* Don't care: PG_NODUMP, PG_WINATCFLS, PG_ZERO. */ if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP && object->type != OBJT_VNODE) run_ext = 0; else if ((m->flags & PG_CACHED) != 0 || m != vm_page_lookup(object, m->pindex)) { /* * The page is cached or recently converted * from cached to free. */ #if VM_NRESERVLEVEL > 0 if (level >= 0) { /* * The page is reserved. Extend the * current run by one page. */ run_ext = 1; } else #endif if ((order = m->order) < VM_NFREEORDER) { /* * The page is enqueued in the * physical memory allocator's cache/ * free page queues. Moreover, it is * the first page in a power-of-two- * sized run of contiguous cache/free * pages. Add these pages to the end * of the current run, and jump * ahead. */ run_ext = 1 << order; m_inc = 1 << order; } else run_ext = 0; #if VM_NRESERVLEVEL > 0 } else if ((options & VPSC_NOSUPER) != 0 && (level = vm_reserv_level_iffullpop(m)) >= 0) { run_ext = 0; /* Advance to the end of the superpage. */ pa = VM_PAGE_TO_PHYS(m); m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) - pa); #endif } else if (object->memattr == VM_MEMATTR_DEFAULT && m->queue != PQ_NONE && !vm_page_busied(m)) { /* * The page is allocated but eligible for * relocation. Extend the current run by one * page. */ KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, ("page %p has an unexpected memattr", m)); KASSERT((m->oflags & (VPO_SWAPINPROG | VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0, ("page %p has unexpected oflags", m)); /* Don't care: VPO_NOSYNC. */ run_ext = 1; } else run_ext = 0; unlock: VM_OBJECT_RUNLOCK(object); #if VM_NRESERVLEVEL > 0 } else if (level >= 0) { /* * The page is reserved but not yet allocated. In * other words, it is still cached or free. Extend * the current run by one page. */ run_ext = 1; #endif } else if ((order = m->order) < VM_NFREEORDER) { /* * The page is enqueued in the physical memory * allocator's cache/free page queues. Moreover, it * is the first page in a power-of-two-sized run of * contiguous cache/free pages. Add these pages to * the end of the current run, and jump ahead. */ run_ext = 1 << order; m_inc = 1 << order; } else { /* * Skip the page for one of the following reasons: (1) * It is enqueued in the physical memory allocator's * cache/free page queues. However, it is not the * first page in a run of contiguous cache/free pages. * (This case rarely occurs because the scan is * performed in ascending order.) (2) It is not * reserved, and it is transitioning from free to * allocated. (Conversely, the transition from * allocated to free for managed pages is blocked by * the page lock.) (3) It is allocated but not * contained by an object and not wired, e.g., * allocated by Xen's balloon driver. */ run_ext = 0; } /* * Extend or reset the current run of pages. */ if (run_ext > 0) { if (run_len == 0) m_run = m; run_len += run_ext; } else { if (run_len > 0) { m_run = NULL; run_len = 0; } } } if (m_mtx != NULL) mtx_unlock(m_mtx); if (run_len >= npages) return (m_run); return (NULL); } /* * vm_page_reclaim_run: * * Try to relocate each of the allocated virtual pages within the * specified run of physical pages to a new physical address. Free the * physical pages underlying the relocated virtual pages. A virtual page * is relocatable if and only if it could be laundered or reclaimed by * the page daemon. Whenever possible, a virtual page is relocated to a * physical address above "high". * * Returns 0 if every physical page within the run was already free or * just freed by a successful relocation. Otherwise, returns a non-zero * value indicating why the last attempt to relocate a virtual page was * unsuccessful. * * "req_class" must be an allocation class. */ static int vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run, vm_paddr_t high) { struct mtx *m_mtx, *new_mtx; struct spglist free; vm_object_t object; vm_paddr_t pa; vm_page_t m, m_end, m_new; int error, order, req; KASSERT((req_class & VM_ALLOC_CLASS_MASK) == req_class, ("req_class is not an allocation class")); SLIST_INIT(&free); error = 0; m = m_run; m_end = m_run + npages; m_mtx = NULL; for (; error == 0 && m < m_end; m++) { KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0, ("page %p is PG_FICTITIOUS or PG_MARKER", m)); /* * Avoid releasing and reacquiring the same page lock. */ new_mtx = vm_page_lockptr(m); if (m_mtx != new_mtx) { if (m_mtx != NULL) mtx_unlock(m_mtx); m_mtx = new_mtx; mtx_lock(m_mtx); } retry: if (m->wire_count != 0 || m->hold_count != 0) error = EBUSY; else if ((object = m->object) != NULL) { /* * The page is relocated if and only if it could be * laundered or reclaimed by the page daemon. */ if (!VM_OBJECT_TRYWLOCK(object)) { mtx_unlock(m_mtx); VM_OBJECT_WLOCK(object); mtx_lock(m_mtx); if (m->object != object) { /* * The page may have been freed. */ VM_OBJECT_WUNLOCK(object); goto retry; } else if (m->wire_count != 0 || m->hold_count != 0) { error = EBUSY; goto unlock; } } KASSERT((m->flags & PG_UNHOLDFREE) == 0, ("page %p is PG_UNHOLDFREE", m)); /* Don't care: PG_NODUMP, PG_WINATCFLS, PG_ZERO. */ if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP && object->type != OBJT_VNODE) error = EINVAL; else if ((m->flags & PG_CACHED) != 0 || m != vm_page_lookup(object, m->pindex)) { /* * The page is cached or recently converted * from cached to free. */ VM_OBJECT_WUNLOCK(object); goto cached; } else if (object->memattr != VM_MEMATTR_DEFAULT) error = EINVAL; else if (m->queue != PQ_NONE && !vm_page_busied(m)) { KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, ("page %p has an unexpected memattr", m)); KASSERT((m->oflags & (VPO_SWAPINPROG | VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0, ("page %p has unexpected oflags", m)); /* Don't care: VPO_NOSYNC. */ if (m->valid != 0) { /* * First, try to allocate a new page * that is above "high". Failing * that, try to allocate a new page * that is below "m_run". Allocate * the new page between the end of * "m_run" and "high" only as a last * resort. */ req = req_class | VM_ALLOC_NOOBJ; if ((m->flags & PG_NODUMP) != 0) req |= VM_ALLOC_NODUMP; if (trunc_page(high) != ~(vm_paddr_t)PAGE_MASK) { m_new = vm_page_alloc_contig( NULL, 0, req, 1, round_page(high), ~(vm_paddr_t)0, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); } else m_new = NULL; if (m_new == NULL) { pa = VM_PAGE_TO_PHYS(m_run); m_new = vm_page_alloc_contig( NULL, 0, req, 1, 0, pa - 1, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); } if (m_new == NULL) { pa += ptoa(npages); m_new = vm_page_alloc_contig( NULL, 0, req, 1, pa, high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); } if (m_new == NULL) { error = ENOMEM; goto unlock; } KASSERT(m_new->wire_count == 0, ("page %p is wired", m)); /* * Replace "m" with the new page. For * vm_page_replace(), "m" must be busy * and dequeued. Finally, change "m" * as if vm_page_free() was called. */ if (object->ref_count != 0) pmap_remove_all(m); m_new->aflags = m->aflags; KASSERT(m_new->oflags == VPO_UNMANAGED, ("page %p is managed", m)); m_new->oflags = m->oflags & VPO_NOSYNC; pmap_copy_page(m, m_new); m_new->valid = m->valid; m_new->dirty = m->dirty; m->flags &= ~PG_ZERO; vm_page_xbusy(m); vm_page_remque(m); vm_page_replace_checked(m_new, object, m->pindex, m); m->valid = 0; vm_page_undirty(m); /* * The new page must be deactivated * before the object is unlocked. */ new_mtx = vm_page_lockptr(m_new); if (m_mtx != new_mtx) { mtx_unlock(m_mtx); m_mtx = new_mtx; mtx_lock(m_mtx); } vm_page_deactivate(m_new); } else { m->flags &= ~PG_ZERO; vm_page_remque(m); vm_page_remove(m); KASSERT(m->dirty == 0, ("page %p is dirty", m)); } SLIST_INSERT_HEAD(&free, m, plinks.s.ss); } else error = EBUSY; unlock: VM_OBJECT_WUNLOCK(object); } else { cached: mtx_lock(&vm_page_queue_free_mtx); order = m->order; if (order < VM_NFREEORDER) { /* * The page is enqueued in the physical memory * allocator's cache/free page queues. * Moreover, it is the first page in a power- * of-two-sized run of contiguous cache/free * pages. Jump ahead to the last page within * that run, and continue from there. */ m += (1 << order) - 1; } #if VM_NRESERVLEVEL > 0 else if (vm_reserv_is_page_free(m)) order = 0; #endif mtx_unlock(&vm_page_queue_free_mtx); if (order == VM_NFREEORDER) error = EINVAL; } } if (m_mtx != NULL) mtx_unlock(m_mtx); if ((m = SLIST_FIRST(&free)) != NULL) { mtx_lock(&vm_page_queue_free_mtx); do { SLIST_REMOVE_HEAD(&free, plinks.s.ss); vm_phys_freecnt_adj(m, 1); #if VM_NRESERVLEVEL > 0 if (!vm_reserv_free_page(m)) #else if (true) #endif vm_phys_free_pages(m, 0); } while ((m = SLIST_FIRST(&free)) != NULL); vm_page_zero_idle_wakeup(); vm_page_free_wakeup(); mtx_unlock(&vm_page_queue_free_mtx); } return (error); } #define NRUNS 16 CTASSERT(powerof2(NRUNS)); #define RUN_INDEX(count) ((count) & (NRUNS - 1)) #define MIN_RECLAIM 8 /* * vm_page_reclaim_contig: * * Reclaim allocated, contiguous physical memory satisfying the specified * conditions by relocating the virtual pages using that physical memory. * Returns true if reclamation is successful and false otherwise. Since * relocation requires the allocation of physical pages, reclamation may * fail due to a shortage of cache/free pages. When reclamation fails, * callers are expected to perform VM_WAIT before retrying a failed * allocation operation, e.g., vm_page_alloc_contig(). * * The caller must always specify an allocation class through "req". * * allocation classes: * VM_ALLOC_NORMAL normal process request * VM_ALLOC_SYSTEM system *really* needs a page * VM_ALLOC_INTERRUPT interrupt time request * * The optional allocation flags are ignored. * * "npages" must be greater than zero. Both "alignment" and "boundary" * must be a power of two. */ bool vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { vm_paddr_t curr_low; vm_page_t m_run, m_runs[NRUNS]; u_long count, reclaimed; int error, i, options, req_class; KASSERT(npages > 0, ("npages is 0")); KASSERT(powerof2(alignment), ("alignment is not a power of 2")); KASSERT(powerof2(boundary), ("boundary is not a power of 2")); req_class = req & VM_ALLOC_CLASS_MASK; /* * The page daemon is allowed to dig deeper into the free page list. */ if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) req_class = VM_ALLOC_SYSTEM; /* * Return if the number of cached and free pages cannot satisfy the * requested allocation. */ count = vm_cnt.v_free_count + vm_cnt.v_cache_count; if (count < npages + vm_cnt.v_free_reserved || (count < npages + vm_cnt.v_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) || (count < npages && req_class == VM_ALLOC_INTERRUPT)) return (false); /* * Scan up to three times, relaxing the restrictions ("options") on * the reclamation of reservations and superpages each time. */ for (options = VPSC_NORESERV;;) { /* * Find the highest runs that satisfy the given constraints * and restrictions, and record them in "m_runs". */ curr_low = low; count = 0; for (;;) { m_run = vm_phys_scan_contig(npages, curr_low, high, alignment, boundary, options); if (m_run == NULL) break; curr_low = VM_PAGE_TO_PHYS(m_run) + ptoa(npages); m_runs[RUN_INDEX(count)] = m_run; count++; } /* * Reclaim the highest runs in LIFO (descending) order until * the number of reclaimed pages, "reclaimed", is at least * MIN_RECLAIM. Reset "reclaimed" each time because each * reclamation is idempotent, and runs will (likely) recur * from one scan to the next as restrictions are relaxed. */ reclaimed = 0; for (i = 0; count > 0 && i < NRUNS; i++) { count--; m_run = m_runs[RUN_INDEX(count)]; error = vm_page_reclaim_run(req_class, npages, m_run, high); if (error == 0) { reclaimed += npages; if (reclaimed >= MIN_RECLAIM) return (true); } } /* * Either relax the restrictions on the next scan or return if * the last scan had no restrictions. */ if (options == VPSC_NORESERV) options = VPSC_NOSUPER; else if (options == VPSC_NOSUPER) options = VPSC_ANY; else if (options == VPSC_ANY) return (reclaimed != 0); } } /* * vm_wait: (also see VM_WAIT macro) * * Sleep until free pages are available for allocation. * - Called in various places before memory allocations. */ void vm_wait(void) { mtx_lock(&vm_page_queue_free_mtx); if (curproc == pageproc) { vm_pageout_pages_needed = 1; msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx, PDROP | PSWP, "VMWait", 0); } else { if (!vm_pageout_wanted) { vm_pageout_wanted = true; wakeup(&vm_pageout_wanted); } vm_pages_needed = true; msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM, "vmwait", 0); } } /* * vm_waitpfault: (also see VM_WAITPFAULT macro) * * Sleep until free pages are available for allocation. * - Called only in vm_fault so that processes page faulting * can be easily tracked. * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing * processes will be able to grab memory first. Do not change * this balance without careful testing first. */ void vm_waitpfault(void) { mtx_lock(&vm_page_queue_free_mtx); if (!vm_pageout_wanted) { vm_pageout_wanted = true; wakeup(&vm_pageout_wanted); } vm_pages_needed = true; msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER, "pfault", 0); } struct vm_pagequeue * vm_page_pagequeue(vm_page_t m) { return (&vm_phys_domain(m)->vmd_pagequeues[m->queue]); } /* * vm_page_dequeue: * * Remove the given page from its current page queue. * * The page must be locked. */ void vm_page_dequeue(vm_page_t m) { struct vm_pagequeue *pq; vm_page_assert_locked(m); KASSERT(m->queue < PQ_COUNT, ("vm_page_dequeue: page %p is not queued", m)); pq = vm_page_pagequeue(m); vm_pagequeue_lock(pq); m->queue = PQ_NONE; TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); vm_pagequeue_cnt_dec(pq); vm_pagequeue_unlock(pq); } /* * vm_page_dequeue_locked: * * Remove the given page from its current page queue. * * The page and page queue must be locked. */ void vm_page_dequeue_locked(vm_page_t m) { struct vm_pagequeue *pq; vm_page_lock_assert(m, MA_OWNED); pq = vm_page_pagequeue(m); vm_pagequeue_assert_locked(pq); m->queue = PQ_NONE; TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); vm_pagequeue_cnt_dec(pq); } /* * vm_page_enqueue: * * Add the given page to the specified page queue. * * The page must be locked. */ static void vm_page_enqueue(uint8_t queue, vm_page_t m) { struct vm_pagequeue *pq; vm_page_lock_assert(m, MA_OWNED); KASSERT(queue < PQ_COUNT, ("vm_page_enqueue: invalid queue %u request for page %p", queue, m)); pq = &vm_phys_domain(m)->vmd_pagequeues[queue]; vm_pagequeue_lock(pq); m->queue = queue; TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); vm_pagequeue_cnt_inc(pq); vm_pagequeue_unlock(pq); } /* * vm_page_requeue: * * Move the given page to the tail of its current page queue. * * The page must be locked. */ void vm_page_requeue(vm_page_t m) { struct vm_pagequeue *pq; vm_page_lock_assert(m, MA_OWNED); KASSERT(m->queue != PQ_NONE, ("vm_page_requeue: page %p is not queued", m)); pq = vm_page_pagequeue(m); vm_pagequeue_lock(pq); TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); vm_pagequeue_unlock(pq); } /* * vm_page_requeue_locked: * * Move the given page to the tail of its current page queue. * * The page queue must be locked. */ void vm_page_requeue_locked(vm_page_t m) { struct vm_pagequeue *pq; KASSERT(m->queue != PQ_NONE, ("vm_page_requeue_locked: page %p is not queued", m)); pq = vm_page_pagequeue(m); vm_pagequeue_assert_locked(pq); TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); } /* * vm_page_activate: * * Put the specified page on the active list (if appropriate). * Ensure that act_count is at least ACT_INIT but do not otherwise * mess with it. * * The page must be locked. */ void vm_page_activate(vm_page_t m) { int queue; vm_page_lock_assert(m, MA_OWNED); if ((queue = m->queue) != PQ_ACTIVE) { if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) { if (m->act_count < ACT_INIT) m->act_count = ACT_INIT; if (queue != PQ_NONE) vm_page_dequeue(m); vm_page_enqueue(PQ_ACTIVE, m); } else KASSERT(queue == PQ_NONE, ("vm_page_activate: wired page %p is queued", m)); } else { if (m->act_count < ACT_INIT) m->act_count = ACT_INIT; } } /* * vm_page_free_wakeup: * * Helper routine for vm_page_free_toq() and vm_page_cache(). This * routine is called when a page has been added to the cache or free * queues. * * The page queues must be locked. */ static inline void vm_page_free_wakeup(void) { mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); /* * if pageout daemon needs pages, then tell it that there are * some free. */ if (vm_pageout_pages_needed && vm_cnt.v_cache_count + vm_cnt.v_free_count >= vm_cnt.v_pageout_free_min) { wakeup(&vm_pageout_pages_needed); vm_pageout_pages_needed = 0; } /* * wakeup processes that are waiting on memory if we hit a * high water mark. And wakeup scheduler process if we have * lots of memory. this process will swapin processes. */ if (vm_pages_needed && !vm_page_count_min()) { vm_pages_needed = false; wakeup(&vm_cnt.v_free_count); } } /* * Turn a cached page into a free page, by changing its attributes. * Keep the statistics up-to-date. * * The free page queue must be locked. */ static void vm_page_cache_turn_free(vm_page_t m) { mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); m->object = NULL; m->valid = 0; KASSERT((m->flags & PG_CACHED) != 0, ("vm_page_cache_turn_free: page %p is not cached", m)); m->flags &= ~PG_CACHED; vm_cnt.v_cache_count--; vm_phys_freecnt_adj(m, 1); } /* * vm_page_free_toq: * * Returns the given page to the free list, * disassociating it with any VM object. * * The object must be locked. The page must be locked if it is managed. */ void vm_page_free_toq(vm_page_t m) { if ((m->oflags & VPO_UNMANAGED) == 0) { vm_page_lock_assert(m, MA_OWNED); KASSERT(!pmap_page_is_mapped(m), ("vm_page_free_toq: freeing mapped page %p", m)); } else KASSERT(m->queue == PQ_NONE, ("vm_page_free_toq: unmanaged page %p is queued", m)); PCPU_INC(cnt.v_tfree); if (vm_page_sbusied(m)) panic("vm_page_free: freeing busy page %p", m); /* * Unqueue, then remove page. Note that we cannot destroy * the page here because we do not want to call the pager's * callback routine until after we've put the page on the * appropriate free queue. */ vm_page_remque(m); vm_page_remove(m); /* * If fictitious remove object association and * return, otherwise delay object association removal. */ if ((m->flags & PG_FICTITIOUS) != 0) { return; } m->valid = 0; vm_page_undirty(m); if (m->wire_count != 0) panic("vm_page_free: freeing wired page %p", m); if (m->hold_count != 0) { m->flags &= ~PG_ZERO; KASSERT((m->flags & PG_UNHOLDFREE) == 0, ("vm_page_free: freeing PG_UNHOLDFREE page %p", m)); m->flags |= PG_UNHOLDFREE; } else { /* * Restore the default memory attribute to the page. */ if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT) pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT); /* * Insert the page into the physical memory allocator's * cache/free page queues. */ mtx_lock(&vm_page_queue_free_mtx); vm_phys_freecnt_adj(m, 1); #if VM_NRESERVLEVEL > 0 if (!vm_reserv_free_page(m)) #else if (TRUE) #endif vm_phys_free_pages(m, 0); if ((m->flags & PG_ZERO) != 0) ++vm_page_zero_count; else vm_page_zero_idle_wakeup(); vm_page_free_wakeup(); mtx_unlock(&vm_page_queue_free_mtx); } } /* * vm_page_wire: * * Mark this page as wired down by yet * another map, removing it from paging queues * as necessary. * * If the page is fictitious, then its wire count must remain one. * * The page must be locked. */ void vm_page_wire(vm_page_t m) { /* * Only bump the wire statistics if the page is not already wired, * and only unqueue the page if it is on some queue (if it is unmanaged * it is already off the queues). */ vm_page_lock_assert(m, MA_OWNED); if ((m->flags & PG_FICTITIOUS) != 0) { KASSERT(m->wire_count == 1, ("vm_page_wire: fictitious page %p's wire count isn't one", m)); return; } if (m->wire_count == 0) { KASSERT((m->oflags & VPO_UNMANAGED) == 0 || m->queue == PQ_NONE, ("vm_page_wire: unmanaged page %p is queued", m)); vm_page_remque(m); atomic_add_int(&vm_cnt.v_wire_count, 1); } m->wire_count++; KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m)); } /* * vm_page_unwire: * * Release one wiring of the specified page, potentially allowing it to be * paged out. Returns TRUE if the number of wirings transitions to zero and * FALSE otherwise. * * Only managed pages belonging to an object can be paged out. If the number * of wirings transitions to zero and the page is eligible for page out, then * the page is added to the specified paging queue (unless PQ_NONE is * specified). * * If a page is fictitious, then its wire count must always be one. * * A managed page must be locked. */ boolean_t vm_page_unwire(vm_page_t m, uint8_t queue) { KASSERT(queue < PQ_COUNT || queue == PQ_NONE, ("vm_page_unwire: invalid queue %u request for page %p", queue, m)); if ((m->oflags & VPO_UNMANAGED) == 0) vm_page_assert_locked(m); if ((m->flags & PG_FICTITIOUS) != 0) { KASSERT(m->wire_count == 1, ("vm_page_unwire: fictitious page %p's wire count isn't one", m)); return (FALSE); } if (m->wire_count > 0) { m->wire_count--; if (m->wire_count == 0) { atomic_subtract_int(&vm_cnt.v_wire_count, 1); if ((m->oflags & VPO_UNMANAGED) == 0 && m->object != NULL && queue != PQ_NONE) { if (queue == PQ_INACTIVE) m->flags &= ~PG_WINATCFLS; vm_page_enqueue(queue, m); } return (TRUE); } else return (FALSE); } else panic("vm_page_unwire: page %p's wire count is zero", m); } /* * Move the specified page to the inactive queue. * * Many pages placed on the inactive queue should actually go * into the cache, but it is difficult to figure out which. What * we do instead, if the inactive target is well met, is to put * clean pages at the head of the inactive queue instead of the tail. * This will cause them to be moved to the cache more quickly and * if not actively re-referenced, reclaimed more quickly. If we just * stick these pages at the end of the inactive queue, heavy filesystem * meta-data accesses can cause an unnecessary paging load on memory bound * processes. This optimization causes one-time-use metadata to be * reused more quickly. * * Normally noreuse is FALSE, resulting in LRU operation. noreuse is set * to TRUE if we want this page to be 'as if it were placed in the cache', * except without unmapping it from the process address space. In * practice this is implemented by inserting the page at the head of the * queue, using a marker page to guide FIFO insertion ordering. * * The page must be locked. */ static inline void _vm_page_deactivate(vm_page_t m, boolean_t noreuse) { struct vm_pagequeue *pq; int queue; vm_page_assert_locked(m); /* * Ignore if the page is already inactive, unless it is unlikely to be * reactivated. */ if ((queue = m->queue) == PQ_INACTIVE && !noreuse) return; if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) { pq = &vm_phys_domain(m)->vmd_pagequeues[PQ_INACTIVE]; /* Avoid multiple acquisitions of the inactive queue lock. */ if (queue == PQ_INACTIVE) { vm_pagequeue_lock(pq); vm_page_dequeue_locked(m); } else { if (queue != PQ_NONE) vm_page_dequeue(m); m->flags &= ~PG_WINATCFLS; vm_pagequeue_lock(pq); } m->queue = PQ_INACTIVE; if (noreuse) TAILQ_INSERT_BEFORE(&vm_phys_domain(m)->vmd_inacthead, m, plinks.q); else TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); vm_pagequeue_cnt_inc(pq); vm_pagequeue_unlock(pq); } } /* * Move the specified page to the inactive queue. * * The page must be locked. */ void vm_page_deactivate(vm_page_t m) { _vm_page_deactivate(m, FALSE); } /* * Move the specified page to the inactive queue with the expectation * that it is unlikely to be reused. * * The page must be locked. */ void vm_page_deactivate_noreuse(vm_page_t m) { _vm_page_deactivate(m, TRUE); } /* * vm_page_try_to_cache: * * Returns 0 on failure, 1 on success */ int vm_page_try_to_cache(vm_page_t m) { vm_page_lock_assert(m, MA_OWNED); VM_OBJECT_ASSERT_WLOCKED(m->object); if (m->dirty || m->hold_count || m->wire_count || (m->oflags & VPO_UNMANAGED) != 0 || vm_page_busied(m)) return (0); pmap_remove_all(m); if (m->dirty) return (0); vm_page_cache(m); return (1); } /* * vm_page_try_to_free() * * Attempt to free the page. If we cannot free it, we do nothing. * 1 is returned on success, 0 on failure. */ int vm_page_try_to_free(vm_page_t m) { vm_page_lock_assert(m, MA_OWNED); if (m->object != NULL) VM_OBJECT_ASSERT_WLOCKED(m->object); if (m->dirty || m->hold_count || m->wire_count || (m->oflags & VPO_UNMANAGED) != 0 || vm_page_busied(m)) return (0); pmap_remove_all(m); if (m->dirty) return (0); vm_page_free(m); return (1); } /* * vm_page_cache * * Put the specified page onto the page cache queue (if appropriate). * * The object and page must be locked. */ void vm_page_cache(vm_page_t m) { vm_object_t object; boolean_t cache_was_empty; vm_page_lock_assert(m, MA_OWNED); object = m->object; VM_OBJECT_ASSERT_WLOCKED(object); if (vm_page_busied(m) || (m->oflags & VPO_UNMANAGED) || m->hold_count || m->wire_count) panic("vm_page_cache: attempting to cache busy page"); KASSERT(!pmap_page_is_mapped(m), ("vm_page_cache: page %p is mapped", m)); KASSERT(m->dirty == 0, ("vm_page_cache: page %p is dirty", m)); if (m->valid == 0 || object->type == OBJT_DEFAULT || (object->type == OBJT_SWAP && !vm_pager_has_page(object, m->pindex, NULL, NULL))) { /* * Hypothesis: A cache-eligible page belonging to a * default object or swap object but without a backing * store must be zero filled. */ vm_page_free(m); return; } KASSERT((m->flags & PG_CACHED) == 0, ("vm_page_cache: page %p is already cached", m)); /* * Remove the page from the paging queues. */ vm_page_remque(m); /* * Remove the page from the object's collection of resident * pages. */ vm_radix_remove(&object->rtree, m->pindex); TAILQ_REMOVE(&object->memq, m, listq); object->resident_page_count--; /* * Restore the default memory attribute to the page. */ if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT) pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT); /* * Insert the page into the object's collection of cached pages * and the physical memory allocator's cache/free page queues. */ m->flags &= ~PG_ZERO; mtx_lock(&vm_page_queue_free_mtx); cache_was_empty = vm_radix_is_empty(&object->cache); if (vm_radix_insert(&object->cache, m)) { mtx_unlock(&vm_page_queue_free_mtx); if (object->type == OBJT_VNODE && object->resident_page_count == 0) vdrop(object->handle); m->object = NULL; vm_page_free(m); return; } /* * The above call to vm_radix_insert() could reclaim the one pre- * existing cached page from this object, resulting in a call to * vdrop(). */ if (!cache_was_empty) cache_was_empty = vm_radix_is_singleton(&object->cache); m->flags |= PG_CACHED; vm_cnt.v_cache_count++; PCPU_INC(cnt.v_tcached); #if VM_NRESERVLEVEL > 0 if (!vm_reserv_free_page(m)) { #else if (TRUE) { #endif vm_phys_free_pages(m, 0); } vm_page_free_wakeup(); mtx_unlock(&vm_page_queue_free_mtx); /* * Increment the vnode's hold count if this is the object's only * cached page. Decrement the vnode's hold count if this was * the object's only resident page. */ if (object->type == OBJT_VNODE) { if (cache_was_empty && object->resident_page_count != 0) vhold(object->handle); else if (!cache_was_empty && object->resident_page_count == 0) vdrop(object->handle); } } /* * vm_page_advise * * Deactivate or do nothing, as appropriate. * * The object and page must be locked. */ void vm_page_advise(vm_page_t m, int advice) { vm_page_assert_locked(m); VM_OBJECT_ASSERT_WLOCKED(m->object); if (advice == MADV_FREE) /* * Mark the page clean. This will allow the page to be freed * up by the system. However, such pages are often reused * quickly by malloc() so we do not do anything that would * cause a page fault if we can help it. * * Specifically, we do not try to actually free the page now * nor do we try to put it in the cache (which would cause a * page fault on reuse). * * But we do make the page as freeable as we can without * actually taking the step of unmapping it. */ vm_page_undirty(m); else if (advice != MADV_DONTNEED) return; /* * Clear any references to the page. Otherwise, the page daemon will * immediately reactivate the page. */ vm_page_aflag_clear(m, PGA_REFERENCED); if (advice != MADV_FREE && m->dirty == 0 && pmap_is_modified(m)) vm_page_dirty(m); /* * Place clean pages near the head of the inactive queue rather than * the tail, thus defeating the queue's LRU operation and ensuring that * the page will be reused quickly. Dirty pages are given a chance to * cycle once through the inactive queue before becoming eligible for * laundering. */ _vm_page_deactivate(m, m->dirty == 0); } /* * Grab a page, waiting until we are waken up due to the page * changing state. We keep on waiting, if the page continues * to be in the object. If the page doesn't exist, first allocate it * and then conditionally zero it. * * This routine may sleep. * * The object must be locked on entry. The lock will, however, be released * and reacquired if the routine sleeps. */ vm_page_t vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) { vm_page_t m; int sleep; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || (allocflags & VM_ALLOC_IGN_SBUSY) != 0, ("vm_page_grab: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); retrylookup: if ((m = vm_page_lookup(object, pindex)) != NULL) { sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ? vm_page_xbusied(m) : vm_page_busied(m); if (sleep) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) return (NULL); /* * Reference the page before unlocking and * sleeping so that the page daemon is less * likely to reclaim it. */ vm_page_aflag_set(m, PGA_REFERENCED); vm_page_lock(m); VM_OBJECT_WUNLOCK(object); vm_page_busy_sleep(m, "pgrbwt"); VM_OBJECT_WLOCK(object); goto retrylookup; } else { if ((allocflags & VM_ALLOC_WIRED) != 0) { vm_page_lock(m); vm_page_wire(m); vm_page_unlock(m); } if ((allocflags & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) vm_page_xbusy(m); if ((allocflags & VM_ALLOC_SBUSY) != 0) vm_page_sbusy(m); return (m); } } m = vm_page_alloc(object, pindex, allocflags); if (m == NULL) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) return (NULL); VM_OBJECT_WUNLOCK(object); VM_WAIT; VM_OBJECT_WLOCK(object); goto retrylookup; } else if (m->valid != 0) return (m); if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); return (m); } /* * Mapping function for valid or dirty bits in a page. * * Inputs are required to range within a page. */ vm_page_bits_t vm_page_bits(int base, int size) { int first_bit; int last_bit; KASSERT( base + size <= PAGE_SIZE, ("vm_page_bits: illegal base/size %d/%d", base, size) ); if (size == 0) /* handle degenerate case */ return (0); first_bit = base >> DEV_BSHIFT; last_bit = (base + size - 1) >> DEV_BSHIFT; return (((vm_page_bits_t)2 << last_bit) - ((vm_page_bits_t)1 << first_bit)); } /* * vm_page_set_valid_range: * * Sets portions of a page valid. The arguments are expected * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive * of any partial chunks touched by the range. The invalid portion of * such chunks will be zeroed. * * (base + size) must be less then or equal to PAGE_SIZE. */ void vm_page_set_valid_range(vm_page_t m, int base, int size) { int endoff, frag; VM_OBJECT_ASSERT_WLOCKED(m->object); if (size == 0) /* handle degenerate case */ return; /* * If the base is not DEV_BSIZE aligned and the valid * bit is clear, we have to zero out a portion of the * first block. */ if ((frag = rounddown2(base, DEV_BSIZE)) != base && (m->valid & (1 << (base >> DEV_BSHIFT))) == 0) pmap_zero_page_area(m, frag, base - frag); /* * If the ending offset is not DEV_BSIZE aligned and the * valid bit is clear, we have to zero out a portion of * the last block. */ endoff = base + size; if ((frag = rounddown2(endoff, DEV_BSIZE)) != endoff && (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0) pmap_zero_page_area(m, endoff, DEV_BSIZE - (endoff & (DEV_BSIZE - 1))); /* * Assert that no previously invalid block that is now being validated * is already dirty. */ KASSERT((~m->valid & vm_page_bits(base, size) & m->dirty) == 0, ("vm_page_set_valid_range: page %p is dirty", m)); /* * Set valid bits inclusive of any overlap. */ m->valid |= vm_page_bits(base, size); } /* * Clear the given bits from the specified page's dirty field. */ static __inline void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits) { uintptr_t addr; #if PAGE_SIZE < 16384 int shift; #endif /* * If the object is locked and the page is neither exclusive busy nor * write mapped, then the page's dirty field cannot possibly be * set by a concurrent pmap operation. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && !pmap_page_is_write_mapped(m)) m->dirty &= ~pagebits; else { /* * The pmap layer can call vm_page_dirty() without * holding a distinguished lock. The combination of * the object's lock and an atomic operation suffice * to guarantee consistency of the page dirty field. * * For PAGE_SIZE == 32768 case, compiler already * properly aligns the dirty field, so no forcible * alignment is needed. Only require existence of * atomic_clear_64 when page size is 32768. */ addr = (uintptr_t)&m->dirty; #if PAGE_SIZE == 32768 atomic_clear_64((uint64_t *)addr, pagebits); #elif PAGE_SIZE == 16384 atomic_clear_32((uint32_t *)addr, pagebits); #else /* PAGE_SIZE <= 8192 */ /* * Use a trick to perform a 32-bit atomic on the * containing aligned word, to not depend on the existence * of atomic_clear_{8, 16}. */ shift = addr & (sizeof(uint32_t) - 1); #if BYTE_ORDER == BIG_ENDIAN shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY; #else shift *= NBBY; #endif addr &= ~(sizeof(uint32_t) - 1); atomic_clear_32((uint32_t *)addr, pagebits << shift); #endif /* PAGE_SIZE */ } } /* * vm_page_set_validclean: * * Sets portions of a page valid and clean. The arguments are expected * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive * of any partial chunks touched by the range. The invalid portion of * such chunks will be zero'd. * * (base + size) must be less then or equal to PAGE_SIZE. */ void vm_page_set_validclean(vm_page_t m, int base, int size) { vm_page_bits_t oldvalid, pagebits; int endoff, frag; VM_OBJECT_ASSERT_WLOCKED(m->object); if (size == 0) /* handle degenerate case */ return; /* * If the base is not DEV_BSIZE aligned and the valid * bit is clear, we have to zero out a portion of the * first block. */ if ((frag = rounddown2(base, DEV_BSIZE)) != base && (m->valid & ((vm_page_bits_t)1 << (base >> DEV_BSHIFT))) == 0) pmap_zero_page_area(m, frag, base - frag); /* * If the ending offset is not DEV_BSIZE aligned and the * valid bit is clear, we have to zero out a portion of * the last block. */ endoff = base + size; if ((frag = rounddown2(endoff, DEV_BSIZE)) != endoff && (m->valid & ((vm_page_bits_t)1 << (endoff >> DEV_BSHIFT))) == 0) pmap_zero_page_area(m, endoff, DEV_BSIZE - (endoff & (DEV_BSIZE - 1))); /* * Set valid, clear dirty bits. If validating the entire * page we can safely clear the pmap modify bit. We also * use this opportunity to clear the VPO_NOSYNC flag. If a process * takes a write fault on a MAP_NOSYNC memory area the flag will * be set again. * * We set valid bits inclusive of any overlap, but we can only * clear dirty bits for DEV_BSIZE chunks that are fully within * the range. */ oldvalid = m->valid; pagebits = vm_page_bits(base, size); m->valid |= pagebits; #if 0 /* NOT YET */ if ((frag = base & (DEV_BSIZE - 1)) != 0) { frag = DEV_BSIZE - frag; base += frag; size -= frag; if (size < 0) size = 0; } pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1)); #endif if (base == 0 && size == PAGE_SIZE) { /* * The page can only be modified within the pmap if it is * mapped, and it can only be mapped if it was previously * fully valid. */ if (oldvalid == VM_PAGE_BITS_ALL) /* * Perform the pmap_clear_modify() first. Otherwise, * a concurrent pmap operation, such as * pmap_protect(), could clear a modification in the * pmap and set the dirty field on the page before * pmap_clear_modify() had begun and after the dirty * field was cleared here. */ pmap_clear_modify(m); m->dirty = 0; m->oflags &= ~VPO_NOSYNC; } else if (oldvalid != VM_PAGE_BITS_ALL) m->dirty &= ~pagebits; else vm_page_clear_dirty_mask(m, pagebits); } void vm_page_clear_dirty(vm_page_t m, int base, int size) { vm_page_clear_dirty_mask(m, vm_page_bits(base, size)); } /* * vm_page_set_invalid: * * Invalidates DEV_BSIZE'd chunks within a page. Both the * valid and dirty bits for the effected areas are cleared. */ void vm_page_set_invalid(vm_page_t m, int base, int size) { vm_page_bits_t bits; vm_object_t object; object = m->object; VM_OBJECT_ASSERT_WLOCKED(object); if (object->type == OBJT_VNODE && base == 0 && IDX_TO_OFF(m->pindex) + size >= object->un_pager.vnp.vnp_size) bits = VM_PAGE_BITS_ALL; else bits = vm_page_bits(base, size); if (object->ref_count != 0 && m->valid == VM_PAGE_BITS_ALL && bits != 0) pmap_remove_all(m); KASSERT((bits == 0 && m->valid == VM_PAGE_BITS_ALL) || !pmap_page_is_mapped(m), ("vm_page_set_invalid: page %p is mapped", m)); m->valid &= ~bits; m->dirty &= ~bits; } /* * vm_page_zero_invalid() * * The kernel assumes that the invalid portions of a page contain * garbage, but such pages can be mapped into memory by user code. * When this occurs, we must zero out the non-valid portions of the * page so user code sees what it expects. * * Pages are most often semi-valid when the end of a file is mapped * into memory and the file's size is not page aligned. */ void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid) { int b; int i; VM_OBJECT_ASSERT_WLOCKED(m->object); /* * Scan the valid bits looking for invalid sections that * must be zeroed. Invalid sub-DEV_BSIZE'd areas ( where the * valid bit may be set ) have already been zeroed by * vm_page_set_validclean(). */ for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) { if (i == (PAGE_SIZE / DEV_BSIZE) || (m->valid & ((vm_page_bits_t)1 << i))) { if (i > b) { pmap_zero_page_area(m, b << DEV_BSHIFT, (i - b) << DEV_BSHIFT); } b = i + 1; } } /* * setvalid is TRUE when we can safely set the zero'd areas * as being valid. We can do this if there are no cache consistancy * issues. e.g. it is ok to do with UFS, but not ok to do with NFS. */ if (setvalid) m->valid = VM_PAGE_BITS_ALL; } /* * vm_page_is_valid: * * Is (partial) page valid? Note that the case where size == 0 * will return FALSE in the degenerate case where the page is * entirely invalid, and TRUE otherwise. */ int vm_page_is_valid(vm_page_t m, int base, int size) { vm_page_bits_t bits; VM_OBJECT_ASSERT_LOCKED(m->object); bits = vm_page_bits(base, size); return (m->valid != 0 && (m->valid & bits) == bits); } /* * vm_page_ps_is_valid: * * Returns TRUE if the entire (super)page is valid and FALSE otherwise. */ boolean_t vm_page_ps_is_valid(vm_page_t m) { int i, npages; VM_OBJECT_ASSERT_LOCKED(m->object); npages = atop(pagesizes[m->psind]); /* * The physically contiguous pages that make up a superpage, i.e., a * page with a page size index ("psind") greater than zero, will * occupy adjacent entries in vm_page_array[]. */ for (i = 0; i < npages; i++) { if (m[i].valid != VM_PAGE_BITS_ALL) return (FALSE); } return (TRUE); } /* * Set the page's dirty bits if the page is modified. */ void vm_page_test_dirty(vm_page_t m) { VM_OBJECT_ASSERT_WLOCKED(m->object); if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m)) vm_page_dirty(m); } void vm_page_lock_KBI(vm_page_t m, const char *file, int line) { mtx_lock_flags_(vm_page_lockptr(m), 0, file, line); } void vm_page_unlock_KBI(vm_page_t m, const char *file, int line) { mtx_unlock_flags_(vm_page_lockptr(m), 0, file, line); } int vm_page_trylock_KBI(vm_page_t m, const char *file, int line) { return (mtx_trylock_flags_(vm_page_lockptr(m), 0, file, line)); } #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) void vm_page_assert_locked_KBI(vm_page_t m, const char *file, int line) { vm_page_lock_assert_KBI(m, MA_OWNED, file, line); } void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line) { mtx_assert_(vm_page_lockptr(m), a, file, line); } #endif #ifdef INVARIANTS void vm_page_object_lock_assert(vm_page_t m) { /* * Certain of the page's fields may only be modified by the * holder of the containing object's lock or the exclusive busy. * holder. Unfortunately, the holder of the write busy is * not recorded, and thus cannot be checked here. */ if (m->object != NULL && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_WLOCKED(m->object); } void vm_page_assert_pga_writeable(vm_page_t m, uint8_t bits) { if ((bits & PGA_WRITEABLE) == 0) return; /* * The PGA_WRITEABLE flag can only be set if the page is * managed, is exclusively busied or the object is locked. * Currently, this flag is only set by pmap_enter(). */ KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("PGA_WRITEABLE on unmanaged page")); if (!vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); } #endif #include "opt_ddb.h" #ifdef DDB #include #include DB_SHOW_COMMAND(page, vm_page_print_page_info) { db_printf("vm_cnt.v_free_count: %d\n", vm_cnt.v_free_count); db_printf("vm_cnt.v_cache_count: %d\n", vm_cnt.v_cache_count); db_printf("vm_cnt.v_inactive_count: %d\n", vm_cnt.v_inactive_count); db_printf("vm_cnt.v_active_count: %d\n", vm_cnt.v_active_count); db_printf("vm_cnt.v_wire_count: %d\n", vm_cnt.v_wire_count); db_printf("vm_cnt.v_free_reserved: %d\n", vm_cnt.v_free_reserved); db_printf("vm_cnt.v_free_min: %d\n", vm_cnt.v_free_min); db_printf("vm_cnt.v_free_target: %d\n", vm_cnt.v_free_target); db_printf("vm_cnt.v_inactive_target: %d\n", vm_cnt.v_inactive_target); } DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info) { int dom; db_printf("pq_free %d pq_cache %d\n", vm_cnt.v_free_count, vm_cnt.v_cache_count); for (dom = 0; dom < vm_ndomains; dom++) { db_printf( "dom %d page_cnt %d free %d pq_act %d pq_inact %d pass %d\n", dom, vm_dom[dom].vmd_page_count, vm_dom[dom].vmd_free_count, vm_dom[dom].vmd_pagequeues[PQ_ACTIVE].pq_cnt, vm_dom[dom].vmd_pagequeues[PQ_INACTIVE].pq_cnt, vm_dom[dom].vmd_pass); } } DB_SHOW_COMMAND(pginfo, vm_page_print_pginfo) { vm_page_t m; boolean_t phys; if (!have_addr) { db_printf("show pginfo addr\n"); return; } phys = strchr(modif, 'p') != NULL; if (phys) m = PHYS_TO_VM_PAGE(addr); else m = (vm_page_t)addr; db_printf( "page %p obj %p pidx 0x%jx phys 0x%jx q %d hold %d wire %d\n" " af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n", m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr, m->queue, m->hold_count, m->wire_count, m->aflags, m->oflags, m->flags, m->act_count, m->busy_lock, m->valid, m->dirty); } #endif /* DDB */ Index: projects/clang390-import/tools/tools/crypto/cryptotest.c =================================================================== --- projects/clang390-import/tools/tools/crypto/cryptotest.c (revision 305080) +++ projects/clang390-import/tools/tools/crypto/cryptotest.c (revision 305081) @@ -1,622 +1,627 @@ /*- * Copyright (c) 2004 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * 3. Neither the names of the above-listed copyright holders nor the names * of any contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ /* * Simple tool for testing hardware/system crypto support. * * cryptotest [-czsbv] [-a algorithm] [count] [size ...] * * Run count iterations of a crypt+decrypt or mac operation on a buffer of * size bytes. A random key and iv are used. Options: * -c check the results * -d dev pin work on device dev * -z run all available algorithms on a variety of buffer sizes * -v be verbose * -b mark operations for batching * -p profile kernel crypto operations (must be root) * -t n fork n threads and run tests concurrently * Known algorithms are: * null null cbc * des des cbc * 3des 3des cbc * blf blowfish cbc * cast cast cbc * skj skipjack cbc * aes rijndael/aes 128-bit cbc * aes192 rijndael/aes 192-bit cbc * aes256 rijndael/aes 256-bit cbc * md5 md5 hmac * sha1 sha1 hmac * sha256 256-bit sha2 hmac * sha384 384-bit sha2 hmac * sha512 512--bit sha2 hmac * * For a test of how fast a crypto card is, use something like: * cryptotest -z 1024 * This will run a series of tests using the available crypto/cipher * algorithms over a variety of buffer sizes. The 1024 says to do 1024 * iterations. Extra arguments can be used to specify one or more buffer * sizes to use in doing tests. * * To fork multiple processes all doing the same work, specify -t X on the * command line to get X "threads" running simultaneously. No effort is made * to synchronize the threads or otherwise maximize load. * * If the kernel crypto code is built with CRYPTO_TIMING and you run as root, * then you can specify the -p option to get a "profile" of the time spent * processing crypto operations. At present this data is only meaningful for * symmetric operations. To get meaningful numbers you must run on an idle * machine. * * Expect ~400 Mb/s for a Broadcom 582x for 8K buffers on a reasonable CPU * (64-bit PCI helps). Hifn 7811 parts top out at ~110 Mb/s. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CHUNK 64 /* how much to display */ #define streq(a,b) (strcasecmp(a,b) == 0) void hexdump(char *, int); int verbose = 0; int opflags = 0; int verify = 0; int crid = CRYPTO_FLAG_HARDWARE; struct alg { const char* name; int ishash; int blocksize; int minkeylen; int maxkeylen; int code; } algorithms[] = { #ifdef CRYPTO_NULL_CBC { "null", 0, 8, 1, 256, CRYPTO_NULL_CBC }, #endif { "des", 0, 8, 8, 8, CRYPTO_DES_CBC }, { "3des", 0, 8, 24, 24, CRYPTO_3DES_CBC }, { "blf", 0, 8, 5, 56, CRYPTO_BLF_CBC }, { "cast", 0, 8, 5, 16, CRYPTO_CAST_CBC }, { "skj", 0, 8, 10, 10, CRYPTO_SKIPJACK_CBC }, - { "aes", 0, 16, 16, 16, CRYPTO_RIJNDAEL128_CBC}, - { "aes192", 0, 16, 24, 24, CRYPTO_RIJNDAEL128_CBC}, - { "aes256", 0, 16, 32, 32, CRYPTO_RIJNDAEL128_CBC}, + { "rij", 0, 16, 16, 16, CRYPTO_RIJNDAEL128_CBC}, + { "aes", 0, 16, 16, 16, CRYPTO_AES_CBC}, + { "aes192", 0, 16, 24, 24, CRYPTO_AES_CBC}, + { "aes256", 0, 16, 32, 32, CRYPTO_AES_CBC}, #ifdef notdef { "arc4", 0, 8, 1, 32, CRYPTO_ARC4 }, #endif { "md5", 1, 8, 16, 16, CRYPTO_MD5_HMAC }, { "sha1", 1, 8, 20, 20, CRYPTO_SHA1_HMAC }, { "sha256", 1, 8, 32, 32, CRYPTO_SHA2_256_HMAC }, { "sha384", 1, 8, 48, 48, CRYPTO_SHA2_384_HMAC }, { "sha512", 1, 8, 64, 64, CRYPTO_SHA2_512_HMAC }, }; -static void +void usage(const char* cmd) { printf("usage: %s [-czsbv] [-d dev] [-a algorithm] [count] [size ...]\n", cmd); printf("where algorithm is one of:\n"); - printf(" des 3des (default) blowfish cast skipjack\n"); - printf(" aes (aka rijndael) aes192 aes256 arc4\n"); + printf(" des 3des (default) blowfish cast skipjack rij\n"); + printf(" aes aes192 aes256 arc4\n"); printf("count is the number of encrypt/decrypt ops to do\n"); printf("size is the number of bytes of text to encrypt+decrypt\n"); printf("\n"); printf("-c check the results (slows timing)\n"); - printf("-d use specific device\n"); + printf("-d use specific device, specify 'soft' for testing software implementations\n"); + printf("\tNOTE: to use software you must set:\n\t sysctl kern.cryptodevallowsoft=1\n"); printf("-z run all available algorithms on a variety of sizes\n"); printf("-v be verbose\n"); printf("-b mark operations for batching\n"); printf("-p profile kernel crypto operation (must be root)\n"); exit(-1); } -static struct alg* +struct alg* getalgbycode(int cipher) { int i; for (i = 0; i < nitems(algorithms); i++) if (cipher == algorithms[i].code) return &algorithms[i]; return NULL; } -static struct alg* +struct alg* getalgbyname(const char* name) { int i; for (i = 0; i < nitems(algorithms); i++) if (streq(name, algorithms[i].name)) return &algorithms[i]; return NULL; } -static int +int devcrypto(void) { - static int fd = -1; + int fd = -1; if (fd < 0) { fd = open(_PATH_DEV "crypto", O_RDWR, 0); if (fd < 0) err(1, _PATH_DEV "crypto"); if (fcntl(fd, F_SETFD, 1) == -1) err(1, "fcntl(F_SETFD) (devcrypto)"); } return fd; } -static int +int crlookup(const char *devname) { struct crypt_find_op find; + if (strncmp(devname, "soft", 4) == 0) + return CRYPTO_FLAG_SOFTWARE; + find.crid = -1; strlcpy(find.name, devname, sizeof(find.name)); if (ioctl(devcrypto(), CIOCFINDDEV, &find) == -1) err(1, "ioctl(CIOCFINDDEV)"); return find.crid; } -static const char * +const char * crfind(int crid) { - static struct crypt_find_op find; + struct crypt_find_op find; bzero(&find, sizeof(find)); find.crid = crid; if (ioctl(devcrypto(), CRIOFINDDEV, &find) == -1) err(1, "ioctl(CIOCFINDDEV): crid %d", crid); return find.name; } -static int +int crget(void) { int fd; if (ioctl(devcrypto(), CRIOGET, &fd) == -1) err(1, "ioctl(CRIOGET)"); if (fcntl(fd, F_SETFD, 1) == -1) err(1, "fcntl(F_SETFD) (crget)"); return fd; } -static char +char rdigit(void) { const char a[] = { 0x10,0x54,0x11,0x48,0x45,0x12,0x4f,0x13,0x49,0x53,0x14,0x41, 0x15,0x16,0x4e,0x55,0x54,0x17,0x18,0x4a,0x4f,0x42,0x19,0x01 }; return 0x20+a[random()%nitems(a)]; } -static void +void runtest(struct alg *alg, int count, int size, u_long cmd, struct timeval *tv) { int i, fd = crget(); struct timeval start, stop, dt; char *cleartext, *ciphertext, *originaltext; struct session2_op sop; struct crypt_op cop; char iv[EALG_MAX_BLOCK_LEN]; bzero(&sop, sizeof(sop)); if (!alg->ishash) { sop.keylen = (alg->minkeylen + alg->maxkeylen)/2; sop.key = (char *) malloc(sop.keylen); if (sop.key == NULL) err(1, "malloc (key)"); for (i = 0; i < sop.keylen; i++) sop.key[i] = rdigit(); sop.cipher = alg->code; } else { sop.mackeylen = (alg->minkeylen + alg->maxkeylen)/2; sop.mackey = (char *) malloc(sop.mackeylen); if (sop.mackey == NULL) err(1, "malloc (mac)"); for (i = 0; i < sop.mackeylen; i++) sop.mackey[i] = rdigit(); sop.mac = alg->code; } sop.crid = crid; if (ioctl(fd, cmd, &sop) < 0) { if (cmd == CIOCGSESSION || cmd == CIOCGSESSION2) { close(fd); if (verbose) { printf("cipher %s", alg->name); if (alg->ishash) printf(" mackeylen %u\n", sop.mackeylen); else printf(" keylen %u\n", sop.keylen); perror("CIOCGSESSION"); } /* hardware doesn't support algorithm; skip it */ return; } printf("cipher %s keylen %u mackeylen %u\n", alg->name, sop.keylen, sop.mackeylen); err(1, "CIOCGSESSION"); } originaltext = malloc(3*size); if (originaltext == NULL) err(1, "malloc (text)"); cleartext = originaltext+size; ciphertext = cleartext+size; for (i = 0; i < size; i++) cleartext[i] = rdigit(); memcpy(originaltext, cleartext, size); for (i = 0; i < nitems(iv); i++) iv[i] = rdigit(); if (verbose) { printf("session = 0x%x\n", sop.ses); printf("device = %s\n", crfind(sop.crid)); printf("count = %d, size = %d\n", count, size); if (!alg->ishash) { printf("iv:"); hexdump(iv, sizeof iv); } printf("cleartext:"); hexdump(cleartext, MIN(size, CHUNK)); } gettimeofday(&start, NULL); if (!alg->ishash) { for (i = 0; i < count; i++) { cop.ses = sop.ses; cop.op = COP_ENCRYPT; cop.flags = opflags; cop.len = size; cop.src = cleartext; cop.dst = ciphertext; cop.mac = 0; cop.iv = iv; if (ioctl(fd, CIOCCRYPT, &cop) < 0) err(1, "ioctl(CIOCCRYPT)"); if (verify && bcmp(ciphertext, cleartext, size) == 0) { printf("cipher text unchanged:"); hexdump(ciphertext, size); } memset(cleartext, 'x', MIN(size, CHUNK)); cop.ses = sop.ses; cop.op = COP_DECRYPT; cop.flags = opflags; cop.len = size; cop.src = ciphertext; cop.dst = cleartext; cop.mac = 0; cop.iv = iv; if (ioctl(fd, CIOCCRYPT, &cop) < 0) err(1, "ioctl(CIOCCRYPT)"); if (verify && bcmp(cleartext, originaltext, size) != 0) { printf("decrypt mismatch:\n"); printf("original:"); hexdump(originaltext, size); printf("cleartext:"); hexdump(cleartext, size); } } } else { for (i = 0; i < count; i++) { cop.ses = sop.ses; cop.op = 0; cop.flags = opflags; cop.len = size; cop.src = cleartext; cop.dst = 0; cop.mac = ciphertext; cop.iv = 0; if (ioctl(fd, CIOCCRYPT, &cop) < 0) err(1, "ioctl(CIOCCRYPT)"); } } gettimeofday(&stop, NULL); if (ioctl(fd, CIOCFSESSION, &sop.ses) < 0) perror("ioctl(CIOCFSESSION)"); if (verbose) { printf("cleartext:"); hexdump(cleartext, MIN(size, CHUNK)); } timersub(&stop, &start, tv); free(originaltext); close(fd); } #ifdef __FreeBSD__ -static void +void resetstats() { struct cryptostats stats; size_t slen; slen = sizeof (stats); if (sysctlbyname("kern.crypto_stats", &stats, &slen, NULL, 0) < 0) { perror("kern.crypto_stats"); return; } bzero(&stats.cs_invoke, sizeof (stats.cs_invoke)); bzero(&stats.cs_done, sizeof (stats.cs_done)); bzero(&stats.cs_cb, sizeof (stats.cs_cb)); bzero(&stats.cs_finis, sizeof (stats.cs_finis)); stats.cs_invoke.min.tv_sec = 10000; stats.cs_done.min.tv_sec = 10000; stats.cs_cb.min.tv_sec = 10000; stats.cs_finis.min.tv_sec = 10000; if (sysctlbyname("kern.crypto_stats", NULL, NULL, &stats, sizeof (stats)) < 0) perror("kern.cryptostats"); } -static void +void printt(const char* tag, struct cryptotstat *ts) { uint64_t avg, min, max; if (ts->count == 0) return; avg = (1000000000LL*ts->acc.tv_sec + ts->acc.tv_nsec) / ts->count; min = 1000000000LL*ts->min.tv_sec + ts->min.tv_nsec; max = 1000000000LL*ts->max.tv_sec + ts->max.tv_nsec; printf("%16.16s: avg %6llu ns : min %6llu ns : max %7llu ns [%u samps]\n", tag, avg, min, max, ts->count); } #endif -static void +void runtests(struct alg *alg, int count, int size, u_long cmd, int threads, int profile) { int i, status; double t; void *region; struct timeval *tvp; struct timeval total; int otiming; if (size % alg->blocksize) { if (verbose) printf("skipping blocksize %u 'cuz not a multiple of " "%s blocksize %u\n", size, alg->name, alg->blocksize); return; } region = mmap(NULL, threads * sizeof (struct timeval), PROT_READ|PROT_WRITE, MAP_ANON|MAP_SHARED, -1, 0); if (region == MAP_FAILED) { perror("mmap"); return; } tvp = (struct timeval *) region; #ifdef __FreeBSD__ if (profile) { size_t tlen = sizeof (otiming); int timing = 1; resetstats(); if (sysctlbyname("debug.crypto_timing", &otiming, &tlen, &timing, sizeof (timing)) < 0) perror("debug.crypto_timing"); } #endif if (threads > 1) { for (i = 0; i < threads; i++) if (fork() == 0) { runtest(alg, count, size, cmd, &tvp[i]); exit(0); } while (waitpid(WAIT_MYPGRP, &status, 0) != -1) ; } else runtest(alg, count, size, cmd, tvp); t = 0; for (i = 0; i < threads; i++) t += (((double)tvp[i].tv_sec * 1000000 + tvp[i].tv_usec) / 1000000); if (t) { int nops = alg->ishash ? count : 2*count; #if 0 t /= threads; printf("%6.3lf sec, %7d %6s crypts, %7d bytes, %8.0lf byte/sec, %7.1lf Mb/sec\n", t, nops, alg->name, size, (double)nops*size / t, (double)nops*size / t * 8 / 1024 / 1024); #else nops *= threads; printf("%8.3lf sec, %7d %6s crypts, %7d bytes, %8.0lf byte/sec, %7.1lf Mb/sec\n", t, nops, alg->name, size, (double)nops*size / t, (double)nops*size / t * 8 / 1024 / 1024); #endif } #ifdef __FreeBSD__ if (profile) { struct cryptostats stats; size_t slen = sizeof (stats); if (sysctlbyname("debug.crypto_timing", NULL, NULL, &otiming, sizeof (otiming)) < 0) perror("debug.crypto_timing"); if (sysctlbyname("kern.crypto_stats", &stats, &slen, NULL, 0) < 0) perror("kern.cryptostats"); if (stats.cs_invoke.count) { printt("dispatch->invoke", &stats.cs_invoke); printt("invoke->done", &stats.cs_done); printt("done->cb", &stats.cs_cb); printt("cb->finis", &stats.cs_finis); } } #endif fflush(stdout); } int main(int argc, char **argv) { struct alg *alg = NULL; int count = 1; int sizes[128], nsizes = 0; u_long cmd = CIOCGSESSION2; int testall = 0; int maxthreads = 1; int profile = 0; int i, ch; while ((ch = getopt(argc, argv, "cpzsva:bd:t:")) != -1) { switch (ch) { #ifdef CIOCGSSESSION case 's': cmd = CIOCGSSESSION; break; #endif case 'v': verbose++; break; case 'a': alg = getalgbyname(optarg); if (alg == NULL) { if (streq(optarg, "rijndael")) alg = getalgbyname("aes"); else usage(argv[0]); } break; case 'd': crid = crlookup(optarg); break; case 't': maxthreads = atoi(optarg); break; case 'z': testall = 1; break; case 'p': profile = 1; break; case 'b': opflags |= COP_F_BATCH; break; case 'c': verify = 1; break; default: usage(argv[0]); } } argc -= optind, argv += optind; if (argc > 0) count = atoi(argv[0]); while (argc > 1) { int s = atoi(argv[1]); if (nsizes < nitems(sizes)) { sizes[nsizes++] = s; } else { printf("Too many sizes, ignoring %u\n", s); } argc--, argv++; } if (nsizes == 0) { if (alg) sizes[nsizes++] = alg->blocksize; else sizes[nsizes++] = 8; if (testall) { while (sizes[nsizes-1] < 8*1024) { sizes[nsizes] = sizes[nsizes-1]<<1; nsizes++; } } } if (testall) { for (i = 0; i < nitems(algorithms); i++) { int j; alg = &algorithms[i]; for (j = 0; j < nsizes; j++) runtests(alg, count, sizes[j], cmd, maxthreads, profile); } } else { if (alg == NULL) alg = getalgbycode(CRYPTO_3DES_CBC); for (i = 0; i < nsizes; i++) runtests(alg, count, sizes[i], cmd, maxthreads, profile); } return (0); } void hexdump(char *p, int n) { int i, off; for (off = 0; n > 0; off += 16, n -= 16) { printf("%s%04x:", off == 0 ? "\n" : "", off); i = (n >= 16 ? 16 : n); do { printf(" %02x", *p++ & 0xff); } while (--i); printf("\n"); } } Index: projects/clang390-import/usr.bin/newkey/generic.c =================================================================== --- projects/clang390-import/usr.bin/newkey/generic.c (revision 305080) +++ projects/clang390-import/usr.bin/newkey/generic.c (revision 305081) @@ -1,132 +1,132 @@ /* * Sun RPC is a product of Sun Microsystems, Inc. and is provided for * unrestricted use provided that this legend is included on all tape * media and as a part of the software program in whole or part. Users * may copy or modify Sun RPC without charge, but are not authorized * to license or distribute it to anyone else except as part of a product or * program developed by the user or with the express written consent of * Sun Microsystems, Inc. * * SUN RPC IS PROVIDED AS IS WITH NO WARRANTIES OF ANY KIND INCLUDING THE * WARRANTIES OF DESIGN, MERCHANTIBILITY AND FITNESS FOR A PARTICULAR * PURPOSE, OR ARISING FROM A COURSE OF DEALING, USAGE OR TRADE PRACTICE. * * Sun RPC is provided with no support and without any obligation on the * part of Sun Microsystems, Inc. to assist in its use, correction, * modification or enhancement. * * SUN MICROSYSTEMS, INC. SHALL HAVE NO LIABILITY WITH RESPECT TO THE * INFRINGEMENT OF COPYRIGHTS, TRADE SECRETS OR ANY PATENTS BY SUN RPC * OR ANY PART THEREOF. * * In no event will Sun Microsystems, Inc. be liable for any lost revenue * or profits or other special, indirect and consequential damages, even if * Sun has been advised of the possibility of such damages. * * Sun Microsystems, Inc. * 2550 Garcia Avenue * Mountain View, California 94043 */ #if !defined(lint) && defined(SCCSIDS) #if 0 static char sccsid[] = "@(#)generic.c 1.2 91/03/11 Copyr 1986 Sun Micro"; #endif #endif /* * Copyright (C) 1986, Sun Microsystems, Inc. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include "extern.h" static void adjust(char[], char *); static void getseed(char *, int, unsigned char *); /* * Generate a seed */ static void getseed(char *seed, int seedsize, unsigned char *pass) { int i; for (i = 0; i < seedsize; i++) { seed[i] = (arc4random() & 0xff) ^ pass[i % 8]; } } /* * Generate a random public/secret key pair */ void genkeys(char *public, char *secret, char *pass) { unsigned int i; # define BASEBITS (8*sizeof (short) - 1) # define BASE (1 << BASEBITS) MINT *pk = mp_itom(0); MINT *sk = mp_itom(0); MINT *tmp; - MINT *base = mp_itom(BASE); + MINT *base = mp_itom((short)BASE); MINT *root = mp_itom(PROOT); MINT *modulus = mp_xtom(HEXMODULUS); short r; unsigned short seed[KEYSIZE/BASEBITS + 1]; char *xkey; getseed((char *)seed, sizeof (seed), (u_char *)pass); for (i = 0; i < KEYSIZE/BASEBITS + 1; i++) { r = seed[i] % BASE; tmp = mp_itom(r); mp_mult(sk, base, sk); mp_madd(sk, tmp, sk); mp_mfree(tmp); } tmp = mp_itom(0); mp_mdiv(sk, modulus, tmp, sk); mp_mfree(tmp); mp_pow(root, sk, modulus, pk); xkey = mp_mtox(sk); adjust(secret, xkey); xkey = mp_mtox(pk); adjust(public, xkey); mp_mfree(sk); mp_mfree(base); mp_mfree(pk); mp_mfree(root); mp_mfree(modulus); } /* * Adjust the input key so that it is 0-filled on the left */ static void adjust(char keyout[HEXKEYBYTES+1], char *keyin) { char *p; char *s; for (p = keyin; *p; p++) ; for (s = keyout + HEXKEYBYTES; p >= keyin; p--, s--) { *s = *p; } while (s >= keyout) { *s-- = '0'; } } Index: projects/clang390-import/usr.sbin/bhyve/pci_e82545.c =================================================================== --- projects/clang390-import/usr.sbin/bhyve/pci_e82545.c (revision 305080) +++ projects/clang390-import/usr.sbin/bhyve/pci_e82545.c (revision 305081) @@ -1,2372 +1,2372 @@ /* * Copyright (c) 2016 Alexander Motin * Copyright (c) 2015 Peter Grehan * Copyright (c) 2013 Jeremiah Lott, Avere Systems * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "e1000_regs.h" #include "e1000_defines.h" #include "mii.h" #include "bhyverun.h" #include "pci_emul.h" #include "mevent.h" /* Hardware/register definitions XXX: move some to common code. */ #define E82545_VENDOR_ID_INTEL 0x8086 #define E82545_DEV_ID_82545EM_COPPER 0x100F #define E82545_SUBDEV_ID 0x1008 #define E82545_REVISION_4 4 #define E82545_MDIC_DATA_MASK 0x0000FFFF #define E82545_MDIC_OP_MASK 0x0c000000 #define E82545_MDIC_IE 0x20000000 #define E82545_EECD_FWE_DIS 0x00000010 /* Flash writes disabled */ #define E82545_EECD_FWE_EN 0x00000020 /* Flash writes enabled */ #define E82545_EECD_FWE_MASK 0x00000030 /* Flash writes mask */ #define E82545_BAR_REGISTER 0 #define E82545_BAR_REGISTER_LEN (128*1024) #define E82545_BAR_FLASH 1 #define E82545_BAR_FLASH_LEN (64*1024) #define E82545_BAR_IO 2 #define E82545_BAR_IO_LEN 8 #define E82545_IOADDR 0x00000000 #define E82545_IODATA 0x00000004 #define E82545_IO_REGISTER_MAX 0x0001FFFF #define E82545_IO_FLASH_BASE 0x00080000 #define E82545_IO_FLASH_MAX 0x000FFFFF #define E82545_ARRAY_ENTRY(reg, offset) (reg + (offset<<2)) #define E82545_RAR_MAX 15 #define E82545_MTA_MAX 127 #define E82545_VFTA_MAX 127 /* Slightly modified from the driver versions, hardcoded for 3 opcode bits, * followed by 6 address bits. * TODO: make opcode bits and addr bits configurable? * NVM Commands - Microwire */ #define E82545_NVM_OPCODE_BITS 3 #define E82545_NVM_ADDR_BITS 6 #define E82545_NVM_DATA_BITS 16 #define E82545_NVM_OPADDR_BITS (E82545_NVM_OPCODE_BITS + E82545_NVM_ADDR_BITS) #define E82545_NVM_ADDR_MASK ((1 << E82545_NVM_ADDR_BITS)-1) #define E82545_NVM_OPCODE_MASK \ (((1 << E82545_NVM_OPCODE_BITS) - 1) << E82545_NVM_ADDR_BITS) #define E82545_NVM_OPCODE_READ (0x6 << E82545_NVM_ADDR_BITS) /* read */ #define E82545_NVM_OPCODE_WRITE (0x5 << E82545_NVM_ADDR_BITS) /* write */ #define E82545_NVM_OPCODE_ERASE (0x7 << E82545_NVM_ADDR_BITS) /* erase */ #define E82545_NVM_OPCODE_EWEN (0x4 << E82545_NVM_ADDR_BITS) /* wr-enable */ #define E82545_NVM_EEPROM_SIZE 64 /* 64 * 16-bit values == 128K */ #define E1000_ICR_SRPD 0x00010000 /* This is an arbitrary number. There is no hard limit on the chip. */ #define I82545_MAX_TXSEGS 64 /* Legacy receive descriptor */ struct e1000_rx_desc { uint64_t buffer_addr; /* Address of the descriptor's data buffer */ uint16_t length; /* Length of data DMAed into data buffer */ uint16_t csum; /* Packet checksum */ uint8_t status; /* Descriptor status */ uint8_t errors; /* Descriptor Errors */ uint16_t special; }; /* Transmit descriptor types */ #define E1000_TXD_MASK (E1000_TXD_CMD_DEXT | 0x00F00000) #define E1000_TXD_TYP_L (0) #define E1000_TXD_TYP_C (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_C) #define E1000_TXD_TYP_D (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D) /* Legacy transmit descriptor */ struct e1000_tx_desc { uint64_t buffer_addr; /* Address of the descriptor's data buffer */ union { uint32_t data; struct { uint16_t length; /* Data buffer length */ uint8_t cso; /* Checksum offset */ uint8_t cmd; /* Descriptor control */ } flags; } lower; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t css; /* Checksum start */ uint16_t special; } fields; } upper; }; /* Context descriptor */ struct e1000_context_desc { union { uint32_t ip_config; struct { uint8_t ipcss; /* IP checksum start */ uint8_t ipcso; /* IP checksum offset */ uint16_t ipcse; /* IP checksum end */ } ip_fields; } lower_setup; union { uint32_t tcp_config; struct { uint8_t tucss; /* TCP checksum start */ uint8_t tucso; /* TCP checksum offset */ uint16_t tucse; /* TCP checksum end */ } tcp_fields; } upper_setup; uint32_t cmd_and_length; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t hdr_len; /* Header length */ uint16_t mss; /* Maximum segment size */ } fields; } tcp_seg_setup; }; /* Data descriptor */ struct e1000_data_desc { uint64_t buffer_addr; /* Address of the descriptor's buffer address */ union { uint32_t data; struct { uint16_t length; /* Data buffer length */ uint8_t typ_len_ext; uint8_t cmd; } flags; } lower; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t popts; /* Packet Options */ uint16_t special; } fields; } upper; }; union e1000_tx_udesc { struct e1000_tx_desc td; struct e1000_context_desc cd; struct e1000_data_desc dd; }; /* Tx checksum info for a packet. */ struct ck_info { int ck_valid; /* ck_info is valid */ uint8_t ck_start; /* start byte of cksum calcuation */ uint8_t ck_off; /* offset of cksum insertion */ uint16_t ck_len; /* length of cksum calc: 0 is to packet-end */ }; /* * Debug printf */ static int e82545_debug = 0; #define DPRINTF(msg,params...) if (e82545_debug) fprintf(stderr, "e82545: " msg, params) #define WPRINTF(msg,params...) fprintf(stderr, "e82545: " msg, params) #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) /* s/w representation of the RAL/RAH regs */ struct eth_uni { int eu_valid; int eu_addrsel; struct ether_addr eu_eth; }; struct e82545_softc { struct pci_devinst *esc_pi; struct vmctx *esc_ctx; struct mevent *esc_mevp; struct mevent *esc_mevpitr; pthread_mutex_t esc_mtx; struct ether_addr esc_mac; int esc_tapfd; /* General */ uint32_t esc_CTRL; /* x0000 device ctl */ uint32_t esc_FCAL; /* x0028 flow ctl addr lo */ uint32_t esc_FCAH; /* x002C flow ctl addr hi */ uint32_t esc_FCT; /* x0030 flow ctl type */ uint32_t esc_VET; /* x0038 VLAN eth type */ uint32_t esc_FCTTV; /* x0170 flow ctl tx timer */ uint32_t esc_LEDCTL; /* x0E00 LED control */ uint32_t esc_PBA; /* x1000 pkt buffer allocation */ /* Interrupt control */ int esc_irq_asserted; uint32_t esc_ICR; /* x00C0 cause read/clear */ uint32_t esc_ITR; /* x00C4 intr throttling */ uint32_t esc_ICS; /* x00C8 cause set */ uint32_t esc_IMS; /* x00D0 mask set/read */ uint32_t esc_IMC; /* x00D8 mask clear */ /* Transmit */ union e1000_tx_udesc *esc_txdesc; struct e1000_context_desc esc_txctx; pthread_t esc_tx_tid; pthread_cond_t esc_tx_cond; int esc_tx_enabled; int esc_tx_active; uint32_t esc_TXCW; /* x0178 transmit config */ uint32_t esc_TCTL; /* x0400 transmit ctl */ uint32_t esc_TIPG; /* x0410 inter-packet gap */ uint16_t esc_AIT; /* x0458 Adaptive Interframe Throttle */ uint64_t esc_tdba; /* verified 64-bit desc table addr */ uint32_t esc_TDBAL; /* x3800 desc table addr, low bits */ uint32_t esc_TDBAH; /* x3804 desc table addr, hi 32-bits */ uint32_t esc_TDLEN; /* x3808 # descriptors in bytes */ uint16_t esc_TDH; /* x3810 desc table head idx */ uint16_t esc_TDHr; /* internal read version of TDH */ uint16_t esc_TDT; /* x3818 desc table tail idx */ uint32_t esc_TIDV; /* x3820 intr delay */ uint32_t esc_TXDCTL; /* x3828 desc control */ uint32_t esc_TADV; /* x382C intr absolute delay */ /* L2 frame acceptance */ struct eth_uni esc_uni[16]; /* 16 x unicast MAC addresses */ uint32_t esc_fmcast[128]; /* Multicast filter bit-match */ uint32_t esc_fvlan[128]; /* VLAN 4096-bit filter */ /* Receive */ struct e1000_rx_desc *esc_rxdesc; pthread_cond_t esc_rx_cond; int esc_rx_enabled; int esc_rx_active; int esc_rx_loopback; uint32_t esc_RCTL; /* x0100 receive ctl */ uint32_t esc_FCRTL; /* x2160 flow cntl thresh, low */ uint32_t esc_FCRTH; /* x2168 flow cntl thresh, hi */ uint64_t esc_rdba; /* verified 64-bit desc table addr */ uint32_t esc_RDBAL; /* x2800 desc table addr, low bits */ uint32_t esc_RDBAH; /* x2804 desc table addr, hi 32-bits*/ uint32_t esc_RDLEN; /* x2808 #descriptors */ uint16_t esc_RDH; /* x2810 desc table head idx */ uint16_t esc_RDT; /* x2818 desc table tail idx */ uint32_t esc_RDTR; /* x2820 intr delay */ uint32_t esc_RXDCTL; /* x2828 desc control */ uint32_t esc_RADV; /* x282C intr absolute delay */ uint32_t esc_RSRPD; /* x2C00 recv small packet detect */ uint32_t esc_RXCSUM; /* x5000 receive cksum ctl */ /* IO Port register access */ uint32_t io_addr; /* Shadow copy of MDIC */ uint32_t mdi_control; /* Shadow copy of EECD */ uint32_t eeprom_control; /* Latest NVM in/out */ uint16_t nvm_data; uint16_t nvm_opaddr; /* stats */ uint32_t missed_pkt_count; /* dropped for no room in rx queue */ uint32_t pkt_rx_by_size[6]; uint32_t pkt_tx_by_size[6]; uint32_t good_pkt_rx_count; uint32_t bcast_pkt_rx_count; uint32_t mcast_pkt_rx_count; uint32_t good_pkt_tx_count; uint32_t bcast_pkt_tx_count; uint32_t mcast_pkt_tx_count; uint32_t oversize_rx_count; uint32_t tso_tx_count; uint64_t good_octets_rx; uint64_t good_octets_tx; uint64_t missed_octets; /* counts missed and oversized */ uint8_t nvm_bits:6; /* number of bits remaining in/out */ uint8_t nvm_mode:2; #define E82545_NVM_MODE_OPADDR 0x0 #define E82545_NVM_MODE_DATAIN 0x1 #define E82545_NVM_MODE_DATAOUT 0x2 /* EEPROM data */ uint16_t eeprom_data[E82545_NVM_EEPROM_SIZE]; }; static void e82545_reset(struct e82545_softc *sc, int dev); static void e82545_rx_enable(struct e82545_softc *sc); static void e82545_rx_disable(struct e82545_softc *sc); static void e82545_tap_callback(int fd, enum ev_type type, void *param); static void e82545_tx_start(struct e82545_softc *sc); static void e82545_tx_enable(struct e82545_softc *sc); static void e82545_tx_disable(struct e82545_softc *sc); static inline int e82545_size_stat_index(uint32_t size) { if (size <= 64) { return 0; } else if (size >= 1024) { return 5; } else { /* should be 1-4 */ return (ffs(size) - 6); } } static void e82545_init_eeprom(struct e82545_softc *sc) { uint16_t checksum, i; /* mac addr */ sc->eeprom_data[NVM_MAC_ADDR] = ((uint16_t)sc->esc_mac.octet[0]) | (((uint16_t)sc->esc_mac.octet[1]) << 8); sc->eeprom_data[NVM_MAC_ADDR+1] = ((uint16_t)sc->esc_mac.octet[2]) | (((uint16_t)sc->esc_mac.octet[3]) << 8); sc->eeprom_data[NVM_MAC_ADDR+2] = ((uint16_t)sc->esc_mac.octet[4]) | (((uint16_t)sc->esc_mac.octet[5]) << 8); /* pci ids */ sc->eeprom_data[NVM_SUB_DEV_ID] = E82545_SUBDEV_ID; sc->eeprom_data[NVM_SUB_VEN_ID] = E82545_VENDOR_ID_INTEL; sc->eeprom_data[NVM_DEV_ID] = E82545_DEV_ID_82545EM_COPPER; sc->eeprom_data[NVM_VEN_ID] = E82545_VENDOR_ID_INTEL; /* fill in the checksum */ checksum = 0; for (i = 0; i < NVM_CHECKSUM_REG; i++) { checksum += sc->eeprom_data[i]; } checksum = NVM_SUM - checksum; sc->eeprom_data[NVM_CHECKSUM_REG] = checksum; DPRINTF("eeprom checksum: 0x%x\r\n", checksum); } static void e82545_write_mdi(struct e82545_softc *sc, uint8_t reg_addr, uint8_t phy_addr, uint32_t data) { DPRINTF("Write mdi reg:0x%x phy:0x%x data: 0x%x\r\n", reg_addr, phy_addr, data); } static uint32_t e82545_read_mdi(struct e82545_softc *sc, uint8_t reg_addr, uint8_t phy_addr) { //DPRINTF("Read mdi reg:0x%x phy:0x%x\r\n", reg_addr, phy_addr); switch (reg_addr) { case PHY_STATUS: return (MII_SR_LINK_STATUS | MII_SR_AUTONEG_CAPS | MII_SR_AUTONEG_COMPLETE); case PHY_AUTONEG_ADV: return NWAY_AR_SELECTOR_FIELD; case PHY_LP_ABILITY: return 0; case PHY_1000T_STATUS: return (SR_1000T_LP_FD_CAPS | SR_1000T_REMOTE_RX_STATUS | SR_1000T_LOCAL_RX_STATUS); case PHY_ID1: return (M88E1011_I_PHY_ID >> 16) & 0xFFFF; case PHY_ID2: return (M88E1011_I_PHY_ID | E82545_REVISION_4) & 0xFFFF; default: DPRINTF("Unknown mdi read reg:0x%x phy:0x%x\r\n", reg_addr, phy_addr); return 0; } /* not reached */ } static void e82545_eecd_strobe(struct e82545_softc *sc) { /* Microwire state machine */ /* DPRINTF("eeprom state machine srtobe " "0x%x 0x%x 0x%x 0x%x\r\n", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data);*/ if (sc->nvm_bits == 0) { DPRINTF("eeprom state machine not expecting data! " "0x%x 0x%x 0x%x 0x%x\r\n", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data); return; } sc->nvm_bits--; if (sc->nvm_mode == E82545_NVM_MODE_DATAOUT) { /* shifting out */ if (sc->nvm_data & 0x8000) { sc->eeprom_control |= E1000_EECD_DO; } else { sc->eeprom_control &= ~E1000_EECD_DO; } sc->nvm_data <<= 1; if (sc->nvm_bits == 0) { /* read done, back to opcode mode. */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } else if (sc->nvm_mode == E82545_NVM_MODE_DATAIN) { /* shifting in */ sc->nvm_data <<= 1; if (sc->eeprom_control & E1000_EECD_DI) { sc->nvm_data |= 1; } if (sc->nvm_bits == 0) { /* eeprom write */ uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; uint16_t addr = sc->nvm_opaddr & E82545_NVM_ADDR_MASK; if (op != E82545_NVM_OPCODE_WRITE) { DPRINTF("Illegal eeprom write op 0x%x\r\n", sc->nvm_opaddr); } else if (addr >= E82545_NVM_EEPROM_SIZE) { DPRINTF("Illegal eeprom write addr 0x%x\r\n", sc->nvm_opaddr); } else { DPRINTF("eeprom write eeprom[0x%x] = 0x%x\r\n", addr, sc->nvm_data); sc->eeprom_data[addr] = sc->nvm_data; } /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } else if (sc->nvm_mode == E82545_NVM_MODE_OPADDR) { sc->nvm_opaddr <<= 1; if (sc->eeprom_control & E1000_EECD_DI) { sc->nvm_opaddr |= 1; } if (sc->nvm_bits == 0) { uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; switch (op) { case E82545_NVM_OPCODE_EWEN: DPRINTF("eeprom write enable: 0x%x\r\n", sc->nvm_opaddr); /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; break; case E82545_NVM_OPCODE_READ: { uint16_t addr = sc->nvm_opaddr & E82545_NVM_ADDR_MASK; sc->nvm_mode = E82545_NVM_MODE_DATAOUT; sc->nvm_bits = E82545_NVM_DATA_BITS; if (addr < E82545_NVM_EEPROM_SIZE) { sc->nvm_data = sc->eeprom_data[addr]; DPRINTF("eeprom read: eeprom[0x%x] = 0x%x\r\n", addr, sc->nvm_data); } else { DPRINTF("eeprom illegal read: 0x%x\r\n", sc->nvm_opaddr); sc->nvm_data = 0; } break; } case E82545_NVM_OPCODE_WRITE: sc->nvm_mode = E82545_NVM_MODE_DATAIN; sc->nvm_bits = E82545_NVM_DATA_BITS; sc->nvm_data = 0; break; default: DPRINTF("eeprom unknown op: 0x%x\r\r", sc->nvm_opaddr); /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } } else { DPRINTF("eeprom state machine wrong state! " "0x%x 0x%x 0x%x 0x%x\r\n", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data); } } static void e82545_itr_callback(int fd, enum ev_type type, void *param) { uint32_t new; struct e82545_softc *sc = param; pthread_mutex_lock(&sc->esc_mtx); new = sc->esc_ICR & sc->esc_IMS; if (new && !sc->esc_irq_asserted) { DPRINTF("itr callback: lintr assert %x\r\n", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); } else { mevent_delete(sc->esc_mevpitr); sc->esc_mevpitr = NULL; } pthread_mutex_unlock(&sc->esc_mtx); } static void e82545_icr_assert(struct e82545_softc *sc, uint32_t bits) { uint32_t new; DPRINTF("icr assert: 0x%x\r\n", bits); /* * An interrupt is only generated if bits are set that * aren't already in the ICR, these bits are unmasked, * and there isn't an interrupt already pending. */ new = bits & ~sc->esc_ICR & sc->esc_IMS; sc->esc_ICR |= bits; if (new == 0) { DPRINTF("icr assert: masked %x, ims %x\r\n", new, sc->esc_IMS); } else if (sc->esc_mevpitr != NULL) { DPRINTF("icr assert: throttled %x, ims %x\r\n", new, sc->esc_IMS); } else if (!sc->esc_irq_asserted) { DPRINTF("icr assert: lintr assert %x\r\n", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); if (sc->esc_ITR != 0) { sc->esc_mevpitr = mevent_add( (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ EVF_TIMER, e82545_itr_callback, sc); } } } static void e82545_ims_change(struct e82545_softc *sc, uint32_t bits) { uint32_t new; /* * Changing the mask may allow previously asserted * but masked interrupt requests to generate an interrupt. */ new = bits & sc->esc_ICR & ~sc->esc_IMS; sc->esc_IMS |= bits; if (new == 0) { DPRINTF("ims change: masked %x, ims %x\r\n", new, sc->esc_IMS); } else if (sc->esc_mevpitr != NULL) { DPRINTF("ims change: throttled %x, ims %x\r\n", new, sc->esc_IMS); } else if (!sc->esc_irq_asserted) { DPRINTF("ims change: lintr assert %x\n\r", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); if (sc->esc_ITR != 0) { sc->esc_mevpitr = mevent_add( (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ EVF_TIMER, e82545_itr_callback, sc); } } } static void e82545_icr_deassert(struct e82545_softc *sc, uint32_t bits) { DPRINTF("icr deassert: 0x%x\r\n", bits); sc->esc_ICR &= ~bits; /* * If there are no longer any interrupt sources and there * was an asserted interrupt, clear it */ if (sc->esc_irq_asserted && !(sc->esc_ICR & sc->esc_IMS)) { DPRINTF("icr deassert: lintr deassert %x\r\n", bits); pci_lintr_deassert(sc->esc_pi); sc->esc_irq_asserted = 0; } } static void e82545_intr_write(struct e82545_softc *sc, uint32_t offset, uint32_t value) { DPRINTF("intr_write: off %x, val %x\n\r", offset, value); switch (offset) { case E1000_ICR: e82545_icr_deassert(sc, value); break; case E1000_ITR: sc->esc_ITR = value; break; case E1000_ICS: sc->esc_ICS = value; /* not used: store for debug */ e82545_icr_assert(sc, value); break; case E1000_IMS: e82545_ims_change(sc, value); break; case E1000_IMC: sc->esc_IMC = value; /* for debug */ sc->esc_IMS &= ~value; // XXX clear interrupts if all ICR bits now masked // and interrupt was pending ? break; default: break; } } static uint32_t e82545_intr_read(struct e82545_softc *sc, uint32_t offset) { uint32_t retval; retval = 0; DPRINTF("intr_read: off %x\n\r", offset); switch (offset) { case E1000_ICR: retval = sc->esc_ICR; sc->esc_ICR = 0; e82545_icr_deassert(sc, ~0); break; case E1000_ITR: retval = sc->esc_ITR; break; case E1000_ICS: /* write-only register */ break; case E1000_IMS: retval = sc->esc_IMS; break; case E1000_IMC: /* write-only register */ break; default: break; } return (retval); } static void e82545_devctl(struct e82545_softc *sc, uint32_t val) { sc->esc_CTRL = val & ~E1000_CTRL_RST; if (val & E1000_CTRL_RST) { DPRINTF("e1k: s/w reset, ctl %x\n", val); e82545_reset(sc, 1); } /* XXX check for phy reset ? */ } static void e82545_rx_update_rdba(struct e82545_softc *sc) { /* XXX verify desc base/len within phys mem range */ sc->esc_rdba = (uint64_t)sc->esc_RDBAH << 32 | sc->esc_RDBAL; /* Cache host mapping of guest descriptor array */ sc->esc_rxdesc = paddr_guest2host(sc->esc_ctx, sc->esc_rdba, sc->esc_RDLEN); } static void e82545_rx_ctl(struct e82545_softc *sc, uint32_t val) { int on; on = ((val & E1000_RCTL_EN) == E1000_RCTL_EN); /* Save RCTL after stripping reserved bits 31:27,24,21,14,11:10,0 */ sc->esc_RCTL = val & ~0xF9204c01; DPRINTF("rx_ctl - %s RCTL %x, val %x\n", on ? "on" : "off", sc->esc_RCTL, val); /* state change requested */ if (on != sc->esc_rx_enabled) { if (on) { /* Catch disallowed/unimplemented settings */ //assert(!(val & E1000_RCTL_LBM_TCVR)); if (sc->esc_RCTL & E1000_RCTL_LBM_TCVR) { sc->esc_rx_loopback = 1; } else { sc->esc_rx_loopback = 0; } e82545_rx_update_rdba(sc); e82545_rx_enable(sc); } else { e82545_rx_disable(sc); sc->esc_rx_loopback = 0; sc->esc_rdba = 0; sc->esc_rxdesc = NULL; } } } static void e82545_tx_update_tdba(struct e82545_softc *sc) { /* XXX verify desc base/len within phys mem range */ sc->esc_tdba = (uint64_t)sc->esc_TDBAH << 32 | sc->esc_TDBAL; /* Cache host mapping of guest descriptor array */ sc->esc_txdesc = paddr_guest2host(sc->esc_ctx, sc->esc_tdba, sc->esc_TDLEN); } static void e82545_tx_ctl(struct e82545_softc *sc, uint32_t val) { int on; on = ((val & E1000_TCTL_EN) == E1000_TCTL_EN); /* ignore TCTL_EN settings that don't change state */ if (on == sc->esc_tx_enabled) return; if (on) { e82545_tx_update_tdba(sc); e82545_tx_enable(sc); } else { e82545_tx_disable(sc); sc->esc_tdba = 0; sc->esc_txdesc = NULL; } /* Save TCTL value after stripping reserved bits 31:25,23,2,0 */ sc->esc_TCTL = val & ~0xFE800005; } int e82545_bufsz(uint32_t rctl) { switch (rctl & (E1000_RCTL_BSEX | E1000_RCTL_SZ_256)) { case (E1000_RCTL_SZ_2048): return (2048); case (E1000_RCTL_SZ_1024): return (1024); case (E1000_RCTL_SZ_512): return (512); case (E1000_RCTL_SZ_256): return (256); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_16384): return (16384); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_8192): return (8192); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_4096): return (4096); } return (256); /* Forbidden value. */ } static uint8_t dummybuf[2048]; /* XXX one packet at a time until this is debugged */ static void e82545_tap_callback(int fd, enum ev_type type, void *param) { struct e82545_softc *sc = param; struct e1000_rx_desc *rxd; struct iovec vec[64]; int left, len, lim, maxpktsz, maxpktdesc, bufsz, i, n, size; uint32_t cause = 0; uint16_t *tp, tag, head; pthread_mutex_lock(&sc->esc_mtx); DPRINTF("rx_run: head %x, tail %x\r\n", sc->esc_RDH, sc->esc_RDT); if (!sc->esc_rx_enabled || sc->esc_rx_loopback) { DPRINTF("rx disabled (!%d || %d) -- packet(s) dropped\r\n", sc->esc_rx_enabled, sc->esc_rx_loopback); while (read(sc->esc_tapfd, dummybuf, sizeof(dummybuf)) > 0) { } goto done1; } bufsz = e82545_bufsz(sc->esc_RCTL); maxpktsz = (sc->esc_RCTL & E1000_RCTL_LPE) ? 16384 : 1522; maxpktdesc = (maxpktsz + bufsz - 1) / bufsz; size = sc->esc_RDLEN / 16; head = sc->esc_RDH; left = (size + sc->esc_RDT - head) % size; if (left < maxpktdesc) { DPRINTF("rx overflow (%d < %d) -- packet(s) dropped\r\n", left, maxpktdesc); while (read(sc->esc_tapfd, dummybuf, sizeof(dummybuf)) > 0) { } goto done1; } sc->esc_rx_active = 1; pthread_mutex_unlock(&sc->esc_mtx); for (lim = size / 4; lim > 0 && left >= maxpktdesc; lim -= n) { /* Grab rx descriptor pointed to by the head pointer */ for (i = 0; i < maxpktdesc; i++) { rxd = &sc->esc_rxdesc[(head + i) % size]; vec[i].iov_base = paddr_guest2host(sc->esc_ctx, rxd->buffer_addr, bufsz); vec[i].iov_len = bufsz; } len = readv(sc->esc_tapfd, vec, maxpktdesc); if (len <= 0) { DPRINTF("tap: readv() returned %d\n", len); goto done; } /* * Adjust the packet length based on whether the CRC needs * to be stripped or if the packet is less than the minimum * eth packet size. */ if (len < ETHER_MIN_LEN - ETHER_CRC_LEN) len = ETHER_MIN_LEN - ETHER_CRC_LEN; if (!(sc->esc_RCTL & E1000_RCTL_SECRC)) len += ETHER_CRC_LEN; n = (len + bufsz - 1) / bufsz; DPRINTF("packet read %d bytes, %d segs, head %d\r\n", len, n, head); /* Apply VLAN filter. */ tp = (uint16_t *)vec[0].iov_base + 6; if ((sc->esc_RCTL & E1000_RCTL_VFE) && (ntohs(tp[0]) == sc->esc_VET)) { tag = ntohs(tp[1]) & 0x0fff; if ((sc->esc_fvlan[tag >> 5] & (1 << (tag & 0x1f))) != 0) { DPRINTF("known VLAN %d\r\n", tag); } else { DPRINTF("unknown VLAN %d\r\n", tag); n = 0; continue; } } /* Update all consumed descriptors. */ for (i = 0; i < n - 1; i++) { rxd = &sc->esc_rxdesc[(head + i) % size]; rxd->length = bufsz; rxd->csum = 0; rxd->errors = 0; rxd->special = 0; rxd->status = E1000_RXD_STAT_DD; } rxd = &sc->esc_rxdesc[(head + i) % size]; rxd->length = len % bufsz; rxd->csum = 0; rxd->errors = 0; rxd->special = 0; /* XXX signal no checksum for now */ rxd->status = E1000_RXD_STAT_PIF | E1000_RXD_STAT_IXSM | E1000_RXD_STAT_EOP | E1000_RXD_STAT_DD; /* Schedule receive interrupts. */ if (len <= sc->esc_RSRPD) { cause |= E1000_ICR_SRPD | E1000_ICR_RXT0; } else { /* XXX: RDRT and RADV timers should be here. */ cause |= E1000_ICR_RXT0; } head = (head + n) % size; left -= n; } done: pthread_mutex_lock(&sc->esc_mtx); sc->esc_rx_active = 0; if (sc->esc_rx_enabled == 0) pthread_cond_signal(&sc->esc_rx_cond); sc->esc_RDH = head; /* Respect E1000_RCTL_RDMTS */ left = (size + sc->esc_RDT - head) % size; if (left < (size >> (((sc->esc_RCTL >> 8) & 3) + 1))) cause |= E1000_ICR_RXDMT0; /* Assert all accumulated interrupts. */ if (cause != 0) e82545_icr_assert(sc, cause); done1: DPRINTF("rx_run done: head %x, tail %x\r\n", sc->esc_RDH, sc->esc_RDT); pthread_mutex_unlock(&sc->esc_mtx); } static uint16_t e82545_carry(uint32_t sum) { sum = (sum & 0xFFFF) + (sum >> 16); if (sum > 0xFFFF) sum -= 0xFFFF; return (sum); } static uint16_t e82545_buf_checksum(uint8_t *buf, int len) { int i; uint32_t sum = 0; /* Checksum all the pairs of bytes first... */ for (i = 0; i < (len & ~1U); i += 2) sum += *((u_int16_t *)(buf + i)); /* * If there's a single byte left over, checksum it, too. * Network byte order is big-endian, so the remaining byte is * the high byte. */ if (i < len) sum += htons(buf[i] << 8); return (e82545_carry(sum)); } static uint16_t e82545_iov_checksum(struct iovec *iov, int iovcnt, int off, int len) { int now, odd; uint32_t sum = 0, s; /* Skip completely unneeded vectors. */ while (iovcnt > 0 && iov->iov_len <= off && off > 0) { off -= iov->iov_len; iov++; iovcnt--; } /* Calculate checksum of requested range. */ odd = 0; while (len > 0 && iovcnt > 0) { now = MIN(len, iov->iov_len - off); s = e82545_buf_checksum(iov->iov_base + off, now); sum += odd ? (s << 8) : s; odd ^= (now & 1); len -= now; off = 0; iov++; iovcnt--; } return (e82545_carry(sum)); } /* * Return the transmit descriptor type. */ int e82545_txdesc_type(uint32_t lower) { int type; type = 0; if (lower & E1000_TXD_CMD_DEXT) type = lower & E1000_TXD_MASK; return (type); } static void e82545_transmit_checksum(struct iovec *iov, int iovcnt, struct ck_info *ck) { uint16_t cksum; int cklen; DPRINTF("tx cksum: iovcnt/s/off/len %d/%d/%d/%d\r\n", iovcnt, ck->ck_start, ck->ck_off, ck->ck_len); cklen = ck->ck_len ? ck->ck_len - ck->ck_start + 1 : INT_MAX; cksum = e82545_iov_checksum(iov, iovcnt, ck->ck_start, cklen); *(uint16_t *)((uint8_t *)iov[0].iov_base + ck->ck_off) = ~cksum; } static void e82545_transmit_backend(struct e82545_softc *sc, struct iovec *iov, int iovcnt) { if (sc->esc_tapfd == -1) return; (void) writev(sc->esc_tapfd, iov, iovcnt); } static void e82545_transmit_done(struct e82545_softc *sc, uint16_t head, uint16_t tail, uint16_t dsize, int *tdwb) { union e1000_tx_udesc *dsc; for ( ; head != tail; head = (head + 1) % dsize) { dsc = &sc->esc_txdesc[head]; if (dsc->td.lower.data & E1000_TXD_CMD_RS) { dsc->td.upper.data |= E1000_TXD_STAT_DD; *tdwb = 1; } } } static int e82545_transmit(struct e82545_softc *sc, uint16_t head, uint16_t tail, uint16_t dsize, uint16_t *rhead, int *tdwb) { uint8_t *hdr, *hdrp; struct iovec iovb[I82545_MAX_TXSEGS + 2]; struct iovec tiov[I82545_MAX_TXSEGS + 2]; struct e1000_context_desc *cd; struct ck_info ckinfo[2]; struct iovec *iov; union e1000_tx_udesc *dsc; int desc, dtype, len, ntype, iovcnt, tlen, hdrlen, vlen, tcp, tso; int mss, paylen, seg, tiovcnt, left, now, nleft, nnow, pv, pvoff; uint32_t tcpsum, tcpseq; uint16_t ipcs, tcpcs, ipid, ohead; ckinfo[0].ck_valid = ckinfo[1].ck_valid = 0; iovcnt = 0; tlen = 0; ntype = 0; tso = 0; ohead = head; /* iovb[0/1] may be used for writable copy of headers. */ iov = &iovb[2]; for (desc = 0; ; desc++, head = (head + 1) % dsize) { if (head == tail) { *rhead = head; return (0); } dsc = &sc->esc_txdesc[head]; dtype = e82545_txdesc_type(dsc->td.lower.data); if (desc == 0) { switch (dtype) { case E1000_TXD_TYP_C: DPRINTF("tx ctxt desc idx %d: %016jx " "%08x%08x\r\n", head, dsc->td.buffer_addr, dsc->td.upper.data, dsc->td.lower.data); /* Save context and return */ sc->esc_txctx = dsc->cd; goto done; case E1000_TXD_TYP_L: DPRINTF("tx legacy desc idx %d: %08x%08x\r\n", head, dsc->td.upper.data, dsc->td.lower.data); /* * legacy cksum start valid in first descriptor */ ntype = dtype; ckinfo[0].ck_start = dsc->td.upper.fields.css; break; case E1000_TXD_TYP_D: DPRINTF("tx data desc idx %d: %08x%08x\r\n", head, dsc->td.upper.data, dsc->td.lower.data); ntype = dtype; break; default: break; } } else { /* Descriptor type must be consistent */ assert(dtype == ntype); DPRINTF("tx next desc idx %d: %08x%08x\r\n", head, dsc->td.upper.data, dsc->td.lower.data); } len = (dtype == E1000_TXD_TYP_L) ? dsc->td.lower.flags.length : dsc->dd.lower.data & 0xFFFFF; if (len > 0) { /* Strip checksum supplied by guest. */ if ((dsc->td.lower.data & E1000_TXD_CMD_EOP) != 0 && (dsc->td.lower.data & E1000_TXD_CMD_IFCS) == 0) len -= 2; tlen += len; if (iovcnt < I82545_MAX_TXSEGS) { iov[iovcnt].iov_base = paddr_guest2host( sc->esc_ctx, dsc->td.buffer_addr, len); iov[iovcnt].iov_len = len; } iovcnt++; } /* * Pull out info that is valid in the final descriptor * and exit descriptor loop. */ if (dsc->td.lower.data & E1000_TXD_CMD_EOP) { if (dtype == E1000_TXD_TYP_L) { if (dsc->td.lower.data & E1000_TXD_CMD_IC) { ckinfo[0].ck_valid = 1; ckinfo[0].ck_off = dsc->td.lower.flags.cso; ckinfo[0].ck_len = 0; } } else { cd = &sc->esc_txctx; if (dsc->dd.lower.data & E1000_TXD_CMD_TSE) tso = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_IXSM) ckinfo[0].ck_valid = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_IXSM || tso) { ckinfo[0].ck_start = cd->lower_setup.ip_fields.ipcss; ckinfo[0].ck_off = cd->lower_setup.ip_fields.ipcso; ckinfo[0].ck_len = cd->lower_setup.ip_fields.ipcse; } if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_TXSM) ckinfo[1].ck_valid = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_TXSM || tso) { ckinfo[1].ck_start = cd->upper_setup.tcp_fields.tucss; ckinfo[1].ck_off = cd->upper_setup.tcp_fields.tucso; ckinfo[1].ck_len = cd->upper_setup.tcp_fields.tucse; } } break; } } if (iovcnt > I82545_MAX_TXSEGS) { WPRINTF("tx too many descriptors (%d > %d) -- dropped\r\n", iovcnt, I82545_MAX_TXSEGS); goto done; } hdrlen = vlen = 0; /* Estimate writable space for VLAN header insertion. */ if ((sc->esc_CTRL & E1000_CTRL_VME) && (dsc->td.lower.data & E1000_TXD_CMD_VLE)) { hdrlen = ETHER_ADDR_LEN*2; vlen = ETHER_VLAN_ENCAP_LEN; } if (!tso) { /* Estimate required writable space for checksums. */ if (ckinfo[0].ck_valid) hdrlen = MAX(hdrlen, ckinfo[0].ck_off + 2); if (ckinfo[1].ck_valid) hdrlen = MAX(hdrlen, ckinfo[1].ck_off + 2); /* Round up writable space to the first vector. */ if (hdrlen != 0 && iov[0].iov_len > hdrlen && iov[0].iov_len < hdrlen + 100) hdrlen = iov[0].iov_len; } else { /* In case of TSO header length provided by software. */ hdrlen = sc->esc_txctx.tcp_seg_setup.fields.hdr_len; } /* Allocate, fill and prepend writable header vector. */ if (hdrlen != 0) { hdr = __builtin_alloca(hdrlen + vlen); hdr += vlen; for (left = hdrlen, hdrp = hdr; left > 0; left -= now, hdrp += now) { now = MIN(left, iov->iov_len); memcpy(hdrp, iov->iov_base, now); iov->iov_base += now; iov->iov_len -= now; if (iov->iov_len == 0) { iov++; iovcnt--; } } iov--; iovcnt++; iov->iov_base = hdr; iov->iov_len = hdrlen; } /* Insert VLAN tag. */ if (vlen != 0) { hdr -= ETHER_VLAN_ENCAP_LEN; memmove(hdr, hdr + ETHER_VLAN_ENCAP_LEN, ETHER_ADDR_LEN*2); hdrlen += ETHER_VLAN_ENCAP_LEN; hdr[ETHER_ADDR_LEN*2 + 0] = sc->esc_VET >> 8; hdr[ETHER_ADDR_LEN*2 + 1] = sc->esc_VET & 0xff; hdr[ETHER_ADDR_LEN*2 + 2] = dsc->td.upper.fields.special >> 8; hdr[ETHER_ADDR_LEN*2 + 3] = dsc->td.upper.fields.special & 0xff; iov->iov_base = hdr; iov->iov_len += ETHER_VLAN_ENCAP_LEN; /* Correct checksum offsets after VLAN tag insertion. */ ckinfo[0].ck_start += ETHER_VLAN_ENCAP_LEN; ckinfo[0].ck_off += ETHER_VLAN_ENCAP_LEN; if (ckinfo[0].ck_len != 0) ckinfo[0].ck_len += ETHER_VLAN_ENCAP_LEN; ckinfo[1].ck_start += ETHER_VLAN_ENCAP_LEN; ckinfo[1].ck_off += ETHER_VLAN_ENCAP_LEN; if (ckinfo[1].ck_len != 0) ckinfo[1].ck_len += ETHER_VLAN_ENCAP_LEN; } /* Simple non-TSO case. */ if (!tso) { /* Calculate checksums and transmit. */ if (ckinfo[0].ck_valid) e82545_transmit_checksum(iov, iovcnt, &ckinfo[0]); if (ckinfo[1].ck_valid) e82545_transmit_checksum(iov, iovcnt, &ckinfo[1]); e82545_transmit_backend(sc, iov, iovcnt); goto done; } /* Doing TSO. */ tcp = (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_TCP) != 0; mss = sc->esc_txctx.tcp_seg_setup.fields.mss; paylen = (sc->esc_txctx.cmd_and_length & 0x000fffff); DPRINTF("tx %s segmentation offload %d+%d/%d bytes %d iovs\r\n", tcp ? "TCP" : "UDP", hdrlen, paylen, mss, iovcnt); ipid = ntohs(*(uint16_t *)&hdr[ckinfo[0].ck_start + 4]); tcpseq = ntohl(*(uint32_t *)&hdr[ckinfo[1].ck_start + 4]); ipcs = *(uint16_t *)&hdr[ckinfo[0].ck_off]; tcpcs = 0; if (ckinfo[1].ck_valid) /* Save partial pseudo-header checksum. */ tcpcs = *(uint16_t *)&hdr[ckinfo[1].ck_off]; pv = 1; pvoff = 0; for (seg = 0, left = paylen; left > 0; seg++, left -= now) { now = MIN(left, mss); /* Construct IOVs for the segment. */ /* Include whole original header. */ tiov[0].iov_base = hdr; tiov[0].iov_len = hdrlen; tiovcnt = 1; /* Include respective part of payload IOV. */ for (nleft = now; pv < iovcnt && nleft > 0; nleft -= nnow) { nnow = MIN(nleft, iov[pv].iov_len - pvoff); tiov[tiovcnt].iov_base = iov[pv].iov_base + pvoff; tiov[tiovcnt++].iov_len = nnow; if (pvoff + nnow == iov[pv].iov_len) { pv++; pvoff = 0; } else pvoff += nnow; } DPRINTF("tx segment %d %d+%d bytes %d iovs\r\n", seg, hdrlen, now, tiovcnt); /* Update IP header. */ if (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_IP) { /* IPv4 -- set length and ID */ *(uint16_t *)&hdr[ckinfo[0].ck_start + 2] = htons(hdrlen - ckinfo[0].ck_start + now); *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = htons(ipid + seg); } else { /* IPv6 -- set length */ *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = htons(hdrlen - ckinfo[0].ck_start - 40 + now); } /* Update pseudo-header checksum. */ tcpsum = tcpcs; tcpsum += htons(hdrlen - ckinfo[1].ck_start + now); /* Update TCP/UDP headers. */ if (tcp) { /* Update sequence number and FIN/PUSH flags. */ *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = htonl(tcpseq + paylen - left); if (now < left) { hdr[ckinfo[1].ck_start + 13] &= ~(TH_FIN | TH_PUSH); } } else { /* Update payload length. */ *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = hdrlen - ckinfo[1].ck_start + now; } /* Calculate checksums and transmit. */ if (ckinfo[0].ck_valid) { *(uint16_t *)&hdr[ckinfo[0].ck_off] = ipcs; e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[0]); } if (ckinfo[1].ck_valid) { *(uint16_t *)&hdr[ckinfo[1].ck_off] = e82545_carry(tcpsum); e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[1]); } e82545_transmit_backend(sc, tiov, tiovcnt); } done: head = (head + 1) % dsize; e82545_transmit_done(sc, ohead, head, dsize, tdwb); *rhead = head; return (desc + 1); } static void e82545_tx_run(struct e82545_softc *sc) { uint32_t cause; uint16_t head, rhead, tail, size; int lim, tdwb, sent; head = sc->esc_TDH; tail = sc->esc_TDT; size = sc->esc_TDLEN / 16; DPRINTF("tx_run: head %x, rhead %x, tail %x\r\n", sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); pthread_mutex_unlock(&sc->esc_mtx); rhead = head; tdwb = 0; for (lim = size / 4; sc->esc_tx_enabled && lim > 0; lim -= sent) { sent = e82545_transmit(sc, head, tail, size, &rhead, &tdwb); if (sent == 0) break; head = rhead; } pthread_mutex_lock(&sc->esc_mtx); sc->esc_TDH = head; sc->esc_TDHr = rhead; cause = 0; if (tdwb) cause |= E1000_ICR_TXDW; if (lim != size / 4 && sc->esc_TDH == sc->esc_TDT) cause |= E1000_ICR_TXQE; if (cause) e82545_icr_assert(sc, cause); DPRINTF("tx_run done: head %x, rhead %x, tail %x\r\n", sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); } static void * e82545_tx_thread(void *param) { struct e82545_softc *sc = param; pthread_mutex_lock(&sc->esc_mtx); for (;;) { while (!sc->esc_tx_enabled || sc->esc_TDHr == sc->esc_TDT) { if (sc->esc_tx_enabled && sc->esc_TDHr != sc->esc_TDT) break; sc->esc_tx_active = 0; if (sc->esc_tx_enabled == 0) pthread_cond_signal(&sc->esc_tx_cond); pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); } sc->esc_tx_active = 1; /* Process some tx descriptors. Lock dropped inside. */ e82545_tx_run(sc); } } static void e82545_tx_start(struct e82545_softc *sc) { if (sc->esc_tx_active == 0) pthread_cond_signal(&sc->esc_tx_cond); } static void e82545_tx_enable(struct e82545_softc *sc) { sc->esc_tx_enabled = 1; } static void e82545_tx_disable(struct e82545_softc *sc) { sc->esc_tx_enabled = 0; while (sc->esc_tx_active) pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); } static void e82545_rx_enable(struct e82545_softc *sc) { sc->esc_rx_enabled = 1; } static void e82545_rx_disable(struct e82545_softc *sc) { sc->esc_rx_enabled = 0; while (sc->esc_rx_active) pthread_cond_wait(&sc->esc_rx_cond, &sc->esc_mtx); } static void e82545_write_ra(struct e82545_softc *sc, int reg, uint32_t wval) { struct eth_uni *eu; int idx; idx = reg >> 1; assert(idx < 15); eu = &sc->esc_uni[idx]; if (reg & 0x1) { /* RAH */ eu->eu_valid = ((wval & E1000_RAH_AV) == E1000_RAH_AV); eu->eu_addrsel = (wval >> 16) & 0x3; eu->eu_eth.octet[5] = wval >> 8; eu->eu_eth.octet[4] = wval; } else { /* RAL */ eu->eu_eth.octet[3] = wval >> 24; eu->eu_eth.octet[2] = wval >> 16; eu->eu_eth.octet[1] = wval >> 8; eu->eu_eth.octet[0] = wval; } } static uint32_t e82545_read_ra(struct e82545_softc *sc, int reg) { struct eth_uni *eu; uint32_t retval; int idx; idx = reg >> 1; assert(idx < 15); eu = &sc->esc_uni[idx]; if (reg & 0x1) { /* RAH */ retval = (eu->eu_valid << 31) | (eu->eu_addrsel << 16) | (eu->eu_eth.octet[5] << 8) | eu->eu_eth.octet[4]; } else { /* RAL */ retval = (eu->eu_eth.octet[3] << 24) | (eu->eu_eth.octet[2] << 16) | (eu->eu_eth.octet[1] << 8) | eu->eu_eth.octet[0]; } return (retval); } static void e82545_write_register(struct e82545_softc *sc, uint32_t offset, uint32_t value) { int ridx; if (offset & 0x3) { DPRINTF("Unaligned register write offset:0x%x value:0x%x\r\n", offset, value); return; } DPRINTF("Register write: 0x%x value: 0x%x\r\n", offset, value); switch (offset) { case E1000_CTRL: case E1000_CTRL_DUP: e82545_devctl(sc, value); break; case E1000_FCAL: sc->esc_FCAL = value; break; case E1000_FCAH: sc->esc_FCAH = value & ~0xFFFF0000; break; case E1000_FCT: sc->esc_FCT = value & ~0xFFFF0000; break; case E1000_VET: sc->esc_VET = value & ~0xFFFF0000; break; case E1000_FCTTV: sc->esc_FCTTV = value & ~0xFFFF0000; break; case E1000_LEDCTL: sc->esc_LEDCTL = value & ~0x30303000; break; case E1000_PBA: sc->esc_PBA = value & 0x0000FF80; break; case E1000_ICR: case E1000_ITR: case E1000_ICS: case E1000_IMS: case E1000_IMC: e82545_intr_write(sc, offset, value); break; case E1000_RCTL: e82545_rx_ctl(sc, value); break; case E1000_FCRTL: sc->esc_FCRTL = value & ~0xFFFF0007; break; case E1000_FCRTH: sc->esc_FCRTH = value & ~0xFFFF0007; break; case E1000_RDBAL(0): sc->esc_RDBAL = value & ~0xF; if (sc->esc_rx_enabled) { /* Apparently legal: update cached address */ e82545_rx_update_rdba(sc); } break; case E1000_RDBAH(0): assert(!sc->esc_rx_enabled); sc->esc_RDBAH = value; break; case E1000_RDLEN(0): assert(!sc->esc_rx_enabled); sc->esc_RDLEN = value & ~0xFFF0007F; break; case E1000_RDH(0): /* XXX should only ever be zero ? Range check ? */ sc->esc_RDH = value; break; case E1000_RDT(0): /* XXX if this opens up the rx ring, do something ? */ sc->esc_RDT = value; break; case E1000_RDTR: /* ignore FPD bit 31 */ sc->esc_RDTR = value & ~0xFFFF0000; break; case E1000_RXDCTL(0): sc->esc_RXDCTL = value & ~0xFEC0C0C0; break; case E1000_RADV: sc->esc_RADV = value & ~0xFFFF0000; break; case E1000_RSRPD: sc->esc_RSRPD = value & ~0xFFFFF000; break; case E1000_RXCSUM: sc->esc_RXCSUM = value & ~0xFFFFF800; break; case E1000_TXCW: sc->esc_TXCW = value & ~0x3FFF0000; break; case E1000_TCTL: e82545_tx_ctl(sc, value); break; case E1000_TIPG: sc->esc_TIPG = value; break; case E1000_AIT: sc->esc_AIT = value; break; case E1000_TDBAL(0): sc->esc_TDBAL = value & ~0xF; if (sc->esc_tx_enabled) { /* Apparently legal */ e82545_tx_update_tdba(sc); } break; case E1000_TDBAH(0): //assert(!sc->esc_tx_enabled); sc->esc_TDBAH = value; break; case E1000_TDLEN(0): //assert(!sc->esc_tx_enabled); sc->esc_TDLEN = value & ~0xFFF0007F; break; case E1000_TDH(0): //assert(!sc->esc_tx_enabled); /* XXX should only ever be zero ? Range check ? */ sc->esc_TDHr = sc->esc_TDH = value; break; case E1000_TDT(0): /* XXX range check ? */ sc->esc_TDT = value; if (sc->esc_tx_enabled) e82545_tx_start(sc); break; case E1000_TIDV: sc->esc_TIDV = value & ~0xFFFF0000; break; case E1000_TXDCTL(0): //assert(!sc->esc_tx_enabled); sc->esc_TXDCTL = value & ~0xC0C0C0; break; case E1000_TADV: sc->esc_TADV = value & ~0xFFFF0000; break; case E1000_RAL(0) ... E1000_RAH(15): /* convert to u32 offset */ ridx = (offset - E1000_RAL(0)) >> 2; e82545_write_ra(sc, ridx, value); break; case E1000_MTA ... (E1000_MTA + (127*4)): sc->esc_fmcast[(offset - E1000_MTA) >> 2] = value; break; case E1000_VFTA ... (E1000_VFTA + (127*4)): sc->esc_fvlan[(offset - E1000_VFTA) >> 2] = value; break; case E1000_EECD: { //DPRINTF("EECD write 0x%x -> 0x%x\r\n", sc->eeprom_control, value); /* edge triggered low->high */ uint32_t eecd_strobe = ((sc->eeprom_control & E1000_EECD_SK) ? 0 : (value & E1000_EECD_SK)); uint32_t eecd_mask = (E1000_EECD_SK|E1000_EECD_CS| E1000_EECD_DI|E1000_EECD_REQ); sc->eeprom_control &= ~eecd_mask; sc->eeprom_control |= (value & eecd_mask); /* grant/revoke immediately */ if (value & E1000_EECD_REQ) { sc->eeprom_control |= E1000_EECD_GNT; } else { sc->eeprom_control &= ~E1000_EECD_GNT; } if (eecd_strobe && (sc->eeprom_control & E1000_EECD_CS)) { e82545_eecd_strobe(sc); } return; } case E1000_MDIC: { uint8_t reg_addr = (uint8_t)((value & E1000_MDIC_REG_MASK) >> E1000_MDIC_REG_SHIFT); uint8_t phy_addr = (uint8_t)((value & E1000_MDIC_PHY_MASK) >> E1000_MDIC_PHY_SHIFT); sc->mdi_control = (value & ~(E1000_MDIC_ERROR|E1000_MDIC_DEST)); if ((value & E1000_MDIC_READY) != 0) { DPRINTF("Incorrect MDIC ready bit: 0x%x\r\n", value); return; } switch (value & E82545_MDIC_OP_MASK) { case E1000_MDIC_OP_READ: sc->mdi_control &= ~E82545_MDIC_DATA_MASK; sc->mdi_control |= e82545_read_mdi(sc, reg_addr, phy_addr); break; case E1000_MDIC_OP_WRITE: e82545_write_mdi(sc, reg_addr, phy_addr, value & E82545_MDIC_DATA_MASK); break; default: DPRINTF("Unknown MDIC op: 0x%x\r\n", value); return; } /* TODO: barrier? */ sc->mdi_control |= E1000_MDIC_READY; if (value & E82545_MDIC_IE) { // TODO: generate interrupt } return; } case E1000_MANC: case E1000_STATUS: return; default: DPRINTF("Unknown write register: 0x%x value:%x\r\n", offset, value); return; } } static uint32_t e82545_read_register(struct e82545_softc *sc, uint32_t offset) { uint32_t retval; int ridx; if (offset & 0x3) { DPRINTF("Unaligned register read offset:0x%x\r\n", offset); return 0; } DPRINTF("Register read: 0x%x\r\n", offset); switch (offset) { case E1000_CTRL: retval = sc->esc_CTRL; break; case E1000_STATUS: retval = E1000_STATUS_FD | E1000_STATUS_LU | E1000_STATUS_SPEED_1000; break; case E1000_FCAL: retval = sc->esc_FCAL; break; case E1000_FCAH: retval = sc->esc_FCAH; break; case E1000_FCT: retval = sc->esc_FCT; break; case E1000_VET: retval = sc->esc_VET; break; case E1000_FCTTV: retval = sc->esc_FCTTV; break; case E1000_LEDCTL: retval = sc->esc_LEDCTL; break; case E1000_PBA: retval = sc->esc_PBA; break; case E1000_ICR: case E1000_ITR: case E1000_ICS: case E1000_IMS: case E1000_IMC: retval = e82545_intr_read(sc, offset); break; case E1000_RCTL: retval = sc->esc_RCTL; break; case E1000_FCRTL: retval = sc->esc_FCRTL; break; case E1000_FCRTH: retval = sc->esc_FCRTH; break; case E1000_RDBAL(0): retval = sc->esc_RDBAL; break; case E1000_RDBAH(0): retval = sc->esc_RDBAH; break; case E1000_RDLEN(0): retval = sc->esc_RDLEN; break; case E1000_RDH(0): retval = sc->esc_RDH; break; case E1000_RDT(0): retval = sc->esc_RDT; break; case E1000_RDTR: retval = sc->esc_RDTR; break; case E1000_RXDCTL(0): retval = sc->esc_RXDCTL; break; case E1000_RADV: retval = sc->esc_RADV; break; case E1000_RSRPD: retval = sc->esc_RSRPD; break; case E1000_RXCSUM: retval = sc->esc_RXCSUM; break; case E1000_TXCW: retval = sc->esc_TXCW; break; case E1000_TCTL: retval = sc->esc_TCTL; break; case E1000_TIPG: retval = sc->esc_TIPG; break; case E1000_AIT: retval = sc->esc_AIT; break; case E1000_TDBAL(0): retval = sc->esc_TDBAL; break; case E1000_TDBAH(0): retval = sc->esc_TDBAH; break; case E1000_TDLEN(0): retval = sc->esc_TDLEN; break; case E1000_TDH(0): retval = sc->esc_TDH; break; case E1000_TDT(0): retval = sc->esc_TDT; break; case E1000_TIDV: retval = sc->esc_TIDV; break; case E1000_TXDCTL(0): retval = sc->esc_TXDCTL; break; case E1000_TADV: retval = sc->esc_TADV; break; case E1000_RAL(0) ... E1000_RAH(15): /* convert to u32 offset */ ridx = (offset - E1000_RAL(0)) >> 2; retval = e82545_read_ra(sc, ridx); break; case E1000_MTA ... (E1000_MTA + (127*4)): retval = sc->esc_fmcast[(offset - E1000_MTA) >> 2]; break; case E1000_VFTA ... (E1000_VFTA + (127*4)): retval = sc->esc_fvlan[(offset - E1000_VFTA) >> 2]; break; case E1000_EECD: //DPRINTF("EECD read %x\r\n", sc->eeprom_control); retval = sc->eeprom_control; break; case E1000_MDIC: retval = sc->mdi_control; break; case E1000_MANC: retval = 0; break; /* stats that we emulate. */ case E1000_MPC: retval = sc->missed_pkt_count; break; case E1000_PRC64: retval = sc->pkt_rx_by_size[0]; break; case E1000_PRC127: retval = sc->pkt_rx_by_size[1]; break; case E1000_PRC255: retval = sc->pkt_rx_by_size[2]; break; case E1000_PRC511: retval = sc->pkt_rx_by_size[3]; break; case E1000_PRC1023: retval = sc->pkt_rx_by_size[4]; break; case E1000_PRC1522: retval = sc->pkt_rx_by_size[5]; break; case E1000_GPRC: retval = sc->good_pkt_rx_count; break; case E1000_BPRC: retval = sc->bcast_pkt_rx_count; break; case E1000_MPRC: retval = sc->mcast_pkt_rx_count; break; case E1000_GPTC: case E1000_TPT: retval = sc->good_pkt_tx_count; break; case E1000_GORCL: retval = (uint32_t)sc->good_octets_rx; break; case E1000_GORCH: retval = (uint32_t)(sc->good_octets_rx >> 32); break; case E1000_TOTL: case E1000_GOTCL: retval = (uint32_t)sc->good_octets_tx; break; case E1000_TOTH: case E1000_GOTCH: retval = (uint32_t)(sc->good_octets_tx >> 32); break; case E1000_ROC: retval = sc->oversize_rx_count; break; case E1000_TORL: retval = (uint32_t)(sc->good_octets_rx + sc->missed_octets); break; case E1000_TORH: retval = (uint32_t)((sc->good_octets_rx + sc->missed_octets) >> 32); break; case E1000_TPR: retval = sc->good_pkt_rx_count + sc->missed_pkt_count + sc->oversize_rx_count; break; case E1000_PTC64: retval = sc->pkt_tx_by_size[0]; break; case E1000_PTC127: retval = sc->pkt_tx_by_size[1]; break; case E1000_PTC255: retval = sc->pkt_tx_by_size[2]; break; case E1000_PTC511: retval = sc->pkt_tx_by_size[3]; break; case E1000_PTC1023: retval = sc->pkt_tx_by_size[4]; break; case E1000_PTC1522: retval = sc->pkt_tx_by_size[5]; break; case E1000_MPTC: retval = sc->mcast_pkt_tx_count; break; case E1000_BPTC: retval = sc->bcast_pkt_tx_count; break; case E1000_TSCTC: retval = sc->tso_tx_count; break; /* stats that are always 0. */ case E1000_CRCERRS: case E1000_ALGNERRC: case E1000_SYMERRS: case E1000_RXERRC: case E1000_SCC: case E1000_ECOL: case E1000_MCC: case E1000_LATECOL: case E1000_COLC: case E1000_DC: case E1000_TNCRS: case E1000_SEC: case E1000_CEXTERR: case E1000_RLEC: case E1000_XONRXC: case E1000_XONTXC: case E1000_XOFFRXC: case E1000_XOFFTXC: case E1000_FCRUC: case E1000_RNBC: case E1000_RUC: case E1000_RFC: case E1000_RJC: case E1000_MGTPRC: case E1000_MGTPDC: case E1000_MGTPTC: case E1000_TSCTFC: retval = 0; break; default: DPRINTF("Unknown read register: 0x%x\r\n", offset); retval = 0; break; } return (retval); } static void e82545_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct e82545_softc *sc; //DPRINTF("Write bar:%d offset:0x%lx value:0x%lx size:%d\r\n", baridx, offset, value, size); sc = pi->pi_arg; pthread_mutex_lock(&sc->esc_mtx); switch (baridx) { case E82545_BAR_IO: switch (offset) { case E82545_IOADDR: if (size != 4) { DPRINTF("Wrong io addr write sz:%d value:0x%lx\r\n", size, value); } else sc->io_addr = (uint32_t)value; break; case E82545_IODATA: if (size != 4) { DPRINTF("Wrong io data write size:%d value:0x%lx\r\n", size, value); } else if (sc->io_addr > E82545_IO_REGISTER_MAX) { DPRINTF("Non-register io write addr:0x%x value:0x%lx\r\n", sc->io_addr, value); } else e82545_write_register(sc, sc->io_addr, (uint32_t)value); break; default: DPRINTF("Unknown io bar write offset:0x%lx value:0x%lx size:%d\r\n", offset, value, size); break; } break; case E82545_BAR_REGISTER: if (size != 4) { DPRINTF("Wrong register write size:%d offset:0x%lx value:0x%lx\r\n", size, offset, value); } else e82545_write_register(sc, (uint32_t)offset, (uint32_t)value); break; default: DPRINTF("Unknown write bar:%d off:0x%lx val:0x%lx size:%d\r\n", baridx, offset, value, size); } pthread_mutex_unlock(&sc->esc_mtx); } static uint64_t e82545_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct e82545_softc *sc; uint64_t retval; //DPRINTF("Read bar:%d offset:0x%lx size:%d\r\n", baridx, offset, size); sc = pi->pi_arg; retval = 0; pthread_mutex_lock(&sc->esc_mtx); switch (baridx) { case E82545_BAR_IO: switch (offset) { case E82545_IOADDR: if (size != 4) { DPRINTF("Wrong io addr read sz:%d\r\n", size); } else retval = sc->io_addr; break; case E82545_IODATA: if (size != 4) { DPRINTF("Wrong io data read sz:%d\r\n", size); } if (sc->io_addr > E82545_IO_REGISTER_MAX) { DPRINTF("Non-register io read addr:0x%x\r\n", sc->io_addr); } else retval = e82545_read_register(sc, sc->io_addr); break; default: DPRINTF("Unknown io bar read offset:0x%lx size:%d\r\n", offset, size); break; } break; case E82545_BAR_REGISTER: if (size != 4) { DPRINTF("Wrong register read size:%d offset:0x%lx\r\n", size, offset); } else retval = e82545_read_register(sc, (uint32_t)offset); break; default: DPRINTF("Unknown read bar:%d offset:0x%lx size:%d\r\n", baridx, offset, size); break; } pthread_mutex_unlock(&sc->esc_mtx); return (retval); } static void e82545_reset(struct e82545_softc *sc, int drvr) { int i; e82545_rx_disable(sc); e82545_tx_disable(sc); /* clear outstanding interrupts */ if (sc->esc_irq_asserted) pci_lintr_deassert(sc->esc_pi); /* misc */ if (!drvr) { sc->esc_FCAL = 0; sc->esc_FCAH = 0; sc->esc_FCT = 0; sc->esc_VET = 0; sc->esc_FCTTV = 0; } sc->esc_LEDCTL = 0x07061302; sc->esc_PBA = 0x00100030; /* start nvm in opcode mode. */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; sc->eeprom_control = E1000_EECD_PRES | E82545_EECD_FWE_EN; e82545_init_eeprom(sc); /* interrupt */ sc->esc_ICR = 0; sc->esc_ITR = 250; sc->esc_ICS = 0; sc->esc_IMS = 0; sc->esc_IMC = 0; /* L2 filters */ if (!drvr) { memset(sc->esc_fvlan, 0, sizeof(sc->esc_fvlan)); memset(sc->esc_fmcast, 0, sizeof(sc->esc_fmcast)); memset(sc->esc_uni, 0, sizeof(sc->esc_uni)); /* XXX not necessary on 82545 ?? */ sc->esc_uni[0].eu_valid = 1; memcpy(sc->esc_uni[0].eu_eth.octet, sc->esc_mac.octet, ETHER_ADDR_LEN); } else { /* Clear RAH valid bits */ for (i = 0; i < 16; i++) sc->esc_uni[i].eu_valid = 0; } /* receive */ if (!drvr) { sc->esc_RDBAL = 0; sc->esc_RDBAH = 0; } sc->esc_RCTL = 0; sc->esc_FCRTL = 0; sc->esc_FCRTH = 0; sc->esc_RDLEN = 0; sc->esc_RDH = 0; sc->esc_RDT = 0; sc->esc_RDTR = 0; sc->esc_RXDCTL = (1 << 24) | (1 << 16); /* default GRAN/WTHRESH */ sc->esc_RADV = 0; sc->esc_RXCSUM = 0; /* transmit */ if (!drvr) { sc->esc_TDBAL = 0; sc->esc_TDBAH = 0; sc->esc_TIPG = 0; sc->esc_AIT = 0; sc->esc_TIDV = 0; sc->esc_TADV = 0; } sc->esc_tdba = 0; sc->esc_txdesc = NULL; sc->esc_TXCW = 0; sc->esc_TCTL = 0; sc->esc_TDLEN = 0; sc->esc_TDT = 0; sc->esc_TDHr = sc->esc_TDH = 0; sc->esc_TXDCTL = 0; } static void e82545_open_tap(struct e82545_softc *sc, char *opts) { char tbuf[80]; if (opts == NULL) { sc->esc_tapfd = -1; return; } strcpy(tbuf, "/dev/"); strlcat(tbuf, opts, sizeof(tbuf)); sc->esc_tapfd = open(tbuf, O_RDWR); if (sc->esc_tapfd == -1) { DPRINTF("unable to open tap device %s\n", opts); exit(1); } /* * Set non-blocking and register for read * notifications with the event loop */ int opt = 1; if (ioctl(sc->esc_tapfd, FIONBIO, &opt) < 0) { WPRINTF("tap device O_NONBLOCK failed: %d\n", errno); close(sc->esc_tapfd); sc->esc_tapfd = -1; } sc->esc_mevp = mevent_add(sc->esc_tapfd, EVF_READ, e82545_tap_callback, sc); if (sc->esc_mevp == NULL) { DPRINTF("Could not register mevent %d\n", EVF_READ); close(sc->esc_tapfd); sc->esc_tapfd = -1; } } static int e82545_parsemac(char *mac_str, uint8_t *mac_addr) { struct ether_addr *ea; char *tmpstr; char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; tmpstr = strsep(&mac_str,"="); if ((mac_str != NULL) && (!strcmp(tmpstr,"mac"))) { ea = ether_aton(mac_str); if (ea == NULL || ETHER_IS_MULTICAST(ea->octet) || memcmp(ea->octet, zero_addr, ETHER_ADDR_LEN) == 0) { fprintf(stderr, "Invalid MAC %s\n", mac_str); return (1); } else memcpy(mac_addr, ea->octet, ETHER_ADDR_LEN); } return (0); } static int e82545_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) { DPRINTF("Loading with options: %s\r\n", opts); MD5_CTX mdctx; unsigned char digest[16]; char nstr[80]; struct e82545_softc *sc; char *devname; char *vtopts; int mac_provided; /* Setup our softc */ - sc = calloc(sizeof(*sc), 1); + sc = calloc(1, sizeof(*sc)); pi->pi_arg = sc; sc->esc_pi = pi; sc->esc_ctx = ctx; pthread_mutex_init(&sc->esc_mtx, NULL); pthread_cond_init(&sc->esc_rx_cond, NULL); pthread_cond_init(&sc->esc_tx_cond, NULL); pthread_create(&sc->esc_tx_tid, NULL, e82545_tx_thread, sc); snprintf(nstr, sizeof(nstr), "e82545-%d:%d tx", pi->pi_slot, pi->pi_func); pthread_set_name_np(sc->esc_tx_tid, nstr); pci_set_cfgdata16(pi, PCIR_DEVICE, E82545_DEV_ID_82545EM_COPPER); pci_set_cfgdata16(pi, PCIR_VENDOR, E82545_VENDOR_ID_INTEL); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_NETWORK_ETHERNET); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, E82545_SUBDEV_ID); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, E82545_VENDOR_ID_INTEL); pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL); pci_set_cfgdata8(pi, PCIR_INTPIN, 0x1); /* TODO: this card also supports msi, but the freebsd driver for it * does not, so I have not implemented it. */ pci_lintr_request(pi); pci_emul_alloc_bar(pi, E82545_BAR_REGISTER, PCIBAR_MEM32, E82545_BAR_REGISTER_LEN); pci_emul_alloc_bar(pi, E82545_BAR_FLASH, PCIBAR_MEM32, E82545_BAR_FLASH_LEN); pci_emul_alloc_bar(pi, E82545_BAR_IO, PCIBAR_IO, E82545_BAR_IO_LEN); /* * Attempt to open the tap device and read the MAC address * if specified. Copied from virtio-net, slightly modified. */ mac_provided = 0; sc->esc_tapfd = -1; if (opts != NULL) { int err; devname = vtopts = strdup(opts); (void) strsep(&vtopts, ","); if (vtopts != NULL) { err = e82545_parsemac(vtopts, sc->esc_mac.octet); if (err != 0) { free(devname); return (err); } mac_provided = 1; } if (strncmp(devname, "tap", 3) == 0 || strncmp(devname, "vmnet", 5) == 0) e82545_open_tap(sc, devname); free(devname); } /* * The default MAC address is the standard NetApp OUI of 00-a0-98, * followed by an MD5 of the PCI slot/func number and dev name */ if (!mac_provided) { snprintf(nstr, sizeof(nstr), "%d-%d-%s", pi->pi_slot, pi->pi_func, vmname); MD5Init(&mdctx); MD5Update(&mdctx, nstr, strlen(nstr)); MD5Final(digest, &mdctx); sc->esc_mac.octet[0] = 0x00; sc->esc_mac.octet[1] = 0xa0; sc->esc_mac.octet[2] = 0x98; sc->esc_mac.octet[3] = digest[0]; sc->esc_mac.octet[4] = digest[1]; sc->esc_mac.octet[5] = digest[2]; } /* H/w initiated reset */ e82545_reset(sc, 0); return (0); } struct pci_devemu pci_de_e82545 = { .pe_emu = "e1000", .pe_init = e82545_init, .pe_barwrite = e82545_write, .pe_barread = e82545_read }; PCI_EMUL_SET(pci_de_e82545); Index: projects/clang390-import/usr.sbin/devctl/devctl.8 =================================================================== --- projects/clang390-import/usr.sbin/devctl/devctl.8 (revision 305080) +++ projects/clang390-import/usr.sbin/devctl/devctl.8 (revision 305081) @@ -1,159 +1,178 @@ .\" .\" Copyright (c) 2015 John Baldwin .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd April 27, 2016 +.Dd August 29, 2016 .Dt DEVCTL 8 .Os .Sh NAME .Nm devctl .Nd device control utility .Sh SYNOPSIS .Nm .Cm attach .Ar device .Nm +.Cm clear driver +.Op Fl f +.Ar device +.Nm .Cm detach .Op Fl f .Ar device .Nm .Cm disable .Op Fl f .Ar device .Nm .Cm enable .Ar device .Nm .Cm suspend .Ar device .Nm .Cm resume .Ar device .Nm .Cm set driver .Op Fl f .Ar device driver .Nm .Cm rescan .Ar device .Nm .Cm delete .Op Fl f .Ar device .Sh DESCRIPTION The .Nm utility adjusts the state of individual devices in the kernel's internal device hierarchy. Each invocation of .Nm consists of a single command followed by command-specific arguments. Each command operates on a single device specified via the .Ar device argument. The .Ar device may be specified either as the name of an existing device or as a bus-specific address. More details on supported address formats can be found in .Xr devctl 3 . .Pp The following commands are supported: .Bl -tag -width indent .It Cm attach Ar device Force the kernel to re-probe the device. If a suitable driver is found, it is attached to the device. .It Xo Cm detach .Op Fl f .Ar device .Xc Detach the device from its current device driver. If the .Fl f flag is specified, the device driver will be detached even if the device is busy. .It Xo Cm disable .Op Fl f .Ar device .Xc Disable a device. If the device is currently attached to a device driver, the device driver will be detached from the device, but the device will retain its current name. If the .Fl f flag is specified, the device driver will be detached even if the device is busy. .It Cm enable Ar device Enable a device. The device will probe and attach if a suitable device driver is found. Note that this can re-enable a device disabled at boot time via a loader tunable. .It Cm suspend Ar device Suspend a device. This may include placing the device in a reduced power state. .It Cm resume Ar device Resume a suspended device to a fully working state. .It Xo Cm set driver .Op Fl f .Ar device driver .Xc Force the device to use a device driver named .Ar driver . If the device is already attached to a device driver and the .Fl f flag is specified, the device will be detached from its current device driver before it is attached to the new device driver. If the device is already attached to a device driver and the +.Fl f +flag is not specified, +the device will not be changed. +.It Xo Cm clear driver +.Op Fl f +.Ar device +.Xc +Clear a previously-forced driver name so that the device is able to use any +valid device driver. +After the previous name has been cleared, +the device is reprobed so that other device drivers may attach to it. +This can be used to undo an earlier +.Cm set driver +command. +If the device is currently attached to a device driver and the .Fl f flag is not specified, the device will not be changed. .It Cm rescan Ar device Rescan a bus device checking for devices that have been added or removed. .It Xo Cm delete .Op Fl .Ar device .Xc Delete the device from the device tree. If the .Fl f flag is specified, the device will be deleted even if it is physically present. This command should be used with care as a device that is deleted but present can no longer be used unless the parent bus device rediscovers the device via a rescan request. .El .Sh SEE ALSO .Xr devctl 3 , .Xr devinfo 8 .Sh HISTORY The .Nm utility first appeared in .Fx 10.3 . Index: projects/clang390-import/usr.sbin/devctl/devctl.c =================================================================== --- projects/clang390-import/usr.sbin/devctl/devctl.c (revision 305080) +++ projects/clang390-import/usr.sbin/devctl/devctl.c (revision 305081) @@ -1,330 +1,366 @@ /*- * Copyright (c) 2014 John Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include struct devctl_command { const char *name; int (*handler)(int ac, char **av); }; #define DEVCTL_DATASET(name) devctl_ ## name ## _table #define DEVCTL_COMMAND(set, name, function) \ static struct devctl_command function ## _devctl_command = \ { #name, function }; \ DATA_SET(DEVCTL_DATASET(set), function ## _devctl_command) #define DEVCTL_TABLE(set, name) \ SET_DECLARE(DEVCTL_DATASET(name), struct devctl_command); \ \ static int \ devctl_ ## name ## _table_handler(int ac, char **av) \ { \ return (devctl_table_handler(SET_BEGIN(DEVCTL_DATASET(name)), \ SET_LIMIT(DEVCTL_DATASET(name)), ac, av)); \ } \ DEVCTL_COMMAND(set, name, devctl_ ## name ## _table_handler) static int devctl_table_handler(struct devctl_command **start, struct devctl_command **end, int ac, char **av); SET_DECLARE(DEVCTL_DATASET(top), struct devctl_command); +DEVCTL_TABLE(top, clear); DEVCTL_TABLE(top, set); static void usage(void) { - fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", + fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", "usage: devctl attach device", " devctl detach [-f] device", " devctl disable [-f] device", " devctl enable device", " devctl suspend device", " devctl resume device", " devctl set driver [-f] device driver", + " devctl clear driver [-f] device", " devctl rescan device", " devctl delete [-f] device"); exit(1); } static int devctl_table_handler(struct devctl_command **start, struct devctl_command **end, int ac, char **av) { struct devctl_command **cmd; if (ac < 2) { warnx("The %s command requires a sub-command.", av[0]); return (EINVAL); } for (cmd = start; cmd < end; cmd++) { if (strcmp((*cmd)->name, av[1]) == 0) return ((*cmd)->handler(ac - 1, av + 1)); } warnx("%s is not a valid sub-command of %s.", av[1], av[0]); return (ENOENT); } static int help(int ac __unused, char **av __unused) { usage(); return (0); } DEVCTL_COMMAND(top, help, help); static int attach(int ac, char **av) { if (ac != 2) usage(); if (devctl_attach(av[1]) < 0) err(1, "Failed to attach %s", av[1]); return (0); } DEVCTL_COMMAND(top, attach, attach); static void detach_usage(void) { fprintf(stderr, "usage: devctl detach [-f] device\n"); exit(1); } static int detach(int ac, char **av) { bool force; int ch; force = false; while ((ch = getopt(ac, av, "f")) != -1) switch (ch) { case 'f': force = true; break; default: detach_usage(); } ac -= optind; av += optind; if (ac != 1) detach_usage(); if (devctl_detach(av[0], force) < 0) err(1, "Failed to detach %s", av[0]); return (0); } DEVCTL_COMMAND(top, detach, detach); static void disable_usage(void) { fprintf(stderr, "usage: devctl disable [-f] device\n"); exit(1); } static int disable(int ac, char **av) { bool force; int ch; force = false; while ((ch = getopt(ac, av, "f")) != -1) switch (ch) { case 'f': force = true; break; default: disable_usage(); } ac -= optind; av += optind; if (ac != 1) disable_usage(); if (devctl_disable(av[0], force) < 0) err(1, "Failed to disable %s", av[0]); return (0); } DEVCTL_COMMAND(top, disable, disable); static int enable(int ac, char **av) { if (ac != 2) usage(); if (devctl_enable(av[1]) < 0) err(1, "Failed to enable %s", av[1]); return (0); } DEVCTL_COMMAND(top, enable, enable); static int suspend(int ac, char **av) { if (ac != 2) usage(); if (devctl_suspend(av[1]) < 0) err(1, "Failed to suspend %s", av[1]); return (0); } DEVCTL_COMMAND(top, suspend, suspend); static int resume(int ac, char **av) { if (ac != 2) usage(); if (devctl_resume(av[1]) < 0) err(1, "Failed to resume %s", av[1]); return (0); } DEVCTL_COMMAND(top, resume, resume); static void set_driver_usage(void) { fprintf(stderr, "usage: devctl set driver [-f] device driver\n"); exit(1); } static int set_driver(int ac, char **av) { bool force; int ch; force = false; while ((ch = getopt(ac, av, "f")) != -1) switch (ch) { case 'f': force = true; break; default: set_driver_usage(); } ac -= optind; av += optind; if (ac != 2) set_driver_usage(); if (devctl_set_driver(av[0], av[1], force) < 0) err(1, "Failed to set %s driver to %s", av[0], av[1]); return (0); } DEVCTL_COMMAND(set, driver, set_driver); + +static void +clear_driver_usage(void) +{ + + fprintf(stderr, "usage: devctl clear driver [-f] device\n"); + exit(1); +} + +static int +clear_driver(int ac, char **av) +{ + bool force; + int ch; + + force = false; + while ((ch = getopt(ac, av, "f")) != -1) + switch (ch) { + case 'f': + force = true; + break; + default: + clear_driver_usage(); + } + ac -= optind; + av += optind; + + if (ac != 1) + clear_driver_usage(); + if (devctl_clear_driver(av[0], force) < 0) + err(1, "Failed to clear %s driver", av[0]); + return (0); +} +DEVCTL_COMMAND(clear, driver, clear_driver); static int rescan(int ac, char **av) { if (ac != 2) usage(); if (devctl_rescan(av[1]) < 0) err(1, "Failed to rescan %s", av[1]); return (0); } DEVCTL_COMMAND(top, rescan, rescan); static void delete_usage(void) { fprintf(stderr, "usage: devctl delete [-f] device\n"); exit(1); } static int delete(int ac, char **av) { bool force; int ch; force = false; while ((ch = getopt(ac, av, "f")) != -1) switch (ch) { case 'f': force = true; break; default: delete_usage(); } ac -= optind; av += optind; if (ac != 1) delete_usage(); if (devctl_delete(av[0], force) < 0) err(1, "Failed to delete %s", av[0]); return (0); } DEVCTL_COMMAND(top, delete, delete); int main(int ac, char *av[]) { struct devctl_command **cmd; if (ac == 1) usage(); ac--; av++; SET_FOREACH(cmd, DEVCTL_DATASET(top)) { if (strcmp((*cmd)->name, av[0]) == 0) { if ((*cmd)->handler(ac, av) != 0) return (1); else return (0); } } warnx("Unknown command %s.", av[0]); return (1); } Index: projects/clang390-import =================================================================== --- projects/clang390-import (revision 305080) +++ projects/clang390-import (revision 305081) Property changes on: projects/clang390-import ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,2 ## Merged /projects/netbsd-tests-update-12:r305030 Merged /head:r305029-305080