Index: head/sys/compat/ndis/kern_ndis.c =================================================================== --- head/sys/compat/ndis/kern_ndis.c (revision 214797) +++ head/sys/compat/ndis/kern_ndis.c (revision 214798) @@ -1,1438 +1,1414 @@ /*- * Copyright (c) 2003 * Bill Paul . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Bill Paul. * 4. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define NDIS_DUMMY_PATH "\\\\some\\bogus\\path" static void ndis_status_func(ndis_handle, ndis_status, void *, uint32_t); static void ndis_statusdone_func(ndis_handle); static void ndis_setdone_func(ndis_handle, ndis_status); static void ndis_getdone_func(ndis_handle, ndis_status); static void ndis_resetdone_func(ndis_handle, ndis_status, uint8_t); static void ndis_sendrsrcavail_func(ndis_handle); static void ndis_intrsetup(kdpc *, device_object *, irp *, struct ndis_softc *); static void ndis_return(device_object *, void *); static image_patch_table kernndis_functbl[] = { IMPORT_SFUNC(ndis_status_func, 4), IMPORT_SFUNC(ndis_statusdone_func, 1), IMPORT_SFUNC(ndis_setdone_func, 2), IMPORT_SFUNC(ndis_getdone_func, 2), IMPORT_SFUNC(ndis_resetdone_func, 3), IMPORT_SFUNC(ndis_sendrsrcavail_func, 1), IMPORT_SFUNC(ndis_intrsetup, 4), IMPORT_SFUNC(ndis_return, 1), { NULL, NULL, NULL } }; static struct nd_head ndis_devhead; /* * This allows us to export our symbols to other modules. * Note that we call ourselves 'ndisapi' to avoid a namespace * collision with if_ndis.ko, which internally calls itself * 'ndis.' * * Note: some of the subsystems depend on each other, so the * order in which they're started is important. The order of * importance is: * * HAL - spinlocks and IRQL manipulation * ntoskrnl - DPC and workitem threads, object waiting * windrv - driver/device registration * * The HAL should also be the last thing shut down, since * the ntoskrnl subsystem will use spinlocks right up until * the DPC and workitem threads are terminated. */ static int ndis_modevent(module_t mod, int cmd, void *arg) { int error = 0; image_patch_table *patch; switch (cmd) { case MOD_LOAD: /* Initialize subsystems */ hal_libinit(); ntoskrnl_libinit(); windrv_libinit(); ndis_libinit(); usbd_libinit(); patch = kernndis_functbl; while (patch->ipt_func != NULL) { windrv_wrap((funcptr)patch->ipt_func, (funcptr *)&patch->ipt_wrap, patch->ipt_argcnt, patch->ipt_ftype); patch++; } TAILQ_INIT(&ndis_devhead); break; case MOD_SHUTDOWN: if (TAILQ_FIRST(&ndis_devhead) == NULL) { /* Shut down subsystems */ ndis_libfini(); usbd_libfini(); windrv_libfini(); ntoskrnl_libfini(); hal_libfini(); patch = kernndis_functbl; while (patch->ipt_func != NULL) { windrv_unwrap(patch->ipt_wrap); patch++; } } break; case MOD_UNLOAD: /* Shut down subsystems */ ndis_libfini(); usbd_libfini(); windrv_libfini(); ntoskrnl_libfini(); hal_libfini(); patch = kernndis_functbl; while (patch->ipt_func != NULL) { windrv_unwrap(patch->ipt_wrap); patch++; } break; default: error = EINVAL; break; } return (error); } DEV_MODULE(ndisapi, ndis_modevent, NULL); MODULE_VERSION(ndisapi, 1); static void ndis_sendrsrcavail_func(adapter) ndis_handle adapter; { } static void ndis_status_func(adapter, status, sbuf, slen) ndis_handle adapter; ndis_status status; void *sbuf; uint32_t slen; { ndis_miniport_block *block; struct ndis_softc *sc; struct ifnet *ifp; block = adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); ifp = sc->ifp; if (ifp->if_flags & IFF_DEBUG) device_printf(sc->ndis_dev, "status: %x\n", status); } static void ndis_statusdone_func(adapter) ndis_handle adapter; { ndis_miniport_block *block; struct ndis_softc *sc; struct ifnet *ifp; block = adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); ifp = sc->ifp; if (ifp->if_flags & IFF_DEBUG) device_printf(sc->ndis_dev, "status complete\n"); } static void ndis_setdone_func(adapter, status) ndis_handle adapter; ndis_status status; { ndis_miniport_block *block; block = adapter; block->nmb_setstat = status; KeSetEvent(&block->nmb_setevent, IO_NO_INCREMENT, FALSE); } static void ndis_getdone_func(adapter, status) ndis_handle adapter; ndis_status status; { ndis_miniport_block *block; block = adapter; block->nmb_getstat = status; KeSetEvent(&block->nmb_getevent, IO_NO_INCREMENT, FALSE); } static void ndis_resetdone_func(ndis_handle adapter, ndis_status status, uint8_t addressingreset) { ndis_miniport_block *block; struct ndis_softc *sc; struct ifnet *ifp; block = adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); ifp = sc->ifp; if (ifp->if_flags & IFF_DEBUG) device_printf(sc->ndis_dev, "reset done...\n"); KeSetEvent(&block->nmb_resetevent, IO_NO_INCREMENT, FALSE); } int ndis_create_sysctls(arg) void *arg; { struct ndis_softc *sc; ndis_cfg *vals; char buf[256]; struct sysctl_oid *oidp; struct sysctl_ctx_entry *e; if (arg == NULL) return (EINVAL); sc = arg; vals = sc->ndis_regvals; TAILQ_INIT(&sc->ndis_cfglist_head); -#if __FreeBSD_version < 502113 - /* Create the sysctl tree. */ - - sc->ndis_tree = SYSCTL_ADD_NODE(&sc->ndis_ctx, - SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, - device_get_nameunit(sc->ndis_dev), CTLFLAG_RD, 0, - device_get_desc(sc->ndis_dev)); - -#endif /* Add the driver-specific registry keys. */ while(1) { if (vals->nc_cfgkey == NULL) break; if (vals->nc_idx != sc->ndis_devidx) { vals++; continue; } /* See if we already have a sysctl with this name */ oidp = NULL; -#if __FreeBSD_version < 502113 - TAILQ_FOREACH(e, &sc->ndis_ctx, link) { -#else TAILQ_FOREACH(e, device_get_sysctl_ctx(sc->ndis_dev), link) { -#endif oidp = e->entry; if (strcasecmp(oidp->oid_name, vals->nc_cfgkey) == 0) break; oidp = NULL; } if (oidp != NULL) { vals++; continue; } ndis_add_sysctl(sc, vals->nc_cfgkey, vals->nc_cfgdesc, vals->nc_val, CTLFLAG_RW); vals++; } /* Now add a couple of builtin keys. */ /* * Environment can be either Windows (0) or WindowsNT (1). * We qualify as the latter. */ ndis_add_sysctl(sc, "Environment", "Windows environment", "1", CTLFLAG_RD); /* NDIS version should be 5.1. */ ndis_add_sysctl(sc, "NdisVersion", "NDIS API Version", "0x00050001", CTLFLAG_RD); /* Bus type (PCI, PCMCIA, etc...) */ sprintf(buf, "%d", (int)sc->ndis_iftype); ndis_add_sysctl(sc, "BusType", "Bus Type", buf, CTLFLAG_RD); if (sc->ndis_res_io != NULL) { sprintf(buf, "0x%lx", rman_get_start(sc->ndis_res_io)); ndis_add_sysctl(sc, "IOBaseAddress", "Base I/O Address", buf, CTLFLAG_RD); } if (sc->ndis_irq != NULL) { sprintf(buf, "%lu", rman_get_start(sc->ndis_irq)); ndis_add_sysctl(sc, "InterruptNumber", "Interrupt Number", buf, CTLFLAG_RD); } return (0); } int ndis_add_sysctl(arg, key, desc, val, flag) void *arg; char *key; char *desc; char *val; int flag; { struct ndis_softc *sc; struct ndis_cfglist *cfg; char descstr[256]; sc = arg; cfg = malloc(sizeof(struct ndis_cfglist), M_DEVBUF, M_NOWAIT|M_ZERO); if (cfg == NULL) { printf("failed for %s\n", key); return (ENOMEM); } cfg->ndis_cfg.nc_cfgkey = strdup(key, M_DEVBUF); if (desc == NULL) { snprintf(descstr, sizeof(descstr), "%s (dynamic)", key); cfg->ndis_cfg.nc_cfgdesc = strdup(descstr, M_DEVBUF); } else cfg->ndis_cfg.nc_cfgdesc = strdup(desc, M_DEVBUF); strcpy(cfg->ndis_cfg.nc_val, val); TAILQ_INSERT_TAIL(&sc->ndis_cfglist_head, cfg, link); cfg->ndis_oid = -#if __FreeBSD_version < 502113 - SYSCTL_ADD_STRING(&sc->ndis_ctx, SYSCTL_CHILDREN(sc->ndis_tree), - OID_AUTO, cfg->ndis_cfg.nc_cfgkey, flag, - cfg->ndis_cfg.nc_val, sizeof(cfg->ndis_cfg.nc_val), - cfg->ndis_cfg.nc_cfgdesc); -#else SYSCTL_ADD_STRING(device_get_sysctl_ctx(sc->ndis_dev), SYSCTL_CHILDREN(device_get_sysctl_tree(sc->ndis_dev)), OID_AUTO, cfg->ndis_cfg.nc_cfgkey, flag, cfg->ndis_cfg.nc_val, sizeof(cfg->ndis_cfg.nc_val), cfg->ndis_cfg.nc_cfgdesc); -#endif return (0); } /* * Somewhere, somebody decided "hey, let's automatically create * a sysctl tree for each device instance as it's created -- it'll * make life so much easier!" Lies. Why must they turn the kernel * into a house of lies? */ int ndis_flush_sysctls(arg) void *arg; { struct ndis_softc *sc; struct ndis_cfglist *cfg; struct sysctl_ctx_list *clist; sc = arg; -#if __FreeBSD_version < 502113 - clist = &sc->ndis_ctx; -#else clist = device_get_sysctl_ctx(sc->ndis_dev); -#endif while (!TAILQ_EMPTY(&sc->ndis_cfglist_head)) { cfg = TAILQ_FIRST(&sc->ndis_cfglist_head); TAILQ_REMOVE(&sc->ndis_cfglist_head, cfg, link); sysctl_ctx_entry_del(clist, cfg->ndis_oid); sysctl_remove_oid(cfg->ndis_oid, 1, 0); free(cfg->ndis_cfg.nc_cfgkey, M_DEVBUF); free(cfg->ndis_cfg.nc_cfgdesc, M_DEVBUF); free(cfg, M_DEVBUF); } return (0); } static void ndis_return(dobj, arg) device_object *dobj; void *arg; { ndis_miniport_block *block; ndis_miniport_characteristics *ch; ndis_return_handler returnfunc; ndis_handle adapter; ndis_packet *p; uint8_t irql; list_entry *l; block = arg; ch = IoGetDriverObjectExtension(dobj->do_drvobj, (void *)1); p = arg; adapter = block->nmb_miniportadapterctx; if (adapter == NULL) return; returnfunc = ch->nmc_return_packet_func; KeAcquireSpinLock(&block->nmb_returnlock, &irql); while (!IsListEmpty(&block->nmb_returnlist)) { l = RemoveHeadList((&block->nmb_returnlist)); p = CONTAINING_RECORD(l, ndis_packet, np_list); InitializeListHead((&p->np_list)); KeReleaseSpinLock(&block->nmb_returnlock, irql); MSCALL2(returnfunc, adapter, p); KeAcquireSpinLock(&block->nmb_returnlock, &irql); } KeReleaseSpinLock(&block->nmb_returnlock, irql); } void ndis_return_packet(buf, arg) void *buf; /* not used */ void *arg; { ndis_packet *p; ndis_miniport_block *block; if (arg == NULL) return; p = arg; /* Decrement refcount. */ p->np_refcnt--; /* Release packet when refcount hits zero, otherwise return. */ if (p->np_refcnt) return; block = ((struct ndis_softc *)p->np_softc)->ndis_block; KeAcquireSpinLockAtDpcLevel(&block->nmb_returnlock); InitializeListHead((&p->np_list)); InsertHeadList((&block->nmb_returnlist), (&p->np_list)); KeReleaseSpinLockFromDpcLevel(&block->nmb_returnlock); IoQueueWorkItem(block->nmb_returnitem, (io_workitem_func)kernndis_functbl[7].ipt_wrap, WORKQUEUE_CRITICAL, block); } void ndis_free_bufs(b0) ndis_buffer *b0; { ndis_buffer *next; if (b0 == NULL) return; while(b0 != NULL) { next = b0->mdl_next; IoFreeMdl(b0); b0 = next; } } void ndis_free_packet(p) ndis_packet *p; { if (p == NULL) return; ndis_free_bufs(p->np_private.npp_head); NdisFreePacket(p); } int ndis_convert_res(arg) void *arg; { struct ndis_softc *sc; ndis_resource_list *rl = NULL; cm_partial_resource_desc *prd = NULL; ndis_miniport_block *block; device_t dev; struct resource_list *brl; struct resource_list_entry *brle; int error = 0; sc = arg; block = sc->ndis_block; dev = sc->ndis_dev; rl = malloc(sizeof(ndis_resource_list) + (sizeof(cm_partial_resource_desc) * (sc->ndis_rescnt - 1)), M_DEVBUF, M_NOWAIT|M_ZERO); if (rl == NULL) return (ENOMEM); rl->cprl_version = 5; rl->cprl_version = 1; rl->cprl_count = sc->ndis_rescnt; prd = rl->cprl_partial_descs; brl = BUS_GET_RESOURCE_LIST(dev, dev); if (brl != NULL) { STAILQ_FOREACH(brle, brl, link) { switch (brle->type) { case SYS_RES_IOPORT: prd->cprd_type = CmResourceTypePort; prd->cprd_flags = CM_RESOURCE_PORT_IO; prd->cprd_sharedisp = CmResourceShareDeviceExclusive; prd->u.cprd_port.cprd_start.np_quad = brle->start; prd->u.cprd_port.cprd_len = brle->count; break; case SYS_RES_MEMORY: prd->cprd_type = CmResourceTypeMemory; prd->cprd_flags = CM_RESOURCE_MEMORY_READ_WRITE; prd->cprd_sharedisp = CmResourceShareDeviceExclusive; prd->u.cprd_port.cprd_start.np_quad = brle->start; prd->u.cprd_port.cprd_len = brle->count; break; case SYS_RES_IRQ: prd->cprd_type = CmResourceTypeInterrupt; prd->cprd_flags = 0; /* * Always mark interrupt resources as * shared, since in our implementation, * they will be. */ prd->cprd_sharedisp = CmResourceShareShared; prd->u.cprd_intr.cprd_level = brle->start; prd->u.cprd_intr.cprd_vector = brle->start; prd->u.cprd_intr.cprd_affinity = 0; break; default: break; } prd++; } } block->nmb_rlist = rl; return (error); } /* * Map an NDIS packet to an mbuf list. When an NDIS driver receives a * packet, it will hand it to us in the form of an ndis_packet, * which we need to convert to an mbuf that is then handed off * to the stack. Note: we configure the mbuf list so that it uses * the memory regions specified by the ndis_buffer structures in * the ndis_packet as external storage. In most cases, this will * point to a memory region allocated by the driver (either by * ndis_malloc_withtag() or ndis_alloc_sharedmem()). We expect * the driver to handle free()ing this region for is, so we set up * a dummy no-op free handler for it. */ int ndis_ptom(m0, p) struct mbuf **m0; ndis_packet *p; { struct mbuf *m = NULL, *prev = NULL; ndis_buffer *buf; ndis_packet_private *priv; uint32_t totlen = 0; struct ifnet *ifp; struct ether_header *eh; int diff; if (p == NULL || m0 == NULL) return (EINVAL); priv = &p->np_private; buf = priv->npp_head; p->np_refcnt = 0; for (buf = priv->npp_head; buf != NULL; buf = buf->mdl_next) { if (buf == priv->npp_head) #ifdef MT_HEADER MGETHDR(m, M_DONTWAIT, MT_HEADER); #else MGETHDR(m, M_DONTWAIT, MT_DATA); #endif else MGET(m, M_DONTWAIT, MT_DATA); if (m == NULL) { m_freem(*m0); *m0 = NULL; return (ENOBUFS); } m->m_len = MmGetMdlByteCount(buf); m->m_data = MmGetMdlVirtualAddress(buf); MEXTADD(m, m->m_data, m->m_len, ndis_return_packet, m->m_data, p, 0, EXT_NDIS); p->np_refcnt++; totlen += m->m_len; if (m->m_flags & M_PKTHDR) *m0 = m; else prev->m_next = m; prev = m; } /* * This is a hack to deal with the Marvell 8335 driver * which, when associated with an AP in WPA-PSK mode, * seems to overpad its frames by 8 bytes. I don't know * that the extra 8 bytes are for, and they're not there * in open mode, so for now clamp the frame size at 1514 * until I can figure out how to deal with this properly, * otherwise if_ethersubr() will spank us by discarding * the 'oversize' frames. */ eh = mtod((*m0), struct ether_header *); ifp = ((struct ndis_softc *)p->np_softc)->ifp; if (totlen > ETHER_MAX_FRAME(ifp, eh->ether_type, FALSE)) { diff = totlen - ETHER_MAX_FRAME(ifp, eh->ether_type, FALSE); totlen -= diff; m->m_len -= diff; } (*m0)->m_pkthdr.len = totlen; return (0); } /* * Create an NDIS packet from an mbuf chain. * This is used mainly when transmitting packets, where we need * to turn an mbuf off an interface's send queue and transform it * into an NDIS packet which will be fed into the NDIS driver's * send routine. * * NDIS packets consist of two parts: an ndis_packet structure, * which is vaguely analagous to the pkthdr portion of an mbuf, * and one or more ndis_buffer structures, which define the * actual memory segments in which the packet data resides. * We need to allocate one ndis_buffer for each mbuf in a chain, * plus one ndis_packet as the header. */ int ndis_mtop(m0, p) struct mbuf *m0; ndis_packet **p; { struct mbuf *m; ndis_buffer *buf = NULL, *prev = NULL; ndis_packet_private *priv; if (p == NULL || *p == NULL || m0 == NULL) return (EINVAL); priv = &(*p)->np_private; priv->npp_totlen = m0->m_pkthdr.len; for (m = m0; m != NULL; m = m->m_next) { if (m->m_len == 0) continue; buf = IoAllocateMdl(m->m_data, m->m_len, FALSE, FALSE, NULL); if (buf == NULL) { ndis_free_packet(*p); *p = NULL; return (ENOMEM); } MmBuildMdlForNonPagedPool(buf); if (priv->npp_head == NULL) priv->npp_head = buf; else prev->mdl_next = buf; prev = buf; } priv->npp_tail = buf; return (0); } int ndis_get_supported_oids(arg, oids, oidcnt) void *arg; ndis_oid **oids; int *oidcnt; { int len, rval; ndis_oid *o; if (arg == NULL || oids == NULL || oidcnt == NULL) return (EINVAL); len = 0; ndis_get_info(arg, OID_GEN_SUPPORTED_LIST, NULL, &len); o = malloc(len, M_DEVBUF, M_NOWAIT); if (o == NULL) return (ENOMEM); rval = ndis_get_info(arg, OID_GEN_SUPPORTED_LIST, o, &len); if (rval) { free(o, M_DEVBUF); return (rval); } *oids = o; *oidcnt = len / 4; return (0); } int ndis_set_info(arg, oid, buf, buflen) void *arg; ndis_oid oid; void *buf; int *buflen; { struct ndis_softc *sc; ndis_status rval; ndis_handle adapter; ndis_setinfo_handler setfunc; uint32_t byteswritten = 0, bytesneeded = 0; uint8_t irql; uint64_t duetime; /* * According to the NDIS spec, MiniportQueryInformation() * and MiniportSetInformation() requests are handled serially: * once one request has been issued, we must wait for it to * finish before allowing another request to proceed. */ sc = arg; KeResetEvent(&sc->ndis_block->nmb_setevent); KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); if (sc->ndis_block->nmb_pendingreq != NULL) { KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); panic("ndis_set_info() called while other request pending"); } else sc->ndis_block->nmb_pendingreq = (ndis_request *)sc; setfunc = sc->ndis_chars->nmc_setinfo_func; adapter = sc->ndis_block->nmb_miniportadapterctx; if (adapter == NULL || setfunc == NULL || sc->ndis_block->nmb_devicectx == NULL) { sc->ndis_block->nmb_pendingreq = NULL; KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); return (ENXIO); } rval = MSCALL6(setfunc, adapter, oid, buf, *buflen, &byteswritten, &bytesneeded); sc->ndis_block->nmb_pendingreq = NULL; KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); if (rval == NDIS_STATUS_PENDING) { /* Wait up to 5 seconds. */ duetime = (5 * 1000000) * -10; KeWaitForSingleObject(&sc->ndis_block->nmb_setevent, 0, 0, FALSE, &duetime); rval = sc->ndis_block->nmb_setstat; } if (byteswritten) *buflen = byteswritten; if (bytesneeded) *buflen = bytesneeded; if (rval == NDIS_STATUS_INVALID_LENGTH) return (ENOSPC); if (rval == NDIS_STATUS_INVALID_OID) return (EINVAL); if (rval == NDIS_STATUS_NOT_SUPPORTED || rval == NDIS_STATUS_NOT_ACCEPTED) return (ENOTSUP); if (rval != NDIS_STATUS_SUCCESS) return (ENODEV); return (0); } typedef void (*ndis_senddone_func)(ndis_handle, ndis_packet *, ndis_status); int ndis_send_packets(arg, packets, cnt) void *arg; ndis_packet **packets; int cnt; { struct ndis_softc *sc; ndis_handle adapter; ndis_sendmulti_handler sendfunc; ndis_senddone_func senddonefunc; int i; ndis_packet *p; uint8_t irql = 0; sc = arg; adapter = sc->ndis_block->nmb_miniportadapterctx; if (adapter == NULL) return (ENXIO); sendfunc = sc->ndis_chars->nmc_sendmulti_func; senddonefunc = sc->ndis_block->nmb_senddone_func; if (NDIS_SERIALIZED(sc->ndis_block)) KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); MSCALL3(sendfunc, adapter, packets, cnt); for (i = 0; i < cnt; i++) { p = packets[i]; /* * Either the driver already handed the packet to * ndis_txeof() due to a failure, or it wants to keep * it and release it asynchronously later. Skip to the * next one. */ if (p == NULL || p->np_oob.npo_status == NDIS_STATUS_PENDING) continue; MSCALL3(senddonefunc, sc->ndis_block, p, p->np_oob.npo_status); } if (NDIS_SERIALIZED(sc->ndis_block)) KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); return (0); } int ndis_send_packet(arg, packet) void *arg; ndis_packet *packet; { struct ndis_softc *sc; ndis_handle adapter; ndis_status status; ndis_sendsingle_handler sendfunc; ndis_senddone_func senddonefunc; uint8_t irql = 0; sc = arg; adapter = sc->ndis_block->nmb_miniportadapterctx; if (adapter == NULL) return (ENXIO); sendfunc = sc->ndis_chars->nmc_sendsingle_func; senddonefunc = sc->ndis_block->nmb_senddone_func; if (NDIS_SERIALIZED(sc->ndis_block)) KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); status = MSCALL3(sendfunc, adapter, packet, packet->np_private.npp_flags); if (status == NDIS_STATUS_PENDING) { if (NDIS_SERIALIZED(sc->ndis_block)) KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); return (0); } MSCALL3(senddonefunc, sc->ndis_block, packet, status); if (NDIS_SERIALIZED(sc->ndis_block)) KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); return (0); } int ndis_init_dma(arg) void *arg; { struct ndis_softc *sc; int i, error; sc = arg; sc->ndis_tmaps = malloc(sizeof(bus_dmamap_t) * sc->ndis_maxpkts, M_DEVBUF, M_NOWAIT|M_ZERO); if (sc->ndis_tmaps == NULL) return (ENOMEM); for (i = 0; i < sc->ndis_maxpkts; i++) { error = bus_dmamap_create(sc->ndis_ttag, 0, &sc->ndis_tmaps[i]); if (error) { free(sc->ndis_tmaps, M_DEVBUF); return (ENODEV); } } return (0); } int ndis_destroy_dma(arg) void *arg; { struct ndis_softc *sc; struct mbuf *m; ndis_packet *p = NULL; int i; sc = arg; for (i = 0; i < sc->ndis_maxpkts; i++) { if (sc->ndis_txarray[i] != NULL) { p = sc->ndis_txarray[i]; m = (struct mbuf *)p->np_rsvd[1]; if (m != NULL) m_freem(m); ndis_free_packet(sc->ndis_txarray[i]); } bus_dmamap_destroy(sc->ndis_ttag, sc->ndis_tmaps[i]); } free(sc->ndis_tmaps, M_DEVBUF); bus_dma_tag_destroy(sc->ndis_ttag); return (0); } int ndis_reset_nic(arg) void *arg; { struct ndis_softc *sc; ndis_handle adapter; ndis_reset_handler resetfunc; uint8_t addressing_reset; int rval; uint8_t irql = 0; sc = arg; NDIS_LOCK(sc); adapter = sc->ndis_block->nmb_miniportadapterctx; resetfunc = sc->ndis_chars->nmc_reset_func; if (adapter == NULL || resetfunc == NULL || sc->ndis_block->nmb_devicectx == NULL) { NDIS_UNLOCK(sc); return (EIO); } NDIS_UNLOCK(sc); KeResetEvent(&sc->ndis_block->nmb_resetevent); if (NDIS_SERIALIZED(sc->ndis_block)) KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); rval = MSCALL2(resetfunc, &addressing_reset, adapter); if (NDIS_SERIALIZED(sc->ndis_block)) KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); if (rval == NDIS_STATUS_PENDING) KeWaitForSingleObject(&sc->ndis_block->nmb_resetevent, 0, 0, FALSE, NULL); return (0); } int ndis_halt_nic(arg) void *arg; { struct ndis_softc *sc; ndis_handle adapter; ndis_halt_handler haltfunc; ndis_miniport_block *block; int empty = 0; uint8_t irql; sc = arg; block = sc->ndis_block; if (!cold) KeFlushQueuedDpcs(); /* * Wait for all packets to be returned. */ while (1) { KeAcquireSpinLock(&block->nmb_returnlock, &irql); empty = IsListEmpty(&block->nmb_returnlist); KeReleaseSpinLock(&block->nmb_returnlock, irql); if (empty) break; NdisMSleep(1000); } NDIS_LOCK(sc); adapter = sc->ndis_block->nmb_miniportadapterctx; if (adapter == NULL) { NDIS_UNLOCK(sc); return (EIO); } sc->ndis_block->nmb_devicectx = NULL; /* * The adapter context is only valid after the init * handler has been called, and is invalid once the * halt handler has been called. */ haltfunc = sc->ndis_chars->nmc_halt_func; NDIS_UNLOCK(sc); MSCALL1(haltfunc, adapter); NDIS_LOCK(sc); sc->ndis_block->nmb_miniportadapterctx = NULL; NDIS_UNLOCK(sc); return (0); } int ndis_shutdown_nic(arg) void *arg; { struct ndis_softc *sc; ndis_handle adapter; ndis_shutdown_handler shutdownfunc; sc = arg; NDIS_LOCK(sc); adapter = sc->ndis_block->nmb_miniportadapterctx; shutdownfunc = sc->ndis_chars->nmc_shutdown_handler; NDIS_UNLOCK(sc); if (adapter == NULL || shutdownfunc == NULL) return (EIO); if (sc->ndis_chars->nmc_rsvd0 == NULL) MSCALL1(shutdownfunc, adapter); else MSCALL1(shutdownfunc, sc->ndis_chars->nmc_rsvd0); TAILQ_REMOVE(&ndis_devhead, sc->ndis_block, link); return (0); } int ndis_pnpevent_nic(arg, type) void *arg; int type; { device_t dev; struct ndis_softc *sc; ndis_handle adapter; ndis_pnpevent_handler pnpeventfunc; dev = arg; sc = device_get_softc(arg); NDIS_LOCK(sc); adapter = sc->ndis_block->nmb_miniportadapterctx; pnpeventfunc = sc->ndis_chars->nmc_pnpevent_handler; NDIS_UNLOCK(sc); if (adapter == NULL || pnpeventfunc == NULL) return (EIO); if (sc->ndis_chars->nmc_rsvd0 == NULL) MSCALL4(pnpeventfunc, adapter, type, NULL, 0); else MSCALL4(pnpeventfunc, sc->ndis_chars->nmc_rsvd0, type, NULL, 0); return (0); } int ndis_init_nic(arg) void *arg; { struct ndis_softc *sc; ndis_miniport_block *block; ndis_init_handler initfunc; ndis_status status, openstatus = 0; ndis_medium mediumarray[NdisMediumMax]; uint32_t chosenmedium, i; if (arg == NULL) return (EINVAL); sc = arg; NDIS_LOCK(sc); block = sc->ndis_block; initfunc = sc->ndis_chars->nmc_init_func; NDIS_UNLOCK(sc); sc->ndis_block->nmb_timerlist = NULL; for (i = 0; i < NdisMediumMax; i++) mediumarray[i] = i; status = MSCALL6(initfunc, &openstatus, &chosenmedium, mediumarray, NdisMediumMax, block, block); /* * If the init fails, blow away the other exported routines * we obtained from the driver so we can't call them later. * If the init failed, none of these will work. */ if (status != NDIS_STATUS_SUCCESS) { NDIS_LOCK(sc); sc->ndis_block->nmb_miniportadapterctx = NULL; NDIS_UNLOCK(sc); return (ENXIO); } /* * This may look really goofy, but apparently it is possible * to halt a miniport too soon after it's been initialized. * After MiniportInitialize() finishes, pause for 1 second * to give the chip a chance to handle any short-lived timers * that were set in motion. If we call MiniportHalt() too soon, * some of the timers may not be cancelled, because the driver * expects them to fire before the halt is called. */ pause("ndwait", hz); NDIS_LOCK(sc); sc->ndis_block->nmb_devicectx = sc; NDIS_UNLOCK(sc); return (0); } static void ndis_intrsetup(dpc, dobj, ip, sc) kdpc *dpc; device_object *dobj; irp *ip; struct ndis_softc *sc; { ndis_miniport_interrupt *intr; intr = sc->ndis_block->nmb_interrupt; /* Sanity check. */ if (intr == NULL) return; KeAcquireSpinLockAtDpcLevel(&intr->ni_dpccountlock); KeResetEvent(&intr->ni_dpcevt); if (KeInsertQueueDpc(&intr->ni_dpc, NULL, NULL) == TRUE) intr->ni_dpccnt++; KeReleaseSpinLockFromDpcLevel(&intr->ni_dpccountlock); } int ndis_get_info(arg, oid, buf, buflen) void *arg; ndis_oid oid; void *buf; int *buflen; { struct ndis_softc *sc; ndis_status rval; ndis_handle adapter; ndis_queryinfo_handler queryfunc; uint32_t byteswritten = 0, bytesneeded = 0; uint8_t irql; uint64_t duetime; sc = arg; KeResetEvent(&sc->ndis_block->nmb_getevent); KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); if (sc->ndis_block->nmb_pendingreq != NULL) { KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); panic("ndis_get_info() called while other request pending"); } else sc->ndis_block->nmb_pendingreq = (ndis_request *)sc; queryfunc = sc->ndis_chars->nmc_queryinfo_func; adapter = sc->ndis_block->nmb_miniportadapterctx; if (adapter == NULL || queryfunc == NULL || sc->ndis_block->nmb_devicectx == NULL) { sc->ndis_block->nmb_pendingreq = NULL; KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); return (ENXIO); } rval = MSCALL6(queryfunc, adapter, oid, buf, *buflen, &byteswritten, &bytesneeded); sc->ndis_block->nmb_pendingreq = NULL; KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); /* Wait for requests that block. */ if (rval == NDIS_STATUS_PENDING) { /* Wait up to 5 seconds. */ duetime = (5 * 1000000) * -10; KeWaitForSingleObject(&sc->ndis_block->nmb_getevent, 0, 0, FALSE, &duetime); rval = sc->ndis_block->nmb_getstat; } if (byteswritten) *buflen = byteswritten; if (bytesneeded) *buflen = bytesneeded; if (rval == NDIS_STATUS_INVALID_LENGTH || rval == NDIS_STATUS_BUFFER_TOO_SHORT) return (ENOSPC); if (rval == NDIS_STATUS_INVALID_OID) return (EINVAL); if (rval == NDIS_STATUS_NOT_SUPPORTED || rval == NDIS_STATUS_NOT_ACCEPTED) return (ENOTSUP); if (rval != NDIS_STATUS_SUCCESS) return (ENODEV); return (0); } uint32_t NdisAddDevice(drv, pdo) driver_object *drv; device_object *pdo; { device_object *fdo; ndis_miniport_block *block; struct ndis_softc *sc; uint32_t status; int error; sc = device_get_softc(pdo->do_devext); if (sc->ndis_iftype == PCMCIABus || sc->ndis_iftype == PCIBus) { error = bus_setup_intr(sc->ndis_dev, sc->ndis_irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, ntoskrnl_intr, NULL, &sc->ndis_intrhand); if (error) return (NDIS_STATUS_FAILURE); } status = IoCreateDevice(drv, sizeof(ndis_miniport_block), NULL, FILE_DEVICE_UNKNOWN, 0, FALSE, &fdo); if (status != STATUS_SUCCESS) return (status); block = fdo->do_devext; block->nmb_filterdbs.nf_ethdb = block; block->nmb_deviceobj = fdo; block->nmb_physdeviceobj = pdo; block->nmb_nextdeviceobj = IoAttachDeviceToDeviceStack(fdo, pdo); KeInitializeSpinLock(&block->nmb_lock); KeInitializeSpinLock(&block->nmb_returnlock); KeInitializeEvent(&block->nmb_getevent, EVENT_TYPE_NOTIFY, TRUE); KeInitializeEvent(&block->nmb_setevent, EVENT_TYPE_NOTIFY, TRUE); KeInitializeEvent(&block->nmb_resetevent, EVENT_TYPE_NOTIFY, TRUE); InitializeListHead(&block->nmb_parmlist); InitializeListHead(&block->nmb_returnlist); block->nmb_returnitem = IoAllocateWorkItem(fdo); /* * Stash pointers to the miniport block and miniport * characteristics info in the if_ndis softc so the * UNIX wrapper driver can get to them later. */ sc->ndis_block = block; sc->ndis_chars = IoGetDriverObjectExtension(drv, (void *)1); /* * If the driver has a MiniportTransferData() function, * we should allocate a private RX packet pool. */ if (sc->ndis_chars->nmc_transferdata_func != NULL) { NdisAllocatePacketPool(&status, &block->nmb_rxpool, 32, PROTOCOL_RESERVED_SIZE_IN_PACKET); if (status != NDIS_STATUS_SUCCESS) { IoDetachDevice(block->nmb_nextdeviceobj); IoDeleteDevice(fdo); return (status); } InitializeListHead((&block->nmb_packetlist)); } /* Give interrupt handling priority over timers. */ IoInitializeDpcRequest(fdo, kernndis_functbl[6].ipt_wrap); KeSetImportanceDpc(&fdo->do_dpc, KDPC_IMPORTANCE_HIGH); /* Finish up BSD-specific setup. */ block->nmb_signature = (void *)0xcafebabe; block->nmb_status_func = kernndis_functbl[0].ipt_wrap; block->nmb_statusdone_func = kernndis_functbl[1].ipt_wrap; block->nmb_setdone_func = kernndis_functbl[2].ipt_wrap; block->nmb_querydone_func = kernndis_functbl[3].ipt_wrap; block->nmb_resetdone_func = kernndis_functbl[4].ipt_wrap; block->nmb_sendrsrc_func = kernndis_functbl[5].ipt_wrap; block->nmb_pendingreq = NULL; TAILQ_INSERT_TAIL(&ndis_devhead, block, link); return (STATUS_SUCCESS); } int ndis_unload_driver(arg) void *arg; { struct ndis_softc *sc; device_object *fdo; sc = arg; if (sc->ndis_intrhand) bus_teardown_intr(sc->ndis_dev, sc->ndis_irq, sc->ndis_intrhand); if (sc->ndis_block->nmb_rlist != NULL) free(sc->ndis_block->nmb_rlist, M_DEVBUF); ndis_flush_sysctls(sc); TAILQ_REMOVE(&ndis_devhead, sc->ndis_block, link); if (sc->ndis_chars->nmc_transferdata_func != NULL) NdisFreePacketPool(sc->ndis_block->nmb_rxpool); fdo = sc->ndis_block->nmb_deviceobj; IoFreeWorkItem(sc->ndis_block->nmb_returnitem); IoDetachDevice(sc->ndis_block->nmb_nextdeviceobj); IoDeleteDevice(fdo); return (0); } Index: head/sys/compat/ndis/subr_ndis.c =================================================================== --- head/sys/compat/ndis/subr_ndis.c (revision 214797) +++ head/sys/compat/ndis/subr_ndis.c (revision 214798) @@ -1,3360 +1,3340 @@ /*- * Copyright (c) 2003 * Bill Paul . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Bill Paul. * 4. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * This file implements a translation layer between the BSD networking * infrasturcture and Windows(R) NDIS network driver modules. A Windows * NDIS driver calls into several functions in the NDIS.SYS Windows * kernel module and exports a table of functions designed to be called * by the NDIS subsystem. Using the PE loader, we can patch our own * versions of the NDIS routines into a given Windows driver module and * convince the driver that it is in fact running on Windows. * * We provide a table of all our implemented NDIS routines which is patched * into the driver object code. All our exported routines must use the * _stdcall calling convention, since that's what the Windows object code * expects. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static char ndis_filepath[MAXPATHLEN]; SYSCTL_STRING(_hw, OID_AUTO, ndis_filepath, CTLFLAG_RW, ndis_filepath, MAXPATHLEN, "Path used by NdisOpenFile() to search for files"); static void NdisInitializeWrapper(ndis_handle *, driver_object *, void *, void *); static ndis_status NdisMRegisterMiniport(ndis_handle, ndis_miniport_characteristics *, int); static ndis_status NdisAllocateMemoryWithTag(void **, uint32_t, uint32_t); static ndis_status NdisAllocateMemory(void **, uint32_t, uint32_t, ndis_physaddr); static void NdisFreeMemory(void *, uint32_t, uint32_t); static ndis_status NdisMSetAttributesEx(ndis_handle, ndis_handle, uint32_t, uint32_t, ndis_interface_type); static void NdisOpenConfiguration(ndis_status *, ndis_handle *, ndis_handle); static void NdisOpenConfigurationKeyByIndex(ndis_status *, ndis_handle, uint32_t, unicode_string *, ndis_handle *); static void NdisOpenConfigurationKeyByName(ndis_status *, ndis_handle, unicode_string *, ndis_handle *); static ndis_status ndis_encode_parm(ndis_miniport_block *, struct sysctl_oid *, ndis_parm_type, ndis_config_parm **); static ndis_status ndis_decode_parm(ndis_miniport_block *, ndis_config_parm *, char *); static void NdisReadConfiguration(ndis_status *, ndis_config_parm **, ndis_handle, unicode_string *, ndis_parm_type); static void NdisWriteConfiguration(ndis_status *, ndis_handle, unicode_string *, ndis_config_parm *); static void NdisCloseConfiguration(ndis_handle); static void NdisAllocateSpinLock(ndis_spin_lock *); static void NdisFreeSpinLock(ndis_spin_lock *); static void NdisAcquireSpinLock(ndis_spin_lock *); static void NdisReleaseSpinLock(ndis_spin_lock *); static void NdisDprAcquireSpinLock(ndis_spin_lock *); static void NdisDprReleaseSpinLock(ndis_spin_lock *); static void NdisInitializeReadWriteLock(ndis_rw_lock *); static void NdisAcquireReadWriteLock(ndis_rw_lock *, uint8_t, ndis_lock_state *); static void NdisReleaseReadWriteLock(ndis_rw_lock *, ndis_lock_state *); static uint32_t NdisReadPciSlotInformation(ndis_handle, uint32_t, uint32_t, void *, uint32_t); static uint32_t NdisWritePciSlotInformation(ndis_handle, uint32_t, uint32_t, void *, uint32_t); static void NdisWriteErrorLogEntry(ndis_handle, ndis_error_code, uint32_t, ...); static void ndis_map_cb(void *, bus_dma_segment_t *, int, int); static void NdisMStartBufferPhysicalMapping(ndis_handle, ndis_buffer *, uint32_t, uint8_t, ndis_paddr_unit *, uint32_t *); static void NdisMCompleteBufferPhysicalMapping(ndis_handle, ndis_buffer *, uint32_t); static void NdisMInitializeTimer(ndis_miniport_timer *, ndis_handle, ndis_timer_function, void *); static void NdisInitializeTimer(ndis_timer *, ndis_timer_function, void *); static void NdisSetTimer(ndis_timer *, uint32_t); static void NdisMSetPeriodicTimer(ndis_miniport_timer *, uint32_t); static void NdisMCancelTimer(ndis_timer *, uint8_t *); static void ndis_timercall(kdpc *, ndis_miniport_timer *, void *, void *); static void NdisMQueryAdapterResources(ndis_status *, ndis_handle, ndis_resource_list *, uint32_t *); static ndis_status NdisMRegisterIoPortRange(void **, ndis_handle, uint32_t, uint32_t); static void NdisMDeregisterIoPortRange(ndis_handle, uint32_t, uint32_t, void *); static void NdisReadNetworkAddress(ndis_status *, void **, uint32_t *, ndis_handle); static ndis_status NdisQueryMapRegisterCount(uint32_t, uint32_t *); static ndis_status NdisMAllocateMapRegisters(ndis_handle, uint32_t, uint8_t, uint32_t, uint32_t); static void NdisMFreeMapRegisters(ndis_handle); static void ndis_mapshared_cb(void *, bus_dma_segment_t *, int, int); static void NdisMAllocateSharedMemory(ndis_handle, uint32_t, uint8_t, void **, ndis_physaddr *); static void ndis_asyncmem_complete(device_object *, void *); static ndis_status NdisMAllocateSharedMemoryAsync(ndis_handle, uint32_t, uint8_t, void *); static void NdisMFreeSharedMemory(ndis_handle, uint32_t, uint8_t, void *, ndis_physaddr); static ndis_status NdisMMapIoSpace(void **, ndis_handle, ndis_physaddr, uint32_t); static void NdisMUnmapIoSpace(ndis_handle, void *, uint32_t); static uint32_t NdisGetCacheFillSize(void); static uint32_t NdisMGetDmaAlignment(ndis_handle); static ndis_status NdisMInitializeScatterGatherDma(ndis_handle, uint8_t, uint32_t); static void NdisUnchainBufferAtFront(ndis_packet *, ndis_buffer **); static void NdisUnchainBufferAtBack(ndis_packet *, ndis_buffer **); static void NdisAllocateBufferPool(ndis_status *, ndis_handle *, uint32_t); static void NdisFreeBufferPool(ndis_handle); static void NdisAllocateBuffer(ndis_status *, ndis_buffer **, ndis_handle, void *, uint32_t); static void NdisFreeBuffer(ndis_buffer *); static uint32_t NdisBufferLength(ndis_buffer *); static void NdisQueryBuffer(ndis_buffer *, void **, uint32_t *); static void NdisQueryBufferSafe(ndis_buffer *, void **, uint32_t *, uint32_t); static void *NdisBufferVirtualAddress(ndis_buffer *); static void *NdisBufferVirtualAddressSafe(ndis_buffer *, uint32_t); static void NdisAdjustBufferLength(ndis_buffer *, int); static uint32_t NdisInterlockedIncrement(uint32_t *); static uint32_t NdisInterlockedDecrement(uint32_t *); static void NdisInitializeEvent(ndis_event *); static void NdisSetEvent(ndis_event *); static void NdisResetEvent(ndis_event *); static uint8_t NdisWaitEvent(ndis_event *, uint32_t); static ndis_status NdisUnicodeStringToAnsiString(ansi_string *, unicode_string *); static ndis_status NdisAnsiStringToUnicodeString(unicode_string *, ansi_string *); static ndis_status NdisMPciAssignResources(ndis_handle, uint32_t, ndis_resource_list **); static ndis_status NdisMRegisterInterrupt(ndis_miniport_interrupt *, ndis_handle, uint32_t, uint32_t, uint8_t, uint8_t, ndis_interrupt_mode); static void NdisMDeregisterInterrupt(ndis_miniport_interrupt *); static void NdisMRegisterAdapterShutdownHandler(ndis_handle, void *, ndis_shutdown_handler); static void NdisMDeregisterAdapterShutdownHandler(ndis_handle); static uint32_t NDIS_BUFFER_TO_SPAN_PAGES(ndis_buffer *); static void NdisGetBufferPhysicalArraySize(ndis_buffer *, uint32_t *); static void NdisQueryBufferOffset(ndis_buffer *, uint32_t *, uint32_t *); static uint32_t NdisReadPcmciaAttributeMemory(ndis_handle, uint32_t, void *, uint32_t); static uint32_t NdisWritePcmciaAttributeMemory(ndis_handle, uint32_t, void *, uint32_t); static list_entry *NdisInterlockedInsertHeadList(list_entry *, list_entry *, ndis_spin_lock *); static list_entry *NdisInterlockedRemoveHeadList(list_entry *, ndis_spin_lock *); static list_entry *NdisInterlockedInsertTailList(list_entry *, list_entry *, ndis_spin_lock *); static uint8_t NdisMSynchronizeWithInterrupt(ndis_miniport_interrupt *, void *, void *); static void NdisGetCurrentSystemTime(uint64_t *); static void NdisGetSystemUpTime(uint32_t *); static void NdisInitializeString(unicode_string *, char *); static void NdisInitAnsiString(ansi_string *, char *); static void NdisInitUnicodeString(unicode_string *, uint16_t *); static void NdisFreeString(unicode_string *); static ndis_status NdisMRemoveMiniport(ndis_handle *); static void NdisTerminateWrapper(ndis_handle, void *); static void NdisMGetDeviceProperty(ndis_handle, device_object **, device_object **, device_object **, cm_resource_list *, cm_resource_list *); static void NdisGetFirstBufferFromPacket(ndis_packet *, ndis_buffer **, void **, uint32_t *, uint32_t *); static void NdisGetFirstBufferFromPacketSafe(ndis_packet *, ndis_buffer **, void **, uint32_t *, uint32_t *, uint32_t); static int ndis_find_sym(linker_file_t, char *, char *, caddr_t *); static void NdisOpenFile(ndis_status *, ndis_handle *, uint32_t *, unicode_string *, ndis_physaddr); static void NdisMapFile(ndis_status *, void **, ndis_handle); static void NdisUnmapFile(ndis_handle); static void NdisCloseFile(ndis_handle); static uint8_t NdisSystemProcessorCount(void); static void NdisMIndicateStatusComplete(ndis_handle); static void NdisMIndicateStatus(ndis_handle, ndis_status, void *, uint32_t); static uint8_t ndis_intr(kinterrupt *, void *); static void ndis_intrhand(kdpc *, ndis_miniport_interrupt *, void *, void *); static funcptr ndis_findwrap(funcptr); static void NdisCopyFromPacketToPacket(ndis_packet *, uint32_t, uint32_t, ndis_packet *, uint32_t, uint32_t *); static void NdisCopyFromPacketToPacketSafe(ndis_packet *, uint32_t, uint32_t, ndis_packet *, uint32_t, uint32_t *, uint32_t); static void NdisIMCopySendPerPacketInfo(ndis_packet *, ndis_packet *); static ndis_status NdisMRegisterDevice(ndis_handle, unicode_string *, unicode_string *, driver_dispatch **, void **, ndis_handle *); static ndis_status NdisMDeregisterDevice(ndis_handle); static ndis_status NdisMQueryAdapterInstanceName(unicode_string *, ndis_handle); static void NdisMRegisterUnloadHandler(ndis_handle, void *); static void dummy(void); /* * Some really old drivers do not properly check the return value * from NdisAllocatePacket() and NdisAllocateBuffer() and will * sometimes allocate few more buffers/packets that they originally * requested when they created the pool. To prevent this from being * a problem, we allocate a few extra buffers/packets beyond what * the driver asks for. This #define controls how many. */ #define NDIS_POOL_EXTRA 16 int ndis_libinit() { image_patch_table *patch; strcpy(ndis_filepath, "/compat/ndis"); patch = ndis_functbl; while (patch->ipt_func != NULL) { windrv_wrap((funcptr)patch->ipt_func, (funcptr *)&patch->ipt_wrap, patch->ipt_argcnt, patch->ipt_ftype); patch++; } return (0); } int ndis_libfini() { image_patch_table *patch; patch = ndis_functbl; while (patch->ipt_func != NULL) { windrv_unwrap(patch->ipt_wrap); patch++; } return (0); } static funcptr ndis_findwrap(func) funcptr func; { image_patch_table *patch; patch = ndis_functbl; while (patch->ipt_func != NULL) { if ((funcptr)patch->ipt_func == func) return ((funcptr)patch->ipt_wrap); patch++; } return (NULL); } /* * This routine does the messy Windows Driver Model device attachment * stuff on behalf of NDIS drivers. We register our own AddDevice * routine here */ static void NdisInitializeWrapper(wrapper, drv, path, unused) ndis_handle *wrapper; driver_object *drv; void *path; void *unused; { /* * As of yet, I haven't come up with a compelling * reason to define a private NDIS wrapper structure, * so we use a pointer to the driver object as the * wrapper handle. The driver object has the miniport * characteristics struct for this driver hung off it * via IoAllocateDriverObjectExtension(), and that's * really all the private data we need. */ *wrapper = drv; /* * If this was really Windows, we'd be registering dispatch * routines for the NDIS miniport module here, but we're * not Windows so all we really need to do is set up an * AddDevice function that'll be invoked when a new device * instance appears. */ drv->dro_driverext->dre_adddevicefunc = NdisAddDevice; } static void NdisTerminateWrapper(handle, syspec) ndis_handle handle; void *syspec; { /* Nothing to see here, move along. */ } static ndis_status NdisMRegisterMiniport(handle, characteristics, len) ndis_handle handle; ndis_miniport_characteristics *characteristics; int len; { ndis_miniport_characteristics *ch = NULL; driver_object *drv; drv = (driver_object *)handle; /* * We need to save the NDIS miniport characteristics * somewhere. This data is per-driver, not per-device * (all devices handled by the same driver have the * same characteristics) so we hook it onto the driver * object using IoAllocateDriverObjectExtension(). * The extra extension info is automagically deleted when * the driver is unloaded (see windrv_unload()). */ if (IoAllocateDriverObjectExtension(drv, (void *)1, sizeof(ndis_miniport_characteristics), (void **)&ch) != STATUS_SUCCESS) { return (NDIS_STATUS_RESOURCES); } bzero((char *)ch, sizeof(ndis_miniport_characteristics)); bcopy((char *)characteristics, (char *)ch, len); if (ch->nmc_version_major < 5 || ch->nmc_version_minor < 1) { ch->nmc_shutdown_handler = NULL; ch->nmc_canceltxpkts_handler = NULL; ch->nmc_pnpevent_handler = NULL; } return (NDIS_STATUS_SUCCESS); } static ndis_status NdisAllocateMemoryWithTag(vaddr, len, tag) void **vaddr; uint32_t len; uint32_t tag; { void *mem; mem = ExAllocatePoolWithTag(NonPagedPool, len, tag); if (mem == NULL) { return (NDIS_STATUS_RESOURCES); } *vaddr = mem; return (NDIS_STATUS_SUCCESS); } static ndis_status NdisAllocateMemory(vaddr, len, flags, highaddr) void **vaddr; uint32_t len; uint32_t flags; ndis_physaddr highaddr; { void *mem; mem = ExAllocatePoolWithTag(NonPagedPool, len, 0); if (mem == NULL) return (NDIS_STATUS_RESOURCES); *vaddr = mem; return (NDIS_STATUS_SUCCESS); } static void NdisFreeMemory(vaddr, len, flags) void *vaddr; uint32_t len; uint32_t flags; { if (len == 0) return; ExFreePool(vaddr); } static ndis_status NdisMSetAttributesEx(adapter_handle, adapter_ctx, hangsecs, flags, iftype) ndis_handle adapter_handle; ndis_handle adapter_ctx; uint32_t hangsecs; uint32_t flags; ndis_interface_type iftype; { ndis_miniport_block *block; /* * Save the adapter context, we need it for calling * the driver's internal functions. */ block = (ndis_miniport_block *)adapter_handle; block->nmb_miniportadapterctx = adapter_ctx; block->nmb_checkforhangsecs = hangsecs; block->nmb_flags = flags; return (NDIS_STATUS_SUCCESS); } static void NdisOpenConfiguration(status, cfg, wrapctx) ndis_status *status; ndis_handle *cfg; ndis_handle wrapctx; { *cfg = wrapctx; *status = NDIS_STATUS_SUCCESS; } static void NdisOpenConfigurationKeyByName(status, cfg, subkey, subhandle) ndis_status *status; ndis_handle cfg; unicode_string *subkey; ndis_handle *subhandle; { *subhandle = cfg; *status = NDIS_STATUS_SUCCESS; } static void NdisOpenConfigurationKeyByIndex(status, cfg, idx, subkey, subhandle) ndis_status *status; ndis_handle cfg; uint32_t idx; unicode_string *subkey; ndis_handle *subhandle; { *status = NDIS_STATUS_FAILURE; } static ndis_status ndis_encode_parm(block, oid, type, parm) ndis_miniport_block *block; struct sysctl_oid *oid; ndis_parm_type type; ndis_config_parm **parm; { ndis_config_parm *p; ndis_parmlist_entry *np; unicode_string *us; ansi_string as; int base = 0; uint32_t val; char tmp[32]; np = ExAllocatePoolWithTag(NonPagedPool, sizeof(ndis_parmlist_entry), 0); if (np == NULL) return (NDIS_STATUS_RESOURCES); InsertHeadList((&block->nmb_parmlist), (&np->np_list)); *parm = p = &np->np_parm; switch(type) { case ndis_parm_string: /* See if this might be a number. */ val = strtoul((char *)oid->oid_arg1, NULL, 10); us = &p->ncp_parmdata.ncp_stringdata; p->ncp_type = ndis_parm_string; if (val) { snprintf(tmp, 32, "%x", val); RtlInitAnsiString(&as, tmp); } else { RtlInitAnsiString(&as, (char *)oid->oid_arg1); } if (RtlAnsiStringToUnicodeString(us, &as, TRUE)) { ExFreePool(np); return (NDIS_STATUS_RESOURCES); } break; case ndis_parm_int: if (strncmp((char *)oid->oid_arg1, "0x", 2) == 0) base = 16; else base = 10; p->ncp_type = ndis_parm_int; p->ncp_parmdata.ncp_intdata = strtol((char *)oid->oid_arg1, NULL, base); break; case ndis_parm_hexint: #ifdef notdef if (strncmp((char *)oid->oid_arg1, "0x", 2) == 0) base = 16; else base = 10; #endif base = 16; p->ncp_type = ndis_parm_hexint; p->ncp_parmdata.ncp_intdata = strtoul((char *)oid->oid_arg1, NULL, base); break; default: return (NDIS_STATUS_FAILURE); break; } return (NDIS_STATUS_SUCCESS); } static void NdisReadConfiguration(status, parm, cfg, key, type) ndis_status *status; ndis_config_parm **parm; ndis_handle cfg; unicode_string *key; ndis_parm_type type; { char *keystr = NULL; ndis_miniport_block *block; struct ndis_softc *sc; struct sysctl_oid *oidp; struct sysctl_ctx_entry *e; ansi_string as; block = (ndis_miniport_block *)cfg; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); if (key->us_len == 0 || key->us_buf == NULL) { *status = NDIS_STATUS_FAILURE; return; } if (RtlUnicodeStringToAnsiString(&as, key, TRUE)) { *status = NDIS_STATUS_RESOURCES; return; } keystr = as.as_buf; /* * See if registry key is already in a list of known keys * included with the driver. */ -#if __FreeBSD_version < 502113 - TAILQ_FOREACH(e, &sc->ndis_ctx, link) { -#else TAILQ_FOREACH(e, device_get_sysctl_ctx(sc->ndis_dev), link) { -#endif oidp = e->entry; if (strcasecmp(oidp->oid_name, keystr) == 0) { if (strcmp((char *)oidp->oid_arg1, "UNSET") == 0) { RtlFreeAnsiString(&as); *status = NDIS_STATUS_FAILURE; return; } *status = ndis_encode_parm(block, oidp, type, parm); RtlFreeAnsiString(&as); return; } } /* * If the key didn't match, add it to the list of dynamically * created ones. Sometimes, drivers refer to registry keys * that aren't documented in their .INF files. These keys * are supposed to be created by some sort of utility or * control panel snap-in that comes with the driver software. * Sometimes it's useful to be able to manipulate these. * If the driver requests the key in the form of a string, * make its default value an empty string, otherwise default * it to "0". */ if (type == ndis_parm_int || type == ndis_parm_hexint) ndis_add_sysctl(sc, keystr, "(dynamic integer key)", "UNSET", CTLFLAG_RW); else ndis_add_sysctl(sc, keystr, "(dynamic string key)", "UNSET", CTLFLAG_RW); RtlFreeAnsiString(&as); *status = NDIS_STATUS_FAILURE; } static ndis_status ndis_decode_parm(block, parm, val) ndis_miniport_block *block; ndis_config_parm *parm; char *val; { unicode_string *ustr; ansi_string as; switch(parm->ncp_type) { case ndis_parm_string: ustr = &parm->ncp_parmdata.ncp_stringdata; if (RtlUnicodeStringToAnsiString(&as, ustr, TRUE)) return (NDIS_STATUS_RESOURCES); bcopy(as.as_buf, val, as.as_len); RtlFreeAnsiString(&as); break; case ndis_parm_int: sprintf(val, "%d", parm->ncp_parmdata.ncp_intdata); break; case ndis_parm_hexint: sprintf(val, "%xu", parm->ncp_parmdata.ncp_intdata); break; default: return (NDIS_STATUS_FAILURE); break; } return (NDIS_STATUS_SUCCESS); } static void NdisWriteConfiguration(status, cfg, key, parm) ndis_status *status; ndis_handle cfg; unicode_string *key; ndis_config_parm *parm; { ansi_string as; char *keystr = NULL; ndis_miniport_block *block; struct ndis_softc *sc; struct sysctl_oid *oidp; struct sysctl_ctx_entry *e; char val[256]; block = (ndis_miniport_block *)cfg; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); if (RtlUnicodeStringToAnsiString(&as, key, TRUE)) { *status = NDIS_STATUS_RESOURCES; return; } keystr = as.as_buf; /* Decode the parameter into a string. */ bzero(val, sizeof(val)); *status = ndis_decode_parm(block, parm, val); if (*status != NDIS_STATUS_SUCCESS) { RtlFreeAnsiString(&as); return; } /* See if the key already exists. */ -#if __FreeBSD_version < 502113 - TAILQ_FOREACH(e, &sc->ndis_ctx, link) { -#else TAILQ_FOREACH(e, device_get_sysctl_ctx(sc->ndis_dev), link) { -#endif oidp = e->entry; if (strcasecmp(oidp->oid_name, keystr) == 0) { /* Found it, set the value. */ strcpy((char *)oidp->oid_arg1, val); RtlFreeAnsiString(&as); return; } } /* Not found, add a new key with the specified value. */ ndis_add_sysctl(sc, keystr, "(dynamically set key)", val, CTLFLAG_RW); RtlFreeAnsiString(&as); *status = NDIS_STATUS_SUCCESS; } static void NdisCloseConfiguration(cfg) ndis_handle cfg; { list_entry *e; ndis_parmlist_entry *pe; ndis_miniport_block *block; ndis_config_parm *p; block = (ndis_miniport_block *)cfg; while (!IsListEmpty(&block->nmb_parmlist)) { e = RemoveHeadList(&block->nmb_parmlist); pe = CONTAINING_RECORD(e, ndis_parmlist_entry, np_list); p = &pe->np_parm; if (p->ncp_type == ndis_parm_string) RtlFreeUnicodeString(&p->ncp_parmdata.ncp_stringdata); ExFreePool(e); } } /* * Initialize a Windows spinlock. */ static void NdisAllocateSpinLock(lock) ndis_spin_lock *lock; { KeInitializeSpinLock(&lock->nsl_spinlock); lock->nsl_kirql = 0; } /* * Destroy a Windows spinlock. This is a no-op for now. There are two reasons * for this. One is that it's sort of superfluous: we don't have to do anything * special to deallocate the spinlock. The other is that there are some buggy * drivers which call NdisFreeSpinLock() _after_ calling NdisFreeMemory() on * the block of memory in which the spinlock resides. (Yes, ADMtek, I'm * talking to you.) */ static void NdisFreeSpinLock(lock) ndis_spin_lock *lock; { #ifdef notdef KeInitializeSpinLock(&lock->nsl_spinlock); lock->nsl_kirql = 0; #endif } /* * Acquire a spinlock from IRQL <= DISPATCH_LEVEL. */ static void NdisAcquireSpinLock(lock) ndis_spin_lock *lock; { KeAcquireSpinLock(&lock->nsl_spinlock, &lock->nsl_kirql); } /* * Release a spinlock from IRQL == DISPATCH_LEVEL. */ static void NdisReleaseSpinLock(lock) ndis_spin_lock *lock; { KeReleaseSpinLock(&lock->nsl_spinlock, lock->nsl_kirql); } /* * Acquire a spinlock when already running at IRQL == DISPATCH_LEVEL. */ static void NdisDprAcquireSpinLock(lock) ndis_spin_lock *lock; { KeAcquireSpinLockAtDpcLevel(&lock->nsl_spinlock); } /* * Release a spinlock without leaving IRQL == DISPATCH_LEVEL. */ static void NdisDprReleaseSpinLock(lock) ndis_spin_lock *lock; { KeReleaseSpinLockFromDpcLevel(&lock->nsl_spinlock); } static void NdisInitializeReadWriteLock(lock) ndis_rw_lock *lock; { KeInitializeSpinLock(&lock->nrl_spinlock); bzero((char *)&lock->nrl_rsvd, sizeof(lock->nrl_rsvd)); } static void NdisAcquireReadWriteLock(ndis_rw_lock *lock, uint8_t writeacc, ndis_lock_state *state) { if (writeacc == TRUE) { KeAcquireSpinLock(&lock->nrl_spinlock, &state->nls_oldirql); lock->nrl_rsvd[0]++; } else lock->nrl_rsvd[1]++; } static void NdisReleaseReadWriteLock(lock, state) ndis_rw_lock *lock; ndis_lock_state *state; { if (lock->nrl_rsvd[0]) { lock->nrl_rsvd[0]--; KeReleaseSpinLock(&lock->nrl_spinlock, state->nls_oldirql); } else lock->nrl_rsvd[1]--; } static uint32_t NdisReadPciSlotInformation(adapter, slot, offset, buf, len) ndis_handle adapter; uint32_t slot; uint32_t offset; void *buf; uint32_t len; { ndis_miniport_block *block; int i; char *dest; device_t dev; block = (ndis_miniport_block *)adapter; dest = buf; if (block == NULL) return (0); dev = block->nmb_physdeviceobj->do_devext; /* * I have a test system consisting of a Sun w2100z * dual 2.4Ghz Opteron machine and an Atheros 802.11a/b/g * "Aries" miniPCI NIC. (The NIC is installed in the * machine using a miniPCI to PCI bus adapter card.) * When running in SMP mode, I found that * performing a large number of consecutive calls to * NdisReadPciSlotInformation() would result in a * sudden system reset (or in some cases a freeze). * My suspicion is that the multiple reads are somehow * triggering a fatal PCI bus error that leads to a * machine check. The 1us delay in the loop below * seems to prevent this problem. */ for (i = 0; i < len; i++) { DELAY(1); dest[i] = pci_read_config(dev, i + offset, 1); } return (len); } static uint32_t NdisWritePciSlotInformation(adapter, slot, offset, buf, len) ndis_handle adapter; uint32_t slot; uint32_t offset; void *buf; uint32_t len; { ndis_miniport_block *block; int i; char *dest; device_t dev; block = (ndis_miniport_block *)adapter; dest = buf; if (block == NULL) return (0); dev = block->nmb_physdeviceobj->do_devext; for (i = 0; i < len; i++) { DELAY(1); pci_write_config(dev, i + offset, dest[i], 1); } return (len); } /* * The errorlog routine uses a variable argument list, so we * have to declare it this way. */ #define ERRMSGLEN 512 static void NdisWriteErrorLogEntry(ndis_handle adapter, ndis_error_code code, uint32_t numerrors, ...) { ndis_miniport_block *block; va_list ap; int i, error; char *str = NULL; uint16_t flags; device_t dev; driver_object *drv; struct ndis_softc *sc; struct ifnet *ifp; unicode_string us; ansi_string as = { 0, 0, NULL }; block = (ndis_miniport_block *)adapter; dev = block->nmb_physdeviceobj->do_devext; drv = block->nmb_deviceobj->do_drvobj; sc = device_get_softc(dev); ifp = sc->ifp; if (ifp != NULL && ifp->if_flags & IFF_DEBUG) { error = pe_get_message((vm_offset_t)drv->dro_driverstart, code, &str, &i, &flags); if (error == 0) { if (flags & MESSAGE_RESOURCE_UNICODE) { RtlInitUnicodeString(&us, (uint16_t *)str); if (RtlUnicodeStringToAnsiString(&as, &us, TRUE) == STATUS_SUCCESS) str = as.as_buf; else str = NULL; } } } device_printf(dev, "NDIS ERROR: %x (%s)\n", code, str == NULL ? "unknown error" : str); if (ifp != NULL && ifp->if_flags & IFF_DEBUG) { device_printf(dev, "NDIS NUMERRORS: %x\n", numerrors); va_start(ap, numerrors); for (i = 0; i < numerrors; i++) device_printf(dev, "argptr: %p\n", va_arg(ap, void *)); va_end(ap); } if (as.as_len) RtlFreeAnsiString(&as); } static void ndis_map_cb(arg, segs, nseg, error) void *arg; bus_dma_segment_t *segs; int nseg; int error; { struct ndis_map_arg *ctx; int i; if (error) return; ctx = arg; for (i = 0; i < nseg; i++) { ctx->nma_fraglist[i].npu_physaddr.np_quad = segs[i].ds_addr; ctx->nma_fraglist[i].npu_len = segs[i].ds_len; } ctx->nma_cnt = nseg; } static void NdisMStartBufferPhysicalMapping(ndis_handle adapter, ndis_buffer *buf, uint32_t mapreg, uint8_t writedev, ndis_paddr_unit *addrarray, uint32_t *arraysize) { ndis_miniport_block *block; struct ndis_softc *sc; struct ndis_map_arg nma; bus_dmamap_t map; int error; if (adapter == NULL) return; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); if (mapreg > sc->ndis_mmapcnt) return; map = sc->ndis_mmaps[mapreg]; nma.nma_fraglist = addrarray; error = bus_dmamap_load(sc->ndis_mtag, map, MmGetMdlVirtualAddress(buf), MmGetMdlByteCount(buf), ndis_map_cb, (void *)&nma, BUS_DMA_NOWAIT); if (error) return; bus_dmamap_sync(sc->ndis_mtag, map, writedev ? BUS_DMASYNC_PREWRITE : BUS_DMASYNC_PREREAD); *arraysize = nma.nma_cnt; } static void NdisMCompleteBufferPhysicalMapping(adapter, buf, mapreg) ndis_handle adapter; ndis_buffer *buf; uint32_t mapreg; { ndis_miniport_block *block; struct ndis_softc *sc; bus_dmamap_t map; if (adapter == NULL) return; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); if (mapreg > sc->ndis_mmapcnt) return; map = sc->ndis_mmaps[mapreg]; bus_dmamap_sync(sc->ndis_mtag, map, BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->ndis_mtag, map); } /* * This is an older (?) timer init routine which doesn't * accept a miniport context handle. Serialized miniports should * never call this function. */ static void NdisInitializeTimer(timer, func, ctx) ndis_timer *timer; ndis_timer_function func; void *ctx; { KeInitializeTimer(&timer->nt_ktimer); KeInitializeDpc(&timer->nt_kdpc, func, ctx); KeSetImportanceDpc(&timer->nt_kdpc, KDPC_IMPORTANCE_LOW); } static void ndis_timercall(dpc, timer, sysarg1, sysarg2) kdpc *dpc; ndis_miniport_timer *timer; void *sysarg1; void *sysarg2; { /* * Since we're called as a DPC, we should be running * at DISPATCH_LEVEL here. This means to acquire the * spinlock, we can use KeAcquireSpinLockAtDpcLevel() * rather than KeAcquireSpinLock(). */ if (NDIS_SERIALIZED(timer->nmt_block)) KeAcquireSpinLockAtDpcLevel(&timer->nmt_block->nmb_lock); MSCALL4(timer->nmt_timerfunc, dpc, timer->nmt_timerctx, sysarg1, sysarg2); if (NDIS_SERIALIZED(timer->nmt_block)) KeReleaseSpinLockFromDpcLevel(&timer->nmt_block->nmb_lock); } /* * For a long time I wondered why there were two NDIS timer initialization * routines, and why this one needed an NDIS_MINIPORT_TIMER and the * MiniportAdapterHandle. The NDIS_MINIPORT_TIMER has its own callout * function and context pointers separate from those in the DPC, which * allows for another level of indirection: when the timer fires, we * can have our own timer function invoked, and from there we can call * the driver's function. But why go to all that trouble? Then it hit * me: for serialized miniports, the timer callouts are not re-entrant. * By trapping the callouts and having access to the MiniportAdapterHandle, * we can protect the driver callouts by acquiring the NDIS serialization * lock. This is essential for allowing serialized miniports to work * correctly on SMP systems. On UP hosts, setting IRQL to DISPATCH_LEVEL * is enough to prevent other threads from pre-empting you, but with * SMP, you must acquire a lock as well, otherwise the other CPU is * free to clobber you. */ static void NdisMInitializeTimer(timer, handle, func, ctx) ndis_miniport_timer *timer; ndis_handle handle; ndis_timer_function func; void *ctx; { ndis_miniport_block *block; struct ndis_softc *sc; block = (ndis_miniport_block *)handle; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); /* Save the driver's funcptr and context */ timer->nmt_timerfunc = func; timer->nmt_timerctx = ctx; timer->nmt_block = handle; /* * Set up the timer so it will call our intermediate DPC. * Be sure to use the wrapped entry point, since * ntoskrnl_run_dpc() expects to invoke a function with * Microsoft calling conventions. */ KeInitializeTimer(&timer->nmt_ktimer); KeInitializeDpc(&timer->nmt_kdpc, ndis_findwrap((funcptr)ndis_timercall), timer); timer->nmt_ktimer.k_dpc = &timer->nmt_kdpc; } /* * In Windows, there's both an NdisMSetTimer() and an NdisSetTimer(), * but the former is just a macro wrapper around the latter. */ static void NdisSetTimer(timer, msecs) ndis_timer *timer; uint32_t msecs; { /* * KeSetTimer() wants the period in * hundred nanosecond intervals. */ KeSetTimer(&timer->nt_ktimer, ((int64_t)msecs * -10000), &timer->nt_kdpc); } static void NdisMSetPeriodicTimer(timer, msecs) ndis_miniport_timer *timer; uint32_t msecs; { KeSetTimerEx(&timer->nmt_ktimer, ((int64_t)msecs * -10000), msecs, &timer->nmt_kdpc); } /* * Technically, this is really NdisCancelTimer(), but we also * (ab)use it for NdisMCancelTimer(), since in our implementation * we don't need the extra info in the ndis_miniport_timer * structure just to cancel a timer. */ static void NdisMCancelTimer(timer, cancelled) ndis_timer *timer; uint8_t *cancelled; { *cancelled = KeCancelTimer(&timer->nt_ktimer); } static void NdisMQueryAdapterResources(status, adapter, list, buflen) ndis_status *status; ndis_handle adapter; ndis_resource_list *list; uint32_t *buflen; { ndis_miniport_block *block; struct ndis_softc *sc; int rsclen; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); rsclen = sizeof(ndis_resource_list) + (sizeof(cm_partial_resource_desc) * (sc->ndis_rescnt - 1)); if (*buflen < rsclen) { *buflen = rsclen; *status = NDIS_STATUS_INVALID_LENGTH; return; } bcopy((char *)block->nmb_rlist, (char *)list, rsclen); *status = NDIS_STATUS_SUCCESS; } static ndis_status NdisMRegisterIoPortRange(offset, adapter, port, numports) void **offset; ndis_handle adapter; uint32_t port; uint32_t numports; { struct ndis_miniport_block *block; struct ndis_softc *sc; if (adapter == NULL) return (NDIS_STATUS_FAILURE); block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); if (sc->ndis_res_io == NULL) return (NDIS_STATUS_FAILURE); /* Don't let the device map more ports than we have. */ if (rman_get_size(sc->ndis_res_io) < numports) return (NDIS_STATUS_INVALID_LENGTH); *offset = (void *)rman_get_start(sc->ndis_res_io); return (NDIS_STATUS_SUCCESS); } static void NdisMDeregisterIoPortRange(adapter, port, numports, offset) ndis_handle adapter; uint32_t port; uint32_t numports; void *offset; { } static void NdisReadNetworkAddress(status, addr, addrlen, adapter) ndis_status *status; void **addr; uint32_t *addrlen; ndis_handle adapter; { struct ndis_softc *sc; ndis_miniport_block *block; uint8_t empty[] = { 0, 0, 0, 0, 0, 0 }; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); if (sc->ifp == NULL) { *status = NDIS_STATUS_FAILURE; return; } -#ifdef IFP2ENADDR - if (bcmp(IFP2ENADDR(sc->ifp), empty, ETHER_ADDR_LEN) == 0) -#elif __FreeBSD_version >= 700000 if (sc->ifp->if_addr == NULL || bcmp(IF_LLADDR(sc->ifp), empty, ETHER_ADDR_LEN) == 0) -#else - if (bcmp(sc->arpcom.ac_enaddr, empty, ETHER_ADDR_LEN) == 0) -#endif *status = NDIS_STATUS_FAILURE; else { -#ifdef IFP2ENADDR - *addr = IFP2ENADDR(sc->ifp); -#elif __FreeBSD_version >= 700000 *addr = IF_LLADDR(sc->ifp); -#else - *addr = sc->arpcom.ac_enaddr; -#endif *addrlen = ETHER_ADDR_LEN; *status = NDIS_STATUS_SUCCESS; } } static ndis_status NdisQueryMapRegisterCount(bustype, cnt) uint32_t bustype; uint32_t *cnt; { *cnt = 8192; return (NDIS_STATUS_SUCCESS); } static ndis_status NdisMAllocateMapRegisters(ndis_handle adapter, uint32_t dmachannel, uint8_t dmasize, uint32_t physmapneeded, uint32_t maxmap) { struct ndis_softc *sc; ndis_miniport_block *block; int error, i, nseg = NDIS_MAXSEG; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); sc->ndis_mmaps = malloc(sizeof(bus_dmamap_t) * physmapneeded, M_DEVBUF, M_NOWAIT|M_ZERO); if (sc->ndis_mmaps == NULL) return (NDIS_STATUS_RESOURCES); error = bus_dma_tag_create(sc->ndis_parent_tag, ETHER_ALIGN, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxmap * nseg, nseg, maxmap, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->ndis_mtag); if (error) { free(sc->ndis_mmaps, M_DEVBUF); return (NDIS_STATUS_RESOURCES); } for (i = 0; i < physmapneeded; i++) bus_dmamap_create(sc->ndis_mtag, 0, &sc->ndis_mmaps[i]); sc->ndis_mmapcnt = physmapneeded; return (NDIS_STATUS_SUCCESS); } static void NdisMFreeMapRegisters(adapter) ndis_handle adapter; { struct ndis_softc *sc; ndis_miniport_block *block; int i; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); for (i = 0; i < sc->ndis_mmapcnt; i++) bus_dmamap_destroy(sc->ndis_mtag, sc->ndis_mmaps[i]); free(sc->ndis_mmaps, M_DEVBUF); bus_dma_tag_destroy(sc->ndis_mtag); } static void ndis_mapshared_cb(arg, segs, nseg, error) void *arg; bus_dma_segment_t *segs; int nseg; int error; { ndis_physaddr *p; if (error || nseg > 1) return; p = arg; p->np_quad = segs[0].ds_addr; } /* * This maps to bus_dmamem_alloc(). */ static void NdisMAllocateSharedMemory(ndis_handle adapter, uint32_t len, uint8_t cached, void **vaddr, ndis_physaddr *paddr) { ndis_miniport_block *block; struct ndis_softc *sc; struct ndis_shmem *sh; int error; if (adapter == NULL) return; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); sh = malloc(sizeof(struct ndis_shmem), M_DEVBUF, M_NOWAIT|M_ZERO); if (sh == NULL) return; InitializeListHead(&sh->ndis_list); /* * When performing shared memory allocations, create a tag * with a lowaddr limit that restricts physical memory mappings * so that they all fall within the first 1GB of memory. * At least one device/driver combination (Linksys Instant * Wireless PCI Card V2.7, Broadcom 802.11b) seems to have * problems with performing DMA operations with physical * addresses that lie above the 1GB mark. I don't know if this * is a hardware limitation or if the addresses are being * truncated within the driver, but this seems to be the only * way to make these cards work reliably in systems with more * than 1GB of physical memory. */ error = bus_dma_tag_create(sc->ndis_parent_tag, 64, 0, NDIS_BUS_SPACE_SHARED_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, BUS_DMA_ALLOCNOW, NULL, NULL, &sh->ndis_stag); if (error) { free(sh, M_DEVBUF); return; } error = bus_dmamem_alloc(sh->ndis_stag, vaddr, BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sh->ndis_smap); if (error) { bus_dma_tag_destroy(sh->ndis_stag); free(sh, M_DEVBUF); return; } error = bus_dmamap_load(sh->ndis_stag, sh->ndis_smap, *vaddr, len, ndis_mapshared_cb, (void *)paddr, BUS_DMA_NOWAIT); if (error) { bus_dmamem_free(sh->ndis_stag, *vaddr, sh->ndis_smap); bus_dma_tag_destroy(sh->ndis_stag); free(sh, M_DEVBUF); return; } /* * Save the physical address along with the source address. * The AirGo MIMO driver will call NdisMFreeSharedMemory() * with a bogus virtual address sometimes, but with a valid * physical address. To keep this from causing trouble, we * use the physical address to as a sanity check in case * searching based on the virtual address fails. */ NDIS_LOCK(sc); sh->ndis_paddr.np_quad = paddr->np_quad; sh->ndis_saddr = *vaddr; InsertHeadList((&sc->ndis_shlist), (&sh->ndis_list)); NDIS_UNLOCK(sc); } struct ndis_allocwork { uint32_t na_len; uint8_t na_cached; void *na_ctx; io_workitem *na_iw; }; static void ndis_asyncmem_complete(dobj, arg) device_object *dobj; void *arg; { ndis_miniport_block *block; struct ndis_softc *sc; struct ndis_allocwork *w; void *vaddr; ndis_physaddr paddr; ndis_allocdone_handler donefunc; w = arg; block = (ndis_miniport_block *)dobj->do_devext; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); vaddr = NULL; paddr.np_quad = 0; donefunc = sc->ndis_chars->nmc_allocate_complete_func; NdisMAllocateSharedMemory(block, w->na_len, w->na_cached, &vaddr, &paddr); MSCALL5(donefunc, block, vaddr, &paddr, w->na_len, w->na_ctx); IoFreeWorkItem(w->na_iw); free(w, M_DEVBUF); } static ndis_status NdisMAllocateSharedMemoryAsync(ndis_handle adapter, uint32_t len, uint8_t cached, void *ctx) { ndis_miniport_block *block; struct ndis_allocwork *w; io_workitem *iw; io_workitem_func ifw; if (adapter == NULL) return (NDIS_STATUS_FAILURE); block = adapter; iw = IoAllocateWorkItem(block->nmb_deviceobj); if (iw == NULL) return (NDIS_STATUS_FAILURE); w = malloc(sizeof(struct ndis_allocwork), M_TEMP, M_NOWAIT); if (w == NULL) return (NDIS_STATUS_FAILURE); w->na_cached = cached; w->na_len = len; w->na_ctx = ctx; w->na_iw = iw; ifw = (io_workitem_func)ndis_findwrap((funcptr)ndis_asyncmem_complete); IoQueueWorkItem(iw, ifw, WORKQUEUE_DELAYED, w); return (NDIS_STATUS_PENDING); } static void NdisMFreeSharedMemory(ndis_handle adapter, uint32_t len, uint8_t cached, void *vaddr, ndis_physaddr paddr) { ndis_miniport_block *block; struct ndis_softc *sc; struct ndis_shmem *sh = NULL; list_entry *l; if (vaddr == NULL || adapter == NULL) return; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); /* Sanity check: is list empty? */ if (IsListEmpty(&sc->ndis_shlist)) return; NDIS_LOCK(sc); l = sc->ndis_shlist.nle_flink; while (l != &sc->ndis_shlist) { sh = CONTAINING_RECORD(l, struct ndis_shmem, ndis_list); if (sh->ndis_saddr == vaddr) break; /* * Check the physaddr too, just in case the driver lied * about the virtual address. */ if (sh->ndis_paddr.np_quad == paddr.np_quad) break; l = l->nle_flink; } if (sh == NULL) { NDIS_UNLOCK(sc); printf("NDIS: buggy driver tried to free " "invalid shared memory: vaddr: %p paddr: 0x%jx\n", vaddr, (uintmax_t)paddr.np_quad); return; } RemoveEntryList(&sh->ndis_list); NDIS_UNLOCK(sc); bus_dmamap_unload(sh->ndis_stag, sh->ndis_smap); bus_dmamem_free(sh->ndis_stag, sh->ndis_saddr, sh->ndis_smap); bus_dma_tag_destroy(sh->ndis_stag); free(sh, M_DEVBUF); } static ndis_status NdisMMapIoSpace(vaddr, adapter, paddr, len) void **vaddr; ndis_handle adapter; ndis_physaddr paddr; uint32_t len; { if (adapter == NULL) return (NDIS_STATUS_FAILURE); *vaddr = MmMapIoSpace(paddr.np_quad, len, 0); if (*vaddr == NULL) return (NDIS_STATUS_FAILURE); return (NDIS_STATUS_SUCCESS); } static void NdisMUnmapIoSpace(adapter, vaddr, len) ndis_handle adapter; void *vaddr; uint32_t len; { MmUnmapIoSpace(vaddr, len); } static uint32_t NdisGetCacheFillSize(void) { return (128); } static uint32_t NdisMGetDmaAlignment(handle) ndis_handle handle; { return (16); } /* * NDIS has two methods for dealing with NICs that support DMA. * One is to just pass packets to the driver and let it call * NdisMStartBufferPhysicalMapping() to map each buffer in the packet * all by itself, and the other is to let the NDIS library handle the * buffer mapping internally, and hand the driver an already populated * scatter/gather fragment list. If the driver calls * NdisMInitializeScatterGatherDma(), it wants to use the latter * method. */ static ndis_status NdisMInitializeScatterGatherDma(ndis_handle adapter, uint8_t is64, uint32_t maxphysmap) { struct ndis_softc *sc; ndis_miniport_block *block; int error; if (adapter == NULL) return (NDIS_STATUS_FAILURE); block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); /* Don't do this twice. */ if (sc->ndis_sc == 1) return (NDIS_STATUS_SUCCESS); error = bus_dma_tag_create(sc->ndis_parent_tag, ETHER_ALIGN, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES * NDIS_MAXSEG, NDIS_MAXSEG, MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->ndis_ttag); sc->ndis_sc = 1; return (NDIS_STATUS_SUCCESS); } void NdisAllocatePacketPool(status, pool, descnum, protrsvdlen) ndis_status *status; ndis_handle *pool; uint32_t descnum; uint32_t protrsvdlen; { ndis_packet_pool *p; ndis_packet *packets; int i; p = ExAllocatePoolWithTag(NonPagedPool, sizeof(ndis_packet_pool), 0); if (p == NULL) { *status = NDIS_STATUS_RESOURCES; return; } p->np_cnt = descnum + NDIS_POOL_EXTRA; p->np_protrsvd = protrsvdlen; p->np_len = sizeof(ndis_packet) + protrsvdlen; packets = ExAllocatePoolWithTag(NonPagedPool, p->np_cnt * p->np_len, 0); if (packets == NULL) { ExFreePool(p); *status = NDIS_STATUS_RESOURCES; return; } p->np_pktmem = packets; for (i = 0; i < p->np_cnt; i++) InterlockedPushEntrySList(&p->np_head, (struct slist_entry *)&packets[i]); #ifdef NDIS_DEBUG_PACKETS p->np_dead = 0; KeInitializeSpinLock(&p->np_lock); KeInitializeEvent(&p->np_event, EVENT_TYPE_NOTIFY, TRUE); #endif *pool = p; *status = NDIS_STATUS_SUCCESS; } void NdisAllocatePacketPoolEx(status, pool, descnum, oflowdescnum, protrsvdlen) ndis_status *status; ndis_handle *pool; uint32_t descnum; uint32_t oflowdescnum; uint32_t protrsvdlen; { return (NdisAllocatePacketPool(status, pool, descnum + oflowdescnum, protrsvdlen)); } uint32_t NdisPacketPoolUsage(pool) ndis_handle pool; { ndis_packet_pool *p; p = (ndis_packet_pool *)pool; return (p->np_cnt - ExQueryDepthSList(&p->np_head)); } void NdisFreePacketPool(pool) ndis_handle pool; { ndis_packet_pool *p; int usage; #ifdef NDIS_DEBUG_PACKETS uint8_t irql; #endif p = (ndis_packet_pool *)pool; #ifdef NDIS_DEBUG_PACKETS KeAcquireSpinLock(&p->np_lock, &irql); #endif usage = NdisPacketPoolUsage(pool); #ifdef NDIS_DEBUG_PACKETS if (usage) { p->np_dead = 1; KeResetEvent(&p->np_event); KeReleaseSpinLock(&p->np_lock, irql); KeWaitForSingleObject(&p->np_event, 0, 0, FALSE, NULL); } else KeReleaseSpinLock(&p->np_lock, irql); #endif ExFreePool(p->np_pktmem); ExFreePool(p); } void NdisAllocatePacket(status, packet, pool) ndis_status *status; ndis_packet **packet; ndis_handle pool; { ndis_packet_pool *p; ndis_packet *pkt; #ifdef NDIS_DEBUG_PACKETS uint8_t irql; #endif p = (ndis_packet_pool *)pool; #ifdef NDIS_DEBUG_PACKETS KeAcquireSpinLock(&p->np_lock, &irql); if (p->np_dead) { KeReleaseSpinLock(&p->np_lock, irql); printf("NDIS: tried to allocate packet from dead pool %p\n", pool); *status = NDIS_STATUS_RESOURCES; return; } #endif pkt = (ndis_packet *)InterlockedPopEntrySList(&p->np_head); #ifdef NDIS_DEBUG_PACKETS KeReleaseSpinLock(&p->np_lock, irql); #endif if (pkt == NULL) { *status = NDIS_STATUS_RESOURCES; return; } bzero((char *)pkt, sizeof(ndis_packet)); /* Save pointer to the pool. */ pkt->np_private.npp_pool = pool; /* Set the oob offset pointer. Lots of things expect this. */ pkt->np_private.npp_packetooboffset = offsetof(ndis_packet, np_oob); /* * We must initialize the packet flags correctly in order * for the NDIS_SET_PACKET_MEDIA_SPECIFIC_INFO() and * NDIS_GET_PACKET_MEDIA_SPECIFIC_INFO() macros to work * correctly. */ pkt->np_private.npp_ndispktflags = NDIS_PACKET_ALLOCATED_BY_NDIS; pkt->np_private.npp_validcounts = FALSE; *packet = pkt; *status = NDIS_STATUS_SUCCESS; } void NdisFreePacket(packet) ndis_packet *packet; { ndis_packet_pool *p; #ifdef NDIS_DEBUG_PACKETS uint8_t irql; #endif p = (ndis_packet_pool *)packet->np_private.npp_pool; #ifdef NDIS_DEBUG_PACKETS KeAcquireSpinLock(&p->np_lock, &irql); #endif InterlockedPushEntrySList(&p->np_head, (slist_entry *)packet); #ifdef NDIS_DEBUG_PACKETS if (p->np_dead) { if (ExQueryDepthSList(&p->np_head) == p->np_cnt) KeSetEvent(&p->np_event, IO_NO_INCREMENT, FALSE); } KeReleaseSpinLock(&p->np_lock, irql); #endif } static void NdisUnchainBufferAtFront(packet, buf) ndis_packet *packet; ndis_buffer **buf; { ndis_packet_private *priv; if (packet == NULL || buf == NULL) return; priv = &packet->np_private; priv->npp_validcounts = FALSE; if (priv->npp_head == priv->npp_tail) { *buf = priv->npp_head; priv->npp_head = priv->npp_tail = NULL; } else { *buf = priv->npp_head; priv->npp_head = (*buf)->mdl_next; } } static void NdisUnchainBufferAtBack(packet, buf) ndis_packet *packet; ndis_buffer **buf; { ndis_packet_private *priv; ndis_buffer *tmp; if (packet == NULL || buf == NULL) return; priv = &packet->np_private; priv->npp_validcounts = FALSE; if (priv->npp_head == priv->npp_tail) { *buf = priv->npp_head; priv->npp_head = priv->npp_tail = NULL; } else { *buf = priv->npp_tail; tmp = priv->npp_head; while (tmp->mdl_next != priv->npp_tail) tmp = tmp->mdl_next; priv->npp_tail = tmp; tmp->mdl_next = NULL; } } /* * The NDIS "buffer" is really an MDL (memory descriptor list) * which is used to describe a buffer in a way that allows it * to mapped into different contexts. We have to be careful how * we handle them: in some versions of Windows, the NdisFreeBuffer() * routine is an actual function in the NDIS API, but in others * it's just a macro wrapper around IoFreeMdl(). There's really * no way to use the 'descnum' parameter to count how many * "buffers" are allocated since in order to use IoFreeMdl() to * dispose of a buffer, we have to use IoAllocateMdl() to allocate * them, and IoAllocateMdl() just grabs them out of the heap. */ static void NdisAllocateBufferPool(status, pool, descnum) ndis_status *status; ndis_handle *pool; uint32_t descnum; { /* * The only thing we can really do here is verify that descnum * is a reasonable value, but I really don't know what to check * it against. */ *pool = NonPagedPool; *status = NDIS_STATUS_SUCCESS; } static void NdisFreeBufferPool(pool) ndis_handle pool; { } static void NdisAllocateBuffer(status, buffer, pool, vaddr, len) ndis_status *status; ndis_buffer **buffer; ndis_handle pool; void *vaddr; uint32_t len; { ndis_buffer *buf; buf = IoAllocateMdl(vaddr, len, FALSE, FALSE, NULL); if (buf == NULL) { *status = NDIS_STATUS_RESOURCES; return; } MmBuildMdlForNonPagedPool(buf); *buffer = buf; *status = NDIS_STATUS_SUCCESS; } static void NdisFreeBuffer(buf) ndis_buffer *buf; { IoFreeMdl(buf); } /* Aw c'mon. */ static uint32_t NdisBufferLength(buf) ndis_buffer *buf; { return (MmGetMdlByteCount(buf)); } /* * Get the virtual address and length of a buffer. * Note: the vaddr argument is optional. */ static void NdisQueryBuffer(buf, vaddr, len) ndis_buffer *buf; void **vaddr; uint32_t *len; { if (vaddr != NULL) *vaddr = MmGetMdlVirtualAddress(buf); *len = MmGetMdlByteCount(buf); } /* Same as above -- we don't care about the priority. */ static void NdisQueryBufferSafe(buf, vaddr, len, prio) ndis_buffer *buf; void **vaddr; uint32_t *len; uint32_t prio; { if (vaddr != NULL) *vaddr = MmGetMdlVirtualAddress(buf); *len = MmGetMdlByteCount(buf); } /* Damnit Microsoft!! How many ways can you do the same thing?! */ static void * NdisBufferVirtualAddress(buf) ndis_buffer *buf; { return (MmGetMdlVirtualAddress(buf)); } static void * NdisBufferVirtualAddressSafe(buf, prio) ndis_buffer *buf; uint32_t prio; { return (MmGetMdlVirtualAddress(buf)); } static void NdisAdjustBufferLength(buf, len) ndis_buffer *buf; int len; { MmGetMdlByteCount(buf) = len; } static uint32_t NdisInterlockedIncrement(addend) uint32_t *addend; { atomic_add_long((u_long *)addend, 1); return (*addend); } static uint32_t NdisInterlockedDecrement(addend) uint32_t *addend; { atomic_subtract_long((u_long *)addend, 1); return (*addend); } static void NdisInitializeEvent(event) ndis_event *event; { /* * NDIS events are always notification * events, and should be initialized to the * not signaled state. */ KeInitializeEvent(&event->ne_event, EVENT_TYPE_NOTIFY, FALSE); } static void NdisSetEvent(event) ndis_event *event; { KeSetEvent(&event->ne_event, IO_NO_INCREMENT, FALSE); } static void NdisResetEvent(event) ndis_event *event; { KeResetEvent(&event->ne_event); } static uint8_t NdisWaitEvent(event, msecs) ndis_event *event; uint32_t msecs; { int64_t duetime; uint32_t rval; duetime = ((int64_t)msecs * -10000); rval = KeWaitForSingleObject(event, 0, 0, TRUE, msecs ? & duetime : NULL); if (rval == STATUS_TIMEOUT) return (FALSE); return (TRUE); } static ndis_status NdisUnicodeStringToAnsiString(dstr, sstr) ansi_string *dstr; unicode_string *sstr; { uint32_t rval; rval = RtlUnicodeStringToAnsiString(dstr, sstr, FALSE); if (rval == STATUS_INSUFFICIENT_RESOURCES) return (NDIS_STATUS_RESOURCES); if (rval) return (NDIS_STATUS_FAILURE); return (NDIS_STATUS_SUCCESS); } static ndis_status NdisAnsiStringToUnicodeString(dstr, sstr) unicode_string *dstr; ansi_string *sstr; { uint32_t rval; rval = RtlAnsiStringToUnicodeString(dstr, sstr, FALSE); if (rval == STATUS_INSUFFICIENT_RESOURCES) return (NDIS_STATUS_RESOURCES); if (rval) return (NDIS_STATUS_FAILURE); return (NDIS_STATUS_SUCCESS); } static ndis_status NdisMPciAssignResources(adapter, slot, list) ndis_handle adapter; uint32_t slot; ndis_resource_list **list; { ndis_miniport_block *block; if (adapter == NULL || list == NULL) return (NDIS_STATUS_FAILURE); block = (ndis_miniport_block *)adapter; *list = block->nmb_rlist; return (NDIS_STATUS_SUCCESS); } static uint8_t ndis_intr(iobj, arg) kinterrupt *iobj; void *arg; { struct ndis_softc *sc; uint8_t is_our_intr = FALSE; int call_isr = 0; ndis_miniport_interrupt *intr; sc = arg; intr = sc->ndis_block->nmb_interrupt; if (intr == NULL || sc->ndis_block->nmb_miniportadapterctx == NULL) return (FALSE); if (sc->ndis_block->nmb_interrupt->ni_isrreq == TRUE) MSCALL3(intr->ni_isrfunc, &is_our_intr, &call_isr, sc->ndis_block->nmb_miniportadapterctx); else { MSCALL1(sc->ndis_chars->nmc_disable_interrupts_func, sc->ndis_block->nmb_miniportadapterctx); call_isr = 1; } if (call_isr) IoRequestDpc(sc->ndis_block->nmb_deviceobj, NULL, sc); return (is_our_intr); } static void ndis_intrhand(dpc, intr, sysarg1, sysarg2) kdpc *dpc; ndis_miniport_interrupt *intr; void *sysarg1; void *sysarg2; { struct ndis_softc *sc; ndis_miniport_block *block; ndis_handle adapter; block = intr->ni_block; adapter = block->nmb_miniportadapterctx; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); if (NDIS_SERIALIZED(sc->ndis_block)) KeAcquireSpinLockAtDpcLevel(&block->nmb_lock); MSCALL1(intr->ni_dpcfunc, adapter); /* If there's a MiniportEnableInterrupt() routine, call it. */ if (sc->ndis_chars->nmc_enable_interrupts_func != NULL) MSCALL1(sc->ndis_chars->nmc_enable_interrupts_func, adapter); if (NDIS_SERIALIZED(sc->ndis_block)) KeReleaseSpinLockFromDpcLevel(&block->nmb_lock); /* * Set the completion event if we've drained all * pending interrupts. */ KeAcquireSpinLockAtDpcLevel(&intr->ni_dpccountlock); intr->ni_dpccnt--; if (intr->ni_dpccnt == 0) KeSetEvent(&intr->ni_dpcevt, IO_NO_INCREMENT, FALSE); KeReleaseSpinLockFromDpcLevel(&intr->ni_dpccountlock); } static ndis_status NdisMRegisterInterrupt(ndis_miniport_interrupt *intr, ndis_handle adapter, uint32_t ivec, uint32_t ilevel, uint8_t reqisr, uint8_t shared, ndis_interrupt_mode imode) { ndis_miniport_block *block; ndis_miniport_characteristics *ch; struct ndis_softc *sc; int error; block = adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); ch = IoGetDriverObjectExtension(block->nmb_deviceobj->do_drvobj, (void *)1); intr->ni_rsvd = ExAllocatePoolWithTag(NonPagedPool, sizeof(struct mtx), 0); if (intr->ni_rsvd == NULL) return (NDIS_STATUS_RESOURCES); intr->ni_block = adapter; intr->ni_isrreq = reqisr; intr->ni_shared = shared; intr->ni_dpccnt = 0; intr->ni_isrfunc = ch->nmc_isr_func; intr->ni_dpcfunc = ch->nmc_interrupt_func; KeInitializeEvent(&intr->ni_dpcevt, EVENT_TYPE_NOTIFY, TRUE); KeInitializeDpc(&intr->ni_dpc, ndis_findwrap((funcptr)ndis_intrhand), intr); KeSetImportanceDpc(&intr->ni_dpc, KDPC_IMPORTANCE_LOW); error = IoConnectInterrupt(&intr->ni_introbj, ndis_findwrap((funcptr)ndis_intr), sc, NULL, ivec, ilevel, 0, imode, shared, 0, FALSE); if (error != STATUS_SUCCESS) return (NDIS_STATUS_FAILURE); block->nmb_interrupt = intr; return (NDIS_STATUS_SUCCESS); } static void NdisMDeregisterInterrupt(intr) ndis_miniport_interrupt *intr; { ndis_miniport_block *block; uint8_t irql; block = intr->ni_block; /* Should really be KeSynchronizeExecution() */ KeAcquireSpinLock(intr->ni_introbj->ki_lock, &irql); block->nmb_interrupt = NULL; KeReleaseSpinLock(intr->ni_introbj->ki_lock, irql); /* KeFlushQueuedDpcs(); */ /* Disconnect our ISR */ IoDisconnectInterrupt(intr->ni_introbj); KeWaitForSingleObject(&intr->ni_dpcevt, 0, 0, FALSE, NULL); KeResetEvent(&intr->ni_dpcevt); } static void NdisMRegisterAdapterShutdownHandler(adapter, shutdownctx, shutdownfunc) ndis_handle adapter; void *shutdownctx; ndis_shutdown_handler shutdownfunc; { ndis_miniport_block *block; ndis_miniport_characteristics *chars; struct ndis_softc *sc; if (adapter == NULL) return; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); chars = sc->ndis_chars; chars->nmc_shutdown_handler = shutdownfunc; chars->nmc_rsvd0 = shutdownctx; } static void NdisMDeregisterAdapterShutdownHandler(adapter) ndis_handle adapter; { ndis_miniport_block *block; ndis_miniport_characteristics *chars; struct ndis_softc *sc; if (adapter == NULL) return; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); chars = sc->ndis_chars; chars->nmc_shutdown_handler = NULL; chars->nmc_rsvd0 = NULL; } static uint32_t NDIS_BUFFER_TO_SPAN_PAGES(buf) ndis_buffer *buf; { if (buf == NULL) return (0); if (MmGetMdlByteCount(buf) == 0) return (1); return (SPAN_PAGES(MmGetMdlVirtualAddress(buf), MmGetMdlByteCount(buf))); } static void NdisGetBufferPhysicalArraySize(buf, pages) ndis_buffer *buf; uint32_t *pages; { if (buf == NULL) return; *pages = NDIS_BUFFER_TO_SPAN_PAGES(buf); } static void NdisQueryBufferOffset(buf, off, len) ndis_buffer *buf; uint32_t *off; uint32_t *len; { if (buf == NULL) return; *off = MmGetMdlByteOffset(buf); *len = MmGetMdlByteCount(buf); } void NdisMSleep(usecs) uint32_t usecs; { ktimer timer; /* * During system bootstrap, (i.e. cold == 1), we aren't * allowed to sleep, so we have to do a hard DELAY() * instead. */ if (cold) DELAY(usecs); else { KeInitializeTimer(&timer); KeSetTimer(&timer, ((int64_t)usecs * -10), NULL); KeWaitForSingleObject(&timer, 0, 0, FALSE, NULL); } } static uint32_t NdisReadPcmciaAttributeMemory(handle, offset, buf, len) ndis_handle handle; uint32_t offset; void *buf; uint32_t len; { struct ndis_softc *sc; ndis_miniport_block *block; bus_space_handle_t bh; bus_space_tag_t bt; char *dest; int i; if (handle == NULL) return (0); block = (ndis_miniport_block *)handle; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); dest = buf; bh = rman_get_bushandle(sc->ndis_res_am); bt = rman_get_bustag(sc->ndis_res_am); for (i = 0; i < len; i++) dest[i] = bus_space_read_1(bt, bh, (offset + i) * 2); return (i); } static uint32_t NdisWritePcmciaAttributeMemory(handle, offset, buf, len) ndis_handle handle; uint32_t offset; void *buf; uint32_t len; { struct ndis_softc *sc; ndis_miniport_block *block; bus_space_handle_t bh; bus_space_tag_t bt; char *src; int i; if (handle == NULL) return (0); block = (ndis_miniport_block *)handle; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); src = buf; bh = rman_get_bushandle(sc->ndis_res_am); bt = rman_get_bustag(sc->ndis_res_am); for (i = 0; i < len; i++) bus_space_write_1(bt, bh, (offset + i) * 2, src[i]); return (i); } static list_entry * NdisInterlockedInsertHeadList(head, entry, lock) list_entry *head; list_entry *entry; ndis_spin_lock *lock; { list_entry *flink; KeAcquireSpinLock(&lock->nsl_spinlock, &lock->nsl_kirql); flink = head->nle_flink; entry->nle_flink = flink; entry->nle_blink = head; flink->nle_blink = entry; head->nle_flink = entry; KeReleaseSpinLock(&lock->nsl_spinlock, lock->nsl_kirql); return (flink); } static list_entry * NdisInterlockedRemoveHeadList(head, lock) list_entry *head; ndis_spin_lock *lock; { list_entry *flink; list_entry *entry; KeAcquireSpinLock(&lock->nsl_spinlock, &lock->nsl_kirql); entry = head->nle_flink; flink = entry->nle_flink; head->nle_flink = flink; flink->nle_blink = head; KeReleaseSpinLock(&lock->nsl_spinlock, lock->nsl_kirql); return (entry); } static list_entry * NdisInterlockedInsertTailList(head, entry, lock) list_entry *head; list_entry *entry; ndis_spin_lock *lock; { list_entry *blink; KeAcquireSpinLock(&lock->nsl_spinlock, &lock->nsl_kirql); blink = head->nle_blink; entry->nle_flink = head; entry->nle_blink = blink; blink->nle_flink = entry; head->nle_blink = entry; KeReleaseSpinLock(&lock->nsl_spinlock, lock->nsl_kirql); return (blink); } static uint8_t NdisMSynchronizeWithInterrupt(intr, syncfunc, syncctx) ndis_miniport_interrupt *intr; void *syncfunc; void *syncctx; { return (KeSynchronizeExecution(intr->ni_introbj, syncfunc, syncctx)); } static void NdisGetCurrentSystemTime(tval) uint64_t *tval; { ntoskrnl_time(tval); } /* * Return the number of milliseconds since the system booted. */ static void NdisGetSystemUpTime(tval) uint32_t *tval; { struct timespec ts; nanouptime(&ts); *tval = ts.tv_nsec / 1000000 + ts.tv_sec * 1000; } static void NdisInitializeString(dst, src) unicode_string *dst; char *src; { ansi_string as; RtlInitAnsiString(&as, src); RtlAnsiStringToUnicodeString(dst, &as, TRUE); } static void NdisFreeString(str) unicode_string *str; { RtlFreeUnicodeString(str); } static ndis_status NdisMRemoveMiniport(adapter) ndis_handle *adapter; { return (NDIS_STATUS_SUCCESS); } static void NdisInitAnsiString(dst, src) ansi_string *dst; char *src; { RtlInitAnsiString(dst, src); } static void NdisInitUnicodeString(dst, src) unicode_string *dst; uint16_t *src; { RtlInitUnicodeString(dst, src); } static void NdisMGetDeviceProperty(adapter, phydevobj, funcdevobj, nextdevobj, resources, transresources) ndis_handle adapter; device_object **phydevobj; device_object **funcdevobj; device_object **nextdevobj; cm_resource_list *resources; cm_resource_list *transresources; { ndis_miniport_block *block; block = (ndis_miniport_block *)adapter; if (phydevobj != NULL) *phydevobj = block->nmb_physdeviceobj; if (funcdevobj != NULL) *funcdevobj = block->nmb_deviceobj; if (nextdevobj != NULL) *nextdevobj = block->nmb_nextdeviceobj; } static void NdisGetFirstBufferFromPacket(packet, buf, firstva, firstlen, totlen) ndis_packet *packet; ndis_buffer **buf; void **firstva; uint32_t *firstlen; uint32_t *totlen; { ndis_buffer *tmp; tmp = packet->np_private.npp_head; *buf = tmp; if (tmp == NULL) { *firstva = NULL; *firstlen = *totlen = 0; } else { *firstva = MmGetMdlVirtualAddress(tmp); *firstlen = *totlen = MmGetMdlByteCount(tmp); for (tmp = tmp->mdl_next; tmp != NULL; tmp = tmp->mdl_next) *totlen += MmGetMdlByteCount(tmp); } } static void NdisGetFirstBufferFromPacketSafe(packet, buf, firstva, firstlen, totlen, prio) ndis_packet *packet; ndis_buffer **buf; void **firstva; uint32_t *firstlen; uint32_t *totlen; uint32_t prio; { NdisGetFirstBufferFromPacket(packet, buf, firstva, firstlen, totlen); } static int ndis_find_sym(lf, filename, suffix, sym) linker_file_t lf; char *filename; char *suffix; caddr_t *sym; { char *fullsym; char *suf; int i; fullsym = ExAllocatePoolWithTag(NonPagedPool, MAXPATHLEN, 0); if (fullsym == NULL) return (ENOMEM); bzero(fullsym, MAXPATHLEN); strncpy(fullsym, filename, MAXPATHLEN); if (strlen(filename) < 4) { ExFreePool(fullsym); return (EINVAL); } /* If the filename has a .ko suffix, strip if off. */ suf = fullsym + (strlen(filename) - 3); if (strcmp(suf, ".ko") == 0) *suf = '\0'; for (i = 0; i < strlen(fullsym); i++) { if (fullsym[i] == '.') fullsym[i] = '_'; else fullsym[i] = tolower(fullsym[i]); } strcat(fullsym, suffix); *sym = linker_file_lookup_symbol(lf, fullsym, 0); ExFreePool(fullsym); if (*sym == 0) return (ENOENT); return (0); } struct ndis_checkmodule { char *afilename; ndis_fh *fh; }; /* * See if a single module contains the symbols for a specified file. */ static int NdisCheckModule(linker_file_t lf, void *context) { struct ndis_checkmodule *nc; caddr_t kldstart, kldend; nc = (struct ndis_checkmodule *)context; if (ndis_find_sym(lf, nc->afilename, "_start", &kldstart)) return (0); if (ndis_find_sym(lf, nc->afilename, "_end", &kldend)) return (0); nc->fh->nf_vp = lf; nc->fh->nf_map = NULL; nc->fh->nf_type = NDIS_FH_TYPE_MODULE; nc->fh->nf_maplen = (kldend - kldstart) & 0xFFFFFFFF; return (1); } /* can also return NDIS_STATUS_RESOURCES/NDIS_STATUS_ERROR_READING_FILE */ static void NdisOpenFile(status, filehandle, filelength, filename, highestaddr) ndis_status *status; ndis_handle *filehandle; uint32_t *filelength; unicode_string *filename; ndis_physaddr highestaddr; { ansi_string as; char *afilename = NULL; struct thread *td = curthread; struct nameidata nd; int flags, error, vfslocked; struct vattr vat; struct vattr *vap = &vat; ndis_fh *fh; char *path; struct ndis_checkmodule nc; if (RtlUnicodeStringToAnsiString(&as, filename, TRUE)) { *status = NDIS_STATUS_RESOURCES; return; } afilename = strdup(as.as_buf, M_DEVBUF); RtlFreeAnsiString(&as); fh = ExAllocatePoolWithTag(NonPagedPool, sizeof(ndis_fh), 0); if (fh == NULL) { free(afilename, M_DEVBUF); *status = NDIS_STATUS_RESOURCES; return; } fh->nf_name = afilename; /* * During system bootstrap, it's impossible to load files * from the rootfs since it's not mounted yet. We therefore * offer the possibility of opening files that have been * preloaded as modules instead. Both choices will work * when kldloading a module from multiuser, but only the * module option will work during bootstrap. The module * loading option works by using the ndiscvt(8) utility * to convert the arbitrary file into a .ko using objcopy(1). * This file will contain two special symbols: filename_start * and filename_end. All we have to do is traverse the KLD * list in search of those symbols and we've found the file * data. As an added bonus, ndiscvt(8) will also generate * a normal .o file which can be linked statically with * the kernel. This means that the symbols will actual reside * in the kernel's symbol table, but that doesn't matter to * us since the kernel appears to us as just another module. */ nc.afilename = afilename; nc.fh = fh; if (linker_file_foreach(NdisCheckModule, &nc)) { *filelength = fh->nf_maplen; *filehandle = fh; *status = NDIS_STATUS_SUCCESS; return; } if (TAILQ_EMPTY(&mountlist)) { ExFreePool(fh); *status = NDIS_STATUS_FILE_NOT_FOUND; printf("NDIS: could not find file %s in linker list\n", afilename); printf("NDIS: and no filesystems mounted yet, " "aborting NdisOpenFile()\n"); free(afilename, M_DEVBUF); return; } path = ExAllocatePoolWithTag(NonPagedPool, MAXPATHLEN, 0); if (path == NULL) { ExFreePool(fh); free(afilename, M_DEVBUF); *status = NDIS_STATUS_RESOURCES; return; } snprintf(path, MAXPATHLEN, "%s/%s", ndis_filepath, afilename); /* Some threads don't have a current working directory. */ if (td->td_proc->p_fd->fd_rdir == NULL) td->td_proc->p_fd->fd_rdir = rootvnode; if (td->td_proc->p_fd->fd_cdir == NULL) td->td_proc->p_fd->fd_cdir = rootvnode; NDINIT(&nd, LOOKUP, FOLLOW | MPSAFE, UIO_SYSSPACE, path, td); flags = FREAD; error = vn_open(&nd, &flags, 0, NULL); if (error) { *status = NDIS_STATUS_FILE_NOT_FOUND; ExFreePool(fh); printf("NDIS: open file %s failed: %d\n", path, error); ExFreePool(path); free(afilename, M_DEVBUF); return; } vfslocked = NDHASGIANT(&nd); ExFreePool(path); NDFREE(&nd, NDF_ONLY_PNBUF); /* Get the file size. */ VOP_GETATTR(nd.ni_vp, vap, td->td_ucred); VOP_UNLOCK(nd.ni_vp, 0); VFS_UNLOCK_GIANT(vfslocked); fh->nf_vp = nd.ni_vp; fh->nf_map = NULL; fh->nf_type = NDIS_FH_TYPE_VFS; *filehandle = fh; *filelength = fh->nf_maplen = vap->va_size & 0xFFFFFFFF; *status = NDIS_STATUS_SUCCESS; } static void NdisMapFile(status, mappedbuffer, filehandle) ndis_status *status; void **mappedbuffer; ndis_handle filehandle; { ndis_fh *fh; struct thread *td = curthread; linker_file_t lf; caddr_t kldstart; int error, resid, vfslocked; struct vnode *vp; if (filehandle == NULL) { *status = NDIS_STATUS_FAILURE; return; } fh = (ndis_fh *)filehandle; if (fh->nf_vp == NULL) { *status = NDIS_STATUS_FAILURE; return; } if (fh->nf_map != NULL) { *status = NDIS_STATUS_ALREADY_MAPPED; return; } if (fh->nf_type == NDIS_FH_TYPE_MODULE) { lf = fh->nf_vp; if (ndis_find_sym(lf, fh->nf_name, "_start", &kldstart)) { *status = NDIS_STATUS_FAILURE; return; } fh->nf_map = kldstart; *status = NDIS_STATUS_SUCCESS; *mappedbuffer = fh->nf_map; return; } fh->nf_map = ExAllocatePoolWithTag(NonPagedPool, fh->nf_maplen, 0); if (fh->nf_map == NULL) { *status = NDIS_STATUS_RESOURCES; return; } vp = fh->nf_vp; vfslocked = VFS_LOCK_GIANT(vp->v_mount); error = vn_rdwr(UIO_READ, vp, fh->nf_map, fh->nf_maplen, 0, UIO_SYSSPACE, 0, td->td_ucred, NOCRED, &resid, td); VFS_UNLOCK_GIANT(vfslocked); if (error) *status = NDIS_STATUS_FAILURE; else { *status = NDIS_STATUS_SUCCESS; *mappedbuffer = fh->nf_map; } } static void NdisUnmapFile(filehandle) ndis_handle filehandle; { ndis_fh *fh; fh = (ndis_fh *)filehandle; if (fh->nf_map == NULL) return; if (fh->nf_type == NDIS_FH_TYPE_VFS) ExFreePool(fh->nf_map); fh->nf_map = NULL; } static void NdisCloseFile(filehandle) ndis_handle filehandle; { struct thread *td = curthread; ndis_fh *fh; int vfslocked; struct vnode *vp; if (filehandle == NULL) return; fh = (ndis_fh *)filehandle; if (fh->nf_map != NULL) { if (fh->nf_type == NDIS_FH_TYPE_VFS) ExFreePool(fh->nf_map); fh->nf_map = NULL; } if (fh->nf_vp == NULL) return; if (fh->nf_type == NDIS_FH_TYPE_VFS) { vp = fh->nf_vp; vfslocked = VFS_LOCK_GIANT(vp->v_mount); vn_close(vp, FREAD, td->td_ucred, td); VFS_UNLOCK_GIANT(vfslocked); } fh->nf_vp = NULL; free(fh->nf_name, M_DEVBUF); ExFreePool(fh); } static uint8_t NdisSystemProcessorCount() { return (mp_ncpus); } typedef void (*ndis_statusdone_handler)(ndis_handle); typedef void (*ndis_status_handler)(ndis_handle, ndis_status, void *, uint32_t); static void NdisMIndicateStatusComplete(adapter) ndis_handle adapter; { ndis_miniport_block *block; ndis_statusdone_handler statusdonefunc; block = (ndis_miniport_block *)adapter; statusdonefunc = block->nmb_statusdone_func; MSCALL1(statusdonefunc, adapter); } static void NdisMIndicateStatus(adapter, status, sbuf, slen) ndis_handle adapter; ndis_status status; void *sbuf; uint32_t slen; { ndis_miniport_block *block; ndis_status_handler statusfunc; block = (ndis_miniport_block *)adapter; statusfunc = block->nmb_status_func; MSCALL4(statusfunc, adapter, status, sbuf, slen); } /* * The DDK documentation says that you should use IoQueueWorkItem() * instead of ExQueueWorkItem(). The problem is, IoQueueWorkItem() * is fundamentally incompatible with NdisScheduleWorkItem(), which * depends on the API semantics of ExQueueWorkItem(). In our world, * ExQueueWorkItem() is implemented on top of IoAllocateQueueItem() * anyway. * * There are actually three distinct APIs here. NdisScheduleWorkItem() * takes a pointer to an NDIS_WORK_ITEM. ExQueueWorkItem() takes a pointer * to a WORK_QUEUE_ITEM. And finally, IoQueueWorkItem() takes a pointer * to an opaque work item thingie which you get from IoAllocateWorkItem(). * An NDIS_WORK_ITEM is not the same as a WORK_QUEUE_ITEM. However, * the NDIS_WORK_ITEM has some opaque storage at the end of it, and we * (ab)use this storage as a WORK_QUEUE_ITEM, which is what we submit * to ExQueueWorkItem(). * * Got all that? (Sheesh.) */ ndis_status NdisScheduleWorkItem(work) ndis_work_item *work; { work_queue_item *wqi; wqi = (work_queue_item *)work->nwi_wraprsvd; ExInitializeWorkItem(wqi, (work_item_func)work->nwi_func, work->nwi_ctx); ExQueueWorkItem(wqi, WORKQUEUE_DELAYED); return (NDIS_STATUS_SUCCESS); } static void NdisCopyFromPacketToPacket(dpkt, doff, reqlen, spkt, soff, cpylen) ndis_packet *dpkt; uint32_t doff; uint32_t reqlen; ndis_packet *spkt; uint32_t soff; uint32_t *cpylen; { ndis_buffer *src, *dst; char *sptr, *dptr; int resid, copied, len, scnt, dcnt; *cpylen = 0; src = spkt->np_private.npp_head; dst = dpkt->np_private.npp_head; sptr = MmGetMdlVirtualAddress(src); dptr = MmGetMdlVirtualAddress(dst); scnt = MmGetMdlByteCount(src); dcnt = MmGetMdlByteCount(dst); while (soff) { if (MmGetMdlByteCount(src) > soff) { sptr += soff; scnt = MmGetMdlByteCount(src)- soff; break; } soff -= MmGetMdlByteCount(src); src = src->mdl_next; if (src == NULL) return; sptr = MmGetMdlVirtualAddress(src); } while (doff) { if (MmGetMdlByteCount(dst) > doff) { dptr += doff; dcnt = MmGetMdlByteCount(dst) - doff; break; } doff -= MmGetMdlByteCount(dst); dst = dst->mdl_next; if (dst == NULL) return; dptr = MmGetMdlVirtualAddress(dst); } resid = reqlen; copied = 0; while(1) { if (resid < scnt) len = resid; else len = scnt; if (dcnt < len) len = dcnt; bcopy(sptr, dptr, len); copied += len; resid -= len; if (resid == 0) break; dcnt -= len; if (dcnt == 0) { dst = dst->mdl_next; if (dst == NULL) break; dptr = MmGetMdlVirtualAddress(dst); dcnt = MmGetMdlByteCount(dst); } scnt -= len; if (scnt == 0) { src = src->mdl_next; if (src == NULL) break; sptr = MmGetMdlVirtualAddress(src); scnt = MmGetMdlByteCount(src); } } *cpylen = copied; } static void NdisCopyFromPacketToPacketSafe(dpkt, doff, reqlen, spkt, soff, cpylen, prio) ndis_packet *dpkt; uint32_t doff; uint32_t reqlen; ndis_packet *spkt; uint32_t soff; uint32_t *cpylen; uint32_t prio; { NdisCopyFromPacketToPacket(dpkt, doff, reqlen, spkt, soff, cpylen); } static void NdisIMCopySendPerPacketInfo(dpkt, spkt) ndis_packet *dpkt; ndis_packet *spkt; { memcpy(&dpkt->np_ext, &spkt->np_ext, sizeof(ndis_packet_extension)); } static ndis_status NdisMRegisterDevice(handle, devname, symname, majorfuncs, devobj, devhandle) ndis_handle handle; unicode_string *devname; unicode_string *symname; driver_dispatch *majorfuncs[]; void **devobj; ndis_handle *devhandle; { uint32_t status; device_object *dobj; status = IoCreateDevice(handle, 0, devname, FILE_DEVICE_UNKNOWN, 0, FALSE, &dobj); if (status == STATUS_SUCCESS) { *devobj = dobj; *devhandle = dobj; } return (status); } static ndis_status NdisMDeregisterDevice(handle) ndis_handle handle; { IoDeleteDevice(handle); return (NDIS_STATUS_SUCCESS); } static ndis_status NdisMQueryAdapterInstanceName(name, handle) unicode_string *name; ndis_handle handle; { ndis_miniport_block *block; device_t dev; ansi_string as; block = (ndis_miniport_block *)handle; dev = block->nmb_physdeviceobj->do_devext; RtlInitAnsiString(&as, __DECONST(char *, device_get_nameunit(dev))); if (RtlAnsiStringToUnicodeString(name, &as, TRUE)) return (NDIS_STATUS_RESOURCES); return (NDIS_STATUS_SUCCESS); } static void NdisMRegisterUnloadHandler(handle, func) ndis_handle handle; void *func; { } static void dummy() { printf("NDIS dummy called...\n"); } /* * Note: a couple of entries in this table specify the * number of arguments as "foo + 1". These are routines * that accept a 64-bit argument, passed by value. On * x86, these arguments consume two longwords on the stack, * so we lie and say there's one additional argument so * that the wrapping routines will do the right thing. */ image_patch_table ndis_functbl[] = { IMPORT_SFUNC(NdisCopyFromPacketToPacket, 6), IMPORT_SFUNC(NdisCopyFromPacketToPacketSafe, 7), IMPORT_SFUNC(NdisIMCopySendPerPacketInfo, 2), IMPORT_SFUNC(NdisScheduleWorkItem, 1), IMPORT_SFUNC(NdisMIndicateStatusComplete, 1), IMPORT_SFUNC(NdisMIndicateStatus, 4), IMPORT_SFUNC(NdisSystemProcessorCount, 0), IMPORT_SFUNC(NdisUnchainBufferAtBack, 2), IMPORT_SFUNC(NdisGetFirstBufferFromPacket, 5), IMPORT_SFUNC(NdisGetFirstBufferFromPacketSafe, 6), IMPORT_SFUNC(NdisGetBufferPhysicalArraySize, 2), IMPORT_SFUNC(NdisMGetDeviceProperty, 6), IMPORT_SFUNC(NdisInitAnsiString, 2), IMPORT_SFUNC(NdisInitUnicodeString, 2), IMPORT_SFUNC(NdisWriteConfiguration, 4), IMPORT_SFUNC(NdisAnsiStringToUnicodeString, 2), IMPORT_SFUNC(NdisTerminateWrapper, 2), IMPORT_SFUNC(NdisOpenConfigurationKeyByName, 4), IMPORT_SFUNC(NdisOpenConfigurationKeyByIndex, 5), IMPORT_SFUNC(NdisMRemoveMiniport, 1), IMPORT_SFUNC(NdisInitializeString, 2), IMPORT_SFUNC(NdisFreeString, 1), IMPORT_SFUNC(NdisGetCurrentSystemTime, 1), IMPORT_SFUNC(NdisGetSystemUpTime, 1), IMPORT_SFUNC(NdisMSynchronizeWithInterrupt, 3), IMPORT_SFUNC(NdisMAllocateSharedMemoryAsync, 4), IMPORT_SFUNC(NdisInterlockedInsertHeadList, 3), IMPORT_SFUNC(NdisInterlockedInsertTailList, 3), IMPORT_SFUNC(NdisInterlockedRemoveHeadList, 2), IMPORT_SFUNC(NdisInitializeWrapper, 4), IMPORT_SFUNC(NdisMRegisterMiniport, 3), IMPORT_SFUNC(NdisAllocateMemoryWithTag, 3), IMPORT_SFUNC(NdisAllocateMemory, 4 + 1), IMPORT_SFUNC(NdisMSetAttributesEx, 5), IMPORT_SFUNC(NdisCloseConfiguration, 1), IMPORT_SFUNC(NdisReadConfiguration, 5), IMPORT_SFUNC(NdisOpenConfiguration, 3), IMPORT_SFUNC(NdisAcquireSpinLock, 1), IMPORT_SFUNC(NdisReleaseSpinLock, 1), IMPORT_SFUNC(NdisDprAcquireSpinLock, 1), IMPORT_SFUNC(NdisDprReleaseSpinLock, 1), IMPORT_SFUNC(NdisAllocateSpinLock, 1), IMPORT_SFUNC(NdisInitializeReadWriteLock, 1), IMPORT_SFUNC(NdisAcquireReadWriteLock, 3), IMPORT_SFUNC(NdisReleaseReadWriteLock, 2), IMPORT_SFUNC(NdisFreeSpinLock, 1), IMPORT_SFUNC(NdisFreeMemory, 3), IMPORT_SFUNC(NdisReadPciSlotInformation, 5), IMPORT_SFUNC(NdisWritePciSlotInformation, 5), IMPORT_SFUNC_MAP(NdisImmediateReadPciSlotInformation, NdisReadPciSlotInformation, 5), IMPORT_SFUNC_MAP(NdisImmediateWritePciSlotInformation, NdisWritePciSlotInformation, 5), IMPORT_CFUNC(NdisWriteErrorLogEntry, 0), IMPORT_SFUNC(NdisMStartBufferPhysicalMapping, 6), IMPORT_SFUNC(NdisMCompleteBufferPhysicalMapping, 3), IMPORT_SFUNC(NdisMInitializeTimer, 4), IMPORT_SFUNC(NdisInitializeTimer, 3), IMPORT_SFUNC(NdisSetTimer, 2), IMPORT_SFUNC(NdisMCancelTimer, 2), IMPORT_SFUNC_MAP(NdisCancelTimer, NdisMCancelTimer, 2), IMPORT_SFUNC(NdisMSetPeriodicTimer, 2), IMPORT_SFUNC(NdisMQueryAdapterResources, 4), IMPORT_SFUNC(NdisMRegisterIoPortRange, 4), IMPORT_SFUNC(NdisMDeregisterIoPortRange, 4), IMPORT_SFUNC(NdisReadNetworkAddress, 4), IMPORT_SFUNC(NdisQueryMapRegisterCount, 2), IMPORT_SFUNC(NdisMAllocateMapRegisters, 5), IMPORT_SFUNC(NdisMFreeMapRegisters, 1), IMPORT_SFUNC(NdisMAllocateSharedMemory, 5), IMPORT_SFUNC(NdisMMapIoSpace, 4 + 1), IMPORT_SFUNC(NdisMUnmapIoSpace, 3), IMPORT_SFUNC(NdisGetCacheFillSize, 0), IMPORT_SFUNC(NdisMGetDmaAlignment, 1), IMPORT_SFUNC(NdisMInitializeScatterGatherDma, 3), IMPORT_SFUNC(NdisAllocatePacketPool, 4), IMPORT_SFUNC(NdisAllocatePacketPoolEx, 5), IMPORT_SFUNC(NdisAllocatePacket, 3), IMPORT_SFUNC(NdisFreePacket, 1), IMPORT_SFUNC(NdisFreePacketPool, 1), IMPORT_SFUNC_MAP(NdisDprAllocatePacket, NdisAllocatePacket, 3), IMPORT_SFUNC_MAP(NdisDprFreePacket, NdisFreePacket, 1), IMPORT_SFUNC(NdisAllocateBufferPool, 3), IMPORT_SFUNC(NdisAllocateBuffer, 5), IMPORT_SFUNC(NdisQueryBuffer, 3), IMPORT_SFUNC(NdisQueryBufferSafe, 4), IMPORT_SFUNC(NdisBufferVirtualAddress, 1), IMPORT_SFUNC(NdisBufferVirtualAddressSafe, 2), IMPORT_SFUNC(NdisBufferLength, 1), IMPORT_SFUNC(NdisFreeBuffer, 1), IMPORT_SFUNC(NdisFreeBufferPool, 1), IMPORT_SFUNC(NdisInterlockedIncrement, 1), IMPORT_SFUNC(NdisInterlockedDecrement, 1), IMPORT_SFUNC(NdisInitializeEvent, 1), IMPORT_SFUNC(NdisSetEvent, 1), IMPORT_SFUNC(NdisResetEvent, 1), IMPORT_SFUNC(NdisWaitEvent, 2), IMPORT_SFUNC(NdisUnicodeStringToAnsiString, 2), IMPORT_SFUNC(NdisMPciAssignResources, 3), IMPORT_SFUNC(NdisMFreeSharedMemory, 5 + 1), IMPORT_SFUNC(NdisMRegisterInterrupt, 7), IMPORT_SFUNC(NdisMDeregisterInterrupt, 1), IMPORT_SFUNC(NdisMRegisterAdapterShutdownHandler, 3), IMPORT_SFUNC(NdisMDeregisterAdapterShutdownHandler, 1), IMPORT_SFUNC(NDIS_BUFFER_TO_SPAN_PAGES, 1), IMPORT_SFUNC(NdisQueryBufferOffset, 3), IMPORT_SFUNC(NdisAdjustBufferLength, 2), IMPORT_SFUNC(NdisPacketPoolUsage, 1), IMPORT_SFUNC(NdisMSleep, 1), IMPORT_SFUNC(NdisUnchainBufferAtFront, 2), IMPORT_SFUNC(NdisReadPcmciaAttributeMemory, 4), IMPORT_SFUNC(NdisWritePcmciaAttributeMemory, 4), IMPORT_SFUNC(NdisOpenFile, 5 + 1), IMPORT_SFUNC(NdisMapFile, 3), IMPORT_SFUNC(NdisUnmapFile, 1), IMPORT_SFUNC(NdisCloseFile, 1), IMPORT_SFUNC(NdisMRegisterDevice, 6), IMPORT_SFUNC(NdisMDeregisterDevice, 1), IMPORT_SFUNC(NdisMQueryAdapterInstanceName, 2), IMPORT_SFUNC(NdisMRegisterUnloadHandler, 2), IMPORT_SFUNC(ndis_timercall, 4), IMPORT_SFUNC(ndis_asyncmem_complete, 2), IMPORT_SFUNC(ndis_intr, 2), IMPORT_SFUNC(ndis_intrhand, 4), /* * This last entry is a catch-all for any function we haven't * implemented yet. The PE import list patching routine will * use it for any function that doesn't have an explicit match * in this table. */ { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL }, /* End of list. */ { NULL, NULL, NULL } }; Index: head/sys/compat/ndis/subr_ntoskrnl.c =================================================================== --- head/sys/compat/ndis/subr_ntoskrnl.c (revision 214797) +++ head/sys/compat/ndis/subr_ntoskrnl.c (revision 214798) @@ -1,4312 +1,4288 @@ /*- * Copyright (c) 2003 * Bill Paul . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Bill Paul. * 4. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include -#if __FreeBSD_version > 502113 #include -#endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef NTOSKRNL_DEBUG_TIMERS static int sysctl_show_timers(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, CTLFLAG_RW, 0, 0, sysctl_show_timers, "I", "Show ntoskrnl timer stats"); #endif struct kdpc_queue { list_entry kq_disp; struct thread *kq_td; int kq_cpu; int kq_exit; int kq_running; kspin_lock kq_lock; nt_kevent kq_proc; nt_kevent kq_done; }; typedef struct kdpc_queue kdpc_queue; struct wb_ext { struct cv we_cv; struct thread *we_td; }; typedef struct wb_ext wb_ext; #define NTOSKRNL_TIMEOUTS 256 #ifdef NTOSKRNL_DEBUG_TIMERS static uint64_t ntoskrnl_timer_fires; static uint64_t ntoskrnl_timer_sets; static uint64_t ntoskrnl_timer_reloads; static uint64_t ntoskrnl_timer_cancels; #endif struct callout_entry { struct callout ce_callout; list_entry ce_list; }; typedef struct callout_entry callout_entry; static struct list_entry ntoskrnl_calllist; static struct mtx ntoskrnl_calllock; static struct list_entry ntoskrnl_intlist; static kspin_lock ntoskrnl_intlock; static uint8_t RtlEqualUnicodeString(unicode_string *, unicode_string *, uint8_t); static void RtlCopyUnicodeString(unicode_string *, unicode_string *); static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *, void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *); static irp *IoBuildAsynchronousFsdRequest(uint32_t, device_object *, void *, uint32_t, uint64_t *, io_status_block *); static irp *IoBuildDeviceIoControlRequest(uint32_t, device_object *, void *, uint32_t, void *, uint32_t, uint8_t, nt_kevent *, io_status_block *); static irp *IoAllocateIrp(uint8_t, uint8_t); static void IoReuseIrp(irp *, uint32_t); static void IoFreeIrp(irp *); static void IoInitializeIrp(irp *, uint16_t, uint8_t); static irp *IoMakeAssociatedIrp(irp *, uint8_t); static uint32_t KeWaitForMultipleObjects(uint32_t, nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t, int64_t *, wait_block *); static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t); static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *); static void ntoskrnl_satisfy_multiple_waits(wait_block *); static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *); static void ntoskrnl_insert_timer(ktimer *, int); static void ntoskrnl_remove_timer(ktimer *); #ifdef NTOSKRNL_DEBUG_TIMERS static void ntoskrnl_show_timers(void); #endif static void ntoskrnl_timercall(void *); static void ntoskrnl_dpc_thread(void *); static void ntoskrnl_destroy_dpc_threads(void); static void ntoskrnl_destroy_workitem_threads(void); static void ntoskrnl_workitem_thread(void *); static void ntoskrnl_workitem(device_object *, void *); static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int); static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int); static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *); static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t); static uint16_t READ_REGISTER_USHORT(uint16_t *); static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t); static uint32_t READ_REGISTER_ULONG(uint32_t *); static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t); static uint8_t READ_REGISTER_UCHAR(uint8_t *); static int64_t _allmul(int64_t, int64_t); static int64_t _alldiv(int64_t, int64_t); static int64_t _allrem(int64_t, int64_t); static int64_t _allshr(int64_t, uint8_t); static int64_t _allshl(int64_t, uint8_t); static uint64_t _aullmul(uint64_t, uint64_t); static uint64_t _aulldiv(uint64_t, uint64_t); static uint64_t _aullrem(uint64_t, uint64_t); static uint64_t _aullshr(uint64_t, uint8_t); static uint64_t _aullshl(uint64_t, uint8_t); static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *); static slist_entry *ntoskrnl_popsl(slist_header *); static void ExInitializePagedLookasideList(paged_lookaside_list *, lookaside_alloc_func *, lookaside_free_func *, uint32_t, size_t, uint32_t, uint16_t); static void ExDeletePagedLookasideList(paged_lookaside_list *); static void ExInitializeNPagedLookasideList(npaged_lookaside_list *, lookaside_alloc_func *, lookaside_free_func *, uint32_t, size_t, uint32_t, uint16_t); static void ExDeleteNPagedLookasideList(npaged_lookaside_list *); static slist_entry *ExInterlockedPushEntrySList(slist_header *, slist_entry *, kspin_lock *); static slist_entry *ExInterlockedPopEntrySList(slist_header *, kspin_lock *); static uint32_t InterlockedIncrement(volatile uint32_t *); static uint32_t InterlockedDecrement(volatile uint32_t *); static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t); static void *MmAllocateContiguousMemory(uint32_t, uint64_t); static void *MmAllocateContiguousMemorySpecifyCache(uint32_t, uint64_t, uint64_t, uint64_t, uint32_t); static void MmFreeContiguousMemory(void *); static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t, uint32_t); static uint32_t MmSizeOfMdl(void *, size_t); static void *MmMapLockedPages(mdl *, uint8_t); static void *MmMapLockedPagesSpecifyCache(mdl *, uint8_t, uint32_t, void *, uint32_t, uint32_t); static void MmUnmapLockedPages(void *, mdl *); static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **); static void RtlZeroMemory(void *, size_t); static void RtlCopyMemory(void *, const void *, size_t); static size_t RtlCompareMemory(const void *, const void *, size_t); static ndis_status RtlUnicodeStringToInteger(unicode_string *, uint32_t, uint32_t *); static int atoi (const char *); static long atol (const char *); static int rand(void); static void srand(unsigned int); static void KeQuerySystemTime(uint64_t *); static uint32_t KeTickCount(void); static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t); static void ntoskrnl_thrfunc(void *); static ndis_status PsCreateSystemThread(ndis_handle *, uint32_t, void *, ndis_handle, void *, void *, void *); static ndis_status PsTerminateSystemThread(ndis_status); static ndis_status IoGetDeviceObjectPointer(unicode_string *, uint32_t, void *, device_object *); static ndis_status IoGetDeviceProperty(device_object *, uint32_t, uint32_t, void *, uint32_t *); static void KeInitializeMutex(kmutant *, uint32_t); static uint32_t KeReleaseMutex(kmutant *, uint8_t); static uint32_t KeReadStateMutex(kmutant *); static ndis_status ObReferenceObjectByHandle(ndis_handle, uint32_t, void *, uint8_t, void **, void **); static void ObfDereferenceObject(void *); static uint32_t ZwClose(ndis_handle); static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t, uint32_t, void *); static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...); static uint32_t IoWMIRegistrationControl(device_object *, uint32_t); static void *ntoskrnl_memset(void *, int, size_t); static void *ntoskrnl_memmove(void *, void *, size_t); static void *ntoskrnl_memchr(void *, unsigned char, size_t); static char *ntoskrnl_strstr(char *, char *); static char *ntoskrnl_strncat(char *, char *, size_t); static int ntoskrnl_toupper(int); static int ntoskrnl_tolower(int); static funcptr ntoskrnl_findwrap(funcptr); static uint32_t DbgPrint(char *, ...); static void DbgBreakPoint(void); static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long); static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *); static int32_t KeSetPriorityThread(struct thread *, int32_t); static void dummy(void); static struct mtx ntoskrnl_dispatchlock; static struct mtx ntoskrnl_interlock; static kspin_lock ntoskrnl_cancellock; static int ntoskrnl_kth = 0; static struct nt_objref_head ntoskrnl_reflist; static uma_zone_t mdl_zone; static uma_zone_t iw_zone; static struct kdpc_queue *kq_queues; static struct kdpc_queue *wq_queues; static int wq_idx = 0; int ntoskrnl_libinit() { image_patch_table *patch; int error; struct proc *p; kdpc_queue *kq; callout_entry *e; int i; mtx_init(&ntoskrnl_dispatchlock, "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE); mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN); KeInitializeSpinLock(&ntoskrnl_cancellock); KeInitializeSpinLock(&ntoskrnl_intlock); TAILQ_INIT(&ntoskrnl_reflist); InitializeListHead(&ntoskrnl_calllist); InitializeListHead(&ntoskrnl_intlist); mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN); kq_queues = ExAllocatePoolWithTag(NonPagedPool, #ifdef NTOSKRNL_MULTIPLE_DPCS sizeof(kdpc_queue) * mp_ncpus, 0); #else sizeof(kdpc_queue), 0); #endif if (kq_queues == NULL) return (ENOMEM); wq_queues = ExAllocatePoolWithTag(NonPagedPool, sizeof(kdpc_queue) * WORKITEM_THREADS, 0); if (wq_queues == NULL) return (ENOMEM); #ifdef NTOSKRNL_MULTIPLE_DPCS bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus); #else bzero((char *)kq_queues, sizeof(kdpc_queue)); #endif bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS); /* * Launch the DPC threads. */ #ifdef NTOSKRNL_MULTIPLE_DPCS for (i = 0; i < mp_ncpus; i++) { #else for (i = 0; i < 1; i++) { #endif kq = kq_queues + i; kq->kq_cpu = i; error = kproc_create(ntoskrnl_dpc_thread, kq, &p, RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i); if (error) panic("failed to launch DPC thread"); } /* * Launch the workitem threads. */ for (i = 0; i < WORKITEM_THREADS; i++) { kq = wq_queues + i; error = kproc_create(ntoskrnl_workitem_thread, kq, &p, RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i); if (error) panic("failed to launch workitem thread"); } patch = ntoskrnl_functbl; while (patch->ipt_func != NULL) { windrv_wrap((funcptr)patch->ipt_func, (funcptr *)&patch->ipt_wrap, patch->ipt_argcnt, patch->ipt_ftype); patch++; } for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) { e = ExAllocatePoolWithTag(NonPagedPool, sizeof(callout_entry), 0); if (e == NULL) panic("failed to allocate timeouts"); mtx_lock_spin(&ntoskrnl_calllock); InsertHeadList((&ntoskrnl_calllist), (&e->ce_list)); mtx_unlock_spin(&ntoskrnl_calllock); } /* * MDLs are supposed to be variable size (they describe * buffers containing some number of pages, but we don't * know ahead of time how many pages that will be). But * always allocating them off the heap is very slow. As * a compromise, we create an MDL UMA zone big enough to * handle any buffer requiring up to 16 pages, and we * use those for any MDLs for buffers of 16 pages or less * in size. For buffers larger than that (which we assume * will be few and far between, we allocate the MDLs off * the heap. */ mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); return (0); } int ntoskrnl_libfini() { image_patch_table *patch; callout_entry *e; list_entry *l; patch = ntoskrnl_functbl; while (patch->ipt_func != NULL) { windrv_unwrap(patch->ipt_wrap); patch++; } /* Stop the workitem queues. */ ntoskrnl_destroy_workitem_threads(); /* Stop the DPC queues. */ ntoskrnl_destroy_dpc_threads(); ExFreePool(kq_queues); ExFreePool(wq_queues); uma_zdestroy(mdl_zone); uma_zdestroy(iw_zone); mtx_lock_spin(&ntoskrnl_calllock); while(!IsListEmpty(&ntoskrnl_calllist)) { l = RemoveHeadList(&ntoskrnl_calllist); e = CONTAINING_RECORD(l, callout_entry, ce_list); mtx_unlock_spin(&ntoskrnl_calllock); ExFreePool(e); mtx_lock_spin(&ntoskrnl_calllock); } mtx_unlock_spin(&ntoskrnl_calllock); mtx_destroy(&ntoskrnl_dispatchlock); mtx_destroy(&ntoskrnl_interlock); mtx_destroy(&ntoskrnl_calllock); return (0); } /* * We need to be able to reference this externally from the wrapper; * GCC only generates a local implementation of memset. */ static void * ntoskrnl_memset(buf, ch, size) void *buf; int ch; size_t size; { return (memset(buf, ch, size)); } static void * ntoskrnl_memmove(dst, src, size) void *src; void *dst; size_t size; { bcopy(src, dst, size); return (dst); } static void * ntoskrnl_memchr(void *buf, unsigned char ch, size_t len) { if (len != 0) { unsigned char *p = buf; do { if (*p++ == ch) return (p - 1); } while (--len != 0); } return (NULL); } static char * ntoskrnl_strstr(s, find) char *s, *find; { char c, sc; size_t len; if ((c = *find++) != 0) { len = strlen(find); do { do { if ((sc = *s++) == 0) return (NULL); } while (sc != c); } while (strncmp(s, find, len) != 0); s--; } return ((char *)s); } /* Taken from libc */ static char * ntoskrnl_strncat(dst, src, n) char *dst; char *src; size_t n; { if (n != 0) { char *d = dst; const char *s = src; while (*d != 0) d++; do { if ((*d = *s++) == 0) break; d++; } while (--n != 0); *d = 0; } return (dst); } static int ntoskrnl_toupper(c) int c; { return (toupper(c)); } static int ntoskrnl_tolower(c) int c; { return (tolower(c)); } static uint8_t RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2, uint8_t caseinsensitive) { int i; if (str1->us_len != str2->us_len) return (FALSE); for (i = 0; i < str1->us_len; i++) { if (caseinsensitive == TRUE) { if (toupper((char)(str1->us_buf[i] & 0xFF)) != toupper((char)(str2->us_buf[i] & 0xFF))) return (FALSE); } else { if (str1->us_buf[i] != str2->us_buf[i]) return (FALSE); } } return (TRUE); } static void RtlCopyUnicodeString(dest, src) unicode_string *dest; unicode_string *src; { if (dest->us_maxlen >= src->us_len) dest->us_len = src->us_len; else dest->us_len = dest->us_maxlen; memcpy(dest->us_buf, src->us_buf, dest->us_len); } static void ntoskrnl_ascii_to_unicode(ascii, unicode, len) char *ascii; uint16_t *unicode; int len; { int i; uint16_t *ustr; ustr = unicode; for (i = 0; i < len; i++) { *ustr = (uint16_t)ascii[i]; ustr++; } } static void ntoskrnl_unicode_to_ascii(unicode, ascii, len) uint16_t *unicode; char *ascii; int len; { int i; uint8_t *astr; astr = ascii; for (i = 0; i < len / 2; i++) { *astr = (uint8_t)unicode[i]; astr++; } } uint32_t RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate) { if (dest == NULL || src == NULL) return (STATUS_INVALID_PARAMETER); dest->as_len = src->us_len / 2; if (dest->as_maxlen < dest->as_len) dest->as_len = dest->as_maxlen; if (allocate == TRUE) { dest->as_buf = ExAllocatePoolWithTag(NonPagedPool, (src->us_len / 2) + 1, 0); if (dest->as_buf == NULL) return (STATUS_INSUFFICIENT_RESOURCES); dest->as_len = dest->as_maxlen = src->us_len / 2; } else { dest->as_len = src->us_len / 2; /* XXX */ if (dest->as_maxlen < dest->as_len) dest->as_len = dest->as_maxlen; } ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf, dest->as_len * 2); return (STATUS_SUCCESS); } uint32_t RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src, uint8_t allocate) { if (dest == NULL || src == NULL) return (STATUS_INVALID_PARAMETER); if (allocate == TRUE) { dest->us_buf = ExAllocatePoolWithTag(NonPagedPool, src->as_len * 2, 0); if (dest->us_buf == NULL) return (STATUS_INSUFFICIENT_RESOURCES); dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2; } else { dest->us_len = src->as_len * 2; /* XXX */ if (dest->us_maxlen < dest->us_len) dest->us_len = dest->us_maxlen; } ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf, dest->us_len / 2); return (STATUS_SUCCESS); } void * ExAllocatePoolWithTag(pooltype, len, tag) uint32_t pooltype; size_t len; uint32_t tag; { void *buf; buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO); if (buf == NULL) return (NULL); return (buf); } void ExFreePool(buf) void *buf; { free(buf, M_DEVBUF); } uint32_t IoAllocateDriverObjectExtension(drv, clid, extlen, ext) driver_object *drv; void *clid; uint32_t extlen; void **ext; { custom_extension *ce; ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension) + extlen, 0); if (ce == NULL) return (STATUS_INSUFFICIENT_RESOURCES); ce->ce_clid = clid; InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list)); *ext = (void *)(ce + 1); return (STATUS_SUCCESS); } void * IoGetDriverObjectExtension(drv, clid) driver_object *drv; void *clid; { list_entry *e; custom_extension *ce; /* * Sanity check. Our dummy bus drivers don't have * any driver extentions. */ if (drv->dro_driverext == NULL) return (NULL); e = drv->dro_driverext->dre_usrext.nle_flink; while (e != &drv->dro_driverext->dre_usrext) { ce = (custom_extension *)e; if (ce->ce_clid == clid) return ((void *)(ce + 1)); e = e->nle_flink; } return (NULL); } uint32_t IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname, uint32_t devtype, uint32_t devchars, uint8_t exclusive, device_object **newdev) { device_object *dev; dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0); if (dev == NULL) return (STATUS_INSUFFICIENT_RESOURCES); dev->do_type = devtype; dev->do_drvobj = drv; dev->do_currirp = NULL; dev->do_flags = 0; if (devextlen) { dev->do_devext = ExAllocatePoolWithTag(NonPagedPool, devextlen, 0); if (dev->do_devext == NULL) { ExFreePool(dev); return (STATUS_INSUFFICIENT_RESOURCES); } bzero(dev->do_devext, devextlen); } else dev->do_devext = NULL; dev->do_size = sizeof(device_object) + devextlen; dev->do_refcnt = 1; dev->do_attacheddev = NULL; dev->do_nextdev = NULL; dev->do_devtype = devtype; dev->do_stacksize = 1; dev->do_alignreq = 1; dev->do_characteristics = devchars; dev->do_iotimer = NULL; KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE); /* * Vpd is used for disk/tape devices, * but we don't support those. (Yet.) */ dev->do_vpb = NULL; dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool, sizeof(devobj_extension), 0); if (dev->do_devobj_ext == NULL) { if (dev->do_devext != NULL) ExFreePool(dev->do_devext); ExFreePool(dev); return (STATUS_INSUFFICIENT_RESOURCES); } dev->do_devobj_ext->dve_type = 0; dev->do_devobj_ext->dve_size = sizeof(devobj_extension); dev->do_devobj_ext->dve_devobj = dev; /* * Attach this device to the driver object's list * of devices. Note: this is not the same as attaching * the device to the device stack. The driver's AddDevice * routine must explicitly call IoAddDeviceToDeviceStack() * to do that. */ if (drv->dro_devobj == NULL) { drv->dro_devobj = dev; dev->do_nextdev = NULL; } else { dev->do_nextdev = drv->dro_devobj; drv->dro_devobj = dev; } *newdev = dev; return (STATUS_SUCCESS); } void IoDeleteDevice(dev) device_object *dev; { device_object *prev; if (dev == NULL) return; if (dev->do_devobj_ext != NULL) ExFreePool(dev->do_devobj_ext); if (dev->do_devext != NULL) ExFreePool(dev->do_devext); /* Unlink the device from the driver's device list. */ prev = dev->do_drvobj->dro_devobj; if (prev == dev) dev->do_drvobj->dro_devobj = dev->do_nextdev; else { while (prev->do_nextdev != dev) prev = prev->do_nextdev; prev->do_nextdev = dev->do_nextdev; } ExFreePool(dev); } device_object * IoGetAttachedDevice(dev) device_object *dev; { device_object *d; if (dev == NULL) return (NULL); d = dev; while (d->do_attacheddev != NULL) d = d->do_attacheddev; return (d); } static irp * IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status) uint32_t func; device_object *dobj; void *buf; uint32_t len; uint64_t *off; nt_kevent *event; io_status_block *status; { irp *ip; ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status); if (ip == NULL) return (NULL); ip->irp_usrevent = event; return (ip); } static irp * IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status) uint32_t func; device_object *dobj; void *buf; uint32_t len; uint64_t *off; io_status_block *status; { irp *ip; io_stack_location *sl; ip = IoAllocateIrp(dobj->do_stacksize, TRUE); if (ip == NULL) return (NULL); ip->irp_usriostat = status; ip->irp_tail.irp_overlay.irp_thread = NULL; sl = IoGetNextIrpStackLocation(ip); sl->isl_major = func; sl->isl_minor = 0; sl->isl_flags = 0; sl->isl_ctl = 0; sl->isl_devobj = dobj; sl->isl_fileobj = NULL; sl->isl_completionfunc = NULL; ip->irp_userbuf = buf; if (dobj->do_flags & DO_BUFFERED_IO) { ip->irp_assoc.irp_sysbuf = ExAllocatePoolWithTag(NonPagedPool, len, 0); if (ip->irp_assoc.irp_sysbuf == NULL) { IoFreeIrp(ip); return (NULL); } bcopy(buf, ip->irp_assoc.irp_sysbuf, len); } if (dobj->do_flags & DO_DIRECT_IO) { ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip); if (ip->irp_mdl == NULL) { if (ip->irp_assoc.irp_sysbuf != NULL) ExFreePool(ip->irp_assoc.irp_sysbuf); IoFreeIrp(ip); return (NULL); } ip->irp_userbuf = NULL; ip->irp_assoc.irp_sysbuf = NULL; } if (func == IRP_MJ_READ) { sl->isl_parameters.isl_read.isl_len = len; if (off != NULL) sl->isl_parameters.isl_read.isl_byteoff = *off; else sl->isl_parameters.isl_read.isl_byteoff = 0; } if (func == IRP_MJ_WRITE) { sl->isl_parameters.isl_write.isl_len = len; if (off != NULL) sl->isl_parameters.isl_write.isl_byteoff = *off; else sl->isl_parameters.isl_write.isl_byteoff = 0; } return (ip); } static irp * IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf, uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal, nt_kevent *event, io_status_block *status) { irp *ip; io_stack_location *sl; uint32_t buflen; ip = IoAllocateIrp(dobj->do_stacksize, TRUE); if (ip == NULL) return (NULL); ip->irp_usrevent = event; ip->irp_usriostat = status; ip->irp_tail.irp_overlay.irp_thread = NULL; sl = IoGetNextIrpStackLocation(ip); sl->isl_major = isinternal == TRUE ? IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL; sl->isl_minor = 0; sl->isl_flags = 0; sl->isl_ctl = 0; sl->isl_devobj = dobj; sl->isl_fileobj = NULL; sl->isl_completionfunc = NULL; sl->isl_parameters.isl_ioctl.isl_iocode = iocode; sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen; sl->isl_parameters.isl_ioctl.isl_obuflen = olen; switch(IO_METHOD(iocode)) { case METHOD_BUFFERED: if (ilen > olen) buflen = ilen; else buflen = olen; if (buflen) { ip->irp_assoc.irp_sysbuf = ExAllocatePoolWithTag(NonPagedPool, buflen, 0); if (ip->irp_assoc.irp_sysbuf == NULL) { IoFreeIrp(ip); return (NULL); } } if (ilen && ibuf != NULL) { bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen); bzero((char *)ip->irp_assoc.irp_sysbuf + ilen, buflen - ilen); } else bzero(ip->irp_assoc.irp_sysbuf, ilen); ip->irp_userbuf = obuf; break; case METHOD_IN_DIRECT: case METHOD_OUT_DIRECT: if (ilen && ibuf != NULL) { ip->irp_assoc.irp_sysbuf = ExAllocatePoolWithTag(NonPagedPool, ilen, 0); if (ip->irp_assoc.irp_sysbuf == NULL) { IoFreeIrp(ip); return (NULL); } bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen); } if (olen && obuf != NULL) { ip->irp_mdl = IoAllocateMdl(obuf, olen, FALSE, FALSE, ip); /* * Normally we would MmProbeAndLockPages() * here, but we don't have to in our * imlementation. */ } break; case METHOD_NEITHER: ip->irp_userbuf = obuf; sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf; break; default: break; } /* * Ideally, we should associate this IRP with the calling * thread here. */ return (ip); } static irp * IoAllocateIrp(uint8_t stsize, uint8_t chargequota) { irp *i; i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0); if (i == NULL) return (NULL); IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize); return (i); } static irp * IoMakeAssociatedIrp(irp *ip, uint8_t stsize) { irp *associrp; associrp = IoAllocateIrp(stsize, FALSE); if (associrp == NULL) return (NULL); mtx_lock(&ntoskrnl_dispatchlock); associrp->irp_flags |= IRP_ASSOCIATED_IRP; associrp->irp_tail.irp_overlay.irp_thread = ip->irp_tail.irp_overlay.irp_thread; associrp->irp_assoc.irp_master = ip; mtx_unlock(&ntoskrnl_dispatchlock); return (associrp); } static void IoFreeIrp(ip) irp *ip; { ExFreePool(ip); } static void IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize) { bzero((char *)io, IoSizeOfIrp(ssize)); io->irp_size = psize; io->irp_stackcnt = ssize; io->irp_currentstackloc = ssize; InitializeListHead(&io->irp_thlist); io->irp_tail.irp_overlay.irp_csl = (io_stack_location *)(io + 1) + ssize; } static void IoReuseIrp(ip, status) irp *ip; uint32_t status; { uint8_t allocflags; allocflags = ip->irp_allocflags; IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt); ip->irp_iostat.isb_status = status; ip->irp_allocflags = allocflags; } void IoAcquireCancelSpinLock(uint8_t *irql) { KeAcquireSpinLock(&ntoskrnl_cancellock, irql); } void IoReleaseCancelSpinLock(uint8_t irql) { KeReleaseSpinLock(&ntoskrnl_cancellock, irql); } uint8_t IoCancelIrp(irp *ip) { cancel_func cfunc; uint8_t cancelirql; IoAcquireCancelSpinLock(&cancelirql); cfunc = IoSetCancelRoutine(ip, NULL); ip->irp_cancel = TRUE; if (cfunc == NULL) { IoReleaseCancelSpinLock(cancelirql); return (FALSE); } ip->irp_cancelirql = cancelirql; MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip); return (uint8_t)IoSetCancelValue(ip, TRUE); } uint32_t IofCallDriver(dobj, ip) device_object *dobj; irp *ip; { driver_object *drvobj; io_stack_location *sl; uint32_t status; driver_dispatch disp; drvobj = dobj->do_drvobj; if (ip->irp_currentstackloc <= 0) panic("IoCallDriver(): out of stack locations"); IoSetNextIrpStackLocation(ip); sl = IoGetCurrentIrpStackLocation(ip); sl->isl_devobj = dobj; disp = drvobj->dro_dispatch[sl->isl_major]; status = MSCALL2(disp, dobj, ip); return (status); } void IofCompleteRequest(irp *ip, uint8_t prioboost) { uint32_t status; device_object *dobj; io_stack_location *sl; completion_func cf; KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING, ("incorrect IRP(%p) status (STATUS_PENDING)", ip)); sl = IoGetCurrentIrpStackLocation(ip); IoSkipCurrentIrpStackLocation(ip); do { if (sl->isl_ctl & SL_PENDING_RETURNED) ip->irp_pendingreturned = TRUE; if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1)) dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj; else dobj = NULL; if (sl->isl_completionfunc != NULL && ((ip->irp_iostat.isb_status == STATUS_SUCCESS && sl->isl_ctl & SL_INVOKE_ON_SUCCESS) || (ip->irp_iostat.isb_status != STATUS_SUCCESS && sl->isl_ctl & SL_INVOKE_ON_ERROR) || (ip->irp_cancel == TRUE && sl->isl_ctl & SL_INVOKE_ON_CANCEL))) { cf = sl->isl_completionfunc; status = MSCALL3(cf, dobj, ip, sl->isl_completionctx); if (status == STATUS_MORE_PROCESSING_REQUIRED) return; } else { if ((ip->irp_currentstackloc <= ip->irp_stackcnt) && (ip->irp_pendingreturned == TRUE)) IoMarkIrpPending(ip); } /* move to the next. */ IoSkipCurrentIrpStackLocation(ip); sl++; } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1)); if (ip->irp_usriostat != NULL) *ip->irp_usriostat = ip->irp_iostat; if (ip->irp_usrevent != NULL) KeSetEvent(ip->irp_usrevent, prioboost, FALSE); /* Handle any associated IRPs. */ if (ip->irp_flags & IRP_ASSOCIATED_IRP) { uint32_t masterirpcnt; irp *masterirp; mdl *m; masterirp = ip->irp_assoc.irp_master; masterirpcnt = InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt); while ((m = ip->irp_mdl) != NULL) { ip->irp_mdl = m->mdl_next; IoFreeMdl(m); } IoFreeIrp(ip); if (masterirpcnt == 0) IoCompleteRequest(masterirp, IO_NO_INCREMENT); return; } /* With any luck, these conditions will never arise. */ if (ip->irp_flags & IRP_PAGING_IO) { if (ip->irp_mdl != NULL) IoFreeMdl(ip->irp_mdl); IoFreeIrp(ip); } } void ntoskrnl_intr(arg) void *arg; { kinterrupt *iobj; uint8_t irql; uint8_t claimed; list_entry *l; KeAcquireSpinLock(&ntoskrnl_intlock, &irql); l = ntoskrnl_intlist.nle_flink; while (l != &ntoskrnl_intlist) { iobj = CONTAINING_RECORD(l, kinterrupt, ki_list); claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx); if (claimed == TRUE) break; l = l->nle_flink; } KeReleaseSpinLock(&ntoskrnl_intlock, irql); } uint8_t KeAcquireInterruptSpinLock(iobj) kinterrupt *iobj; { uint8_t irql; KeAcquireSpinLock(&ntoskrnl_intlock, &irql); return (irql); } void KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql) { KeReleaseSpinLock(&ntoskrnl_intlock, irql); } uint8_t KeSynchronizeExecution(iobj, syncfunc, syncctx) kinterrupt *iobj; void *syncfunc; void *syncctx; { uint8_t irql; KeAcquireSpinLock(&ntoskrnl_intlock, &irql); MSCALL1(syncfunc, syncctx); KeReleaseSpinLock(&ntoskrnl_intlock, irql); return (TRUE); } /* * IoConnectInterrupt() is passed only the interrupt vector and * irql that a device wants to use, but no device-specific tag * of any kind. This conflicts rather badly with FreeBSD's * bus_setup_intr(), which needs the device_t for the device * requesting interrupt delivery. In order to bypass this * inconsistency, we implement a second level of interrupt * dispatching on top of bus_setup_intr(). All devices use * ntoskrnl_intr() as their ISR, and any device requesting * interrupts will be registered with ntoskrnl_intr()'s interrupt * dispatch list. When an interrupt arrives, we walk the list * and invoke all the registered ISRs. This effectively makes all * interrupts shared, but it's the only way to duplicate the * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly. */ uint32_t IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx, kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql, uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat) { uint8_t curirql; *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0); if (*iobj == NULL) return (STATUS_INSUFFICIENT_RESOURCES); (*iobj)->ki_svcfunc = svcfunc; (*iobj)->ki_svcctx = svcctx; if (lock == NULL) { KeInitializeSpinLock(&(*iobj)->ki_lock_priv); (*iobj)->ki_lock = &(*iobj)->ki_lock_priv; } else (*iobj)->ki_lock = lock; KeAcquireSpinLock(&ntoskrnl_intlock, &curirql); InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list)); KeReleaseSpinLock(&ntoskrnl_intlock, curirql); return (STATUS_SUCCESS); } void IoDisconnectInterrupt(iobj) kinterrupt *iobj; { uint8_t irql; if (iobj == NULL) return; KeAcquireSpinLock(&ntoskrnl_intlock, &irql); RemoveEntryList((&iobj->ki_list)); KeReleaseSpinLock(&ntoskrnl_intlock, irql); ExFreePool(iobj); } device_object * IoAttachDeviceToDeviceStack(src, dst) device_object *src; device_object *dst; { device_object *attached; mtx_lock(&ntoskrnl_dispatchlock); attached = IoGetAttachedDevice(dst); attached->do_attacheddev = src; src->do_attacheddev = NULL; src->do_stacksize = attached->do_stacksize + 1; mtx_unlock(&ntoskrnl_dispatchlock); return (attached); } void IoDetachDevice(topdev) device_object *topdev; { device_object *tail; mtx_lock(&ntoskrnl_dispatchlock); /* First, break the chain. */ tail = topdev->do_attacheddev; if (tail == NULL) { mtx_unlock(&ntoskrnl_dispatchlock); return; } topdev->do_attacheddev = tail->do_attacheddev; topdev->do_refcnt--; /* Now reduce the stacksize count for the takm_il objects. */ tail = topdev->do_attacheddev; while (tail != NULL) { tail->do_stacksize--; tail = tail->do_attacheddev; } mtx_unlock(&ntoskrnl_dispatchlock); } /* * For the most part, an object is considered signalled if * dh_sigstate == TRUE. The exception is for mutant objects * (mutexes), where the logic works like this: * * - If the thread already owns the object and sigstate is * less than or equal to 0, then the object is considered * signalled (recursive acquisition). * - If dh_sigstate == 1, the object is also considered * signalled. */ static int ntoskrnl_is_signalled(obj, td) nt_dispatch_header *obj; struct thread *td; { kmutant *km; if (obj->dh_type == DISP_TYPE_MUTANT) { km = (kmutant *)obj; if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) || obj->dh_sigstate == 1) return (TRUE); return (FALSE); } if (obj->dh_sigstate > 0) return (TRUE); return (FALSE); } static void ntoskrnl_satisfy_wait(obj, td) nt_dispatch_header *obj; struct thread *td; { kmutant *km; switch (obj->dh_type) { case DISP_TYPE_MUTANT: km = (struct kmutant *)obj; obj->dh_sigstate--; /* * If sigstate reaches 0, the mutex is now * non-signalled (the new thread owns it). */ if (obj->dh_sigstate == 0) { km->km_ownerthread = td; if (km->km_abandoned == TRUE) km->km_abandoned = FALSE; } break; /* Synchronization objects get reset to unsignalled. */ case DISP_TYPE_SYNCHRONIZATION_EVENT: case DISP_TYPE_SYNCHRONIZATION_TIMER: obj->dh_sigstate = 0; break; case DISP_TYPE_SEMAPHORE: obj->dh_sigstate--; break; default: break; } } static void ntoskrnl_satisfy_multiple_waits(wb) wait_block *wb; { wait_block *cur; struct thread *td; cur = wb; td = wb->wb_kthread; do { ntoskrnl_satisfy_wait(wb->wb_object, td); cur->wb_awakened = TRUE; cur = cur->wb_next; } while (cur != wb); } /* Always called with dispatcher lock held. */ static void ntoskrnl_waittest(obj, increment) nt_dispatch_header *obj; uint32_t increment; { wait_block *w, *next; list_entry *e; struct thread *td; wb_ext *we; int satisfied; /* * Once an object has been signalled, we walk its list of * wait blocks. If a wait block can be awakened, then satisfy * waits as necessary and wake the thread. * * The rules work like this: * * If a wait block is marked as WAITTYPE_ANY, then * we can satisfy the wait conditions on the current * object and wake the thread right away. Satisfying * the wait also has the effect of breaking us out * of the search loop. * * If the object is marked as WAITTYLE_ALL, then the * wait block will be part of a circularly linked * list of wait blocks belonging to a waiting thread * that's sleeping in KeWaitForMultipleObjects(). In * order to wake the thread, all the objects in the * wait list must be in the signalled state. If they * are, we then satisfy all of them and wake the * thread. * */ e = obj->dh_waitlisthead.nle_flink; while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) { w = CONTAINING_RECORD(e, wait_block, wb_waitlist); we = w->wb_ext; td = we->we_td; satisfied = FALSE; if (w->wb_waittype == WAITTYPE_ANY) { /* * Thread can be awakened if * any wait is satisfied. */ ntoskrnl_satisfy_wait(obj, td); satisfied = TRUE; w->wb_awakened = TRUE; } else { /* * Thread can only be woken up * if all waits are satisfied. * If the thread is waiting on multiple * objects, they should all be linked * through the wb_next pointers in the * wait blocks. */ satisfied = TRUE; next = w->wb_next; while (next != w) { if (ntoskrnl_is_signalled(obj, td) == FALSE) { satisfied = FALSE; break; } next = next->wb_next; } ntoskrnl_satisfy_multiple_waits(w); } if (satisfied == TRUE) cv_broadcastpri(&we->we_cv, (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ? w->wb_oldpri - (increment * 4) : PRI_MIN_KERN); e = e->nle_flink; } } /* * Return the number of 100 nanosecond intervals since * January 1, 1601. (?!?!) */ void ntoskrnl_time(tval) uint64_t *tval; { struct timespec ts; nanotime(&ts); *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 + 11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */ } static void KeQuerySystemTime(current_time) uint64_t *current_time; { ntoskrnl_time(current_time); } static uint32_t KeTickCount(void) { struct timeval tv; getmicrouptime(&tv); return tvtohz(&tv); } /* * KeWaitForSingleObject() is a tricky beast, because it can be used * with several different object types: semaphores, timers, events, * mutexes and threads. Semaphores don't appear very often, but the * other object types are quite common. KeWaitForSingleObject() is * what's normally used to acquire a mutex, and it can be used to * wait for a thread termination. * * The Windows NDIS API is implemented in terms of Windows kernel * primitives, and some of the object manipulation is duplicated in * NDIS. For example, NDIS has timers and events, which are actually * Windows kevents and ktimers. Now, you're supposed to only use the * NDIS variants of these objects within the confines of the NDIS API, * but there are some naughty developers out there who will use * KeWaitForSingleObject() on NDIS timer and event objects, so we * have to support that as well. Conseqently, our NDIS timer and event * code has to be closely tied into our ntoskrnl timer and event code, * just as it is in Windows. * * KeWaitForSingleObject() may do different things for different kinds * of objects: * * - For events, we check if the event has been signalled. If the * event is already in the signalled state, we just return immediately, * otherwise we wait for it to be set to the signalled state by someone * else calling KeSetEvent(). Events can be either synchronization or * notification events. * * - For timers, if the timer has already fired and the timer is in * the signalled state, we just return, otherwise we wait on the * timer. Unlike an event, timers get signalled automatically when * they expire rather than someone having to trip them manually. * Timers initialized with KeInitializeTimer() are always notification * events: KeInitializeTimerEx() lets you initialize a timer as * either a notification or synchronization event. * * - For mutexes, we try to acquire the mutex and if we can't, we wait * on the mutex until it's available and then grab it. When a mutex is * released, it enters the signalled state, which wakes up one of the * threads waiting to acquire it. Mutexes are always synchronization * events. * * - For threads, the only thing we do is wait until the thread object * enters a signalled state, which occurs when the thread terminates. * Threads are always notification events. * * A notification event wakes up all threads waiting on an object. A * synchronization event wakes up just one. Also, a synchronization event * is auto-clearing, which means we automatically set the event back to * the non-signalled state once the wakeup is done. */ uint32_t KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime) { wait_block w; struct thread *td = curthread; struct timeval tv; int error = 0; uint64_t curtime; wb_ext we; nt_dispatch_header *obj; obj = arg; if (obj == NULL) return (STATUS_INVALID_PARAMETER); mtx_lock(&ntoskrnl_dispatchlock); cv_init(&we.we_cv, "KeWFS"); we.we_td = td; /* * Check to see if this object is already signalled, * and just return without waiting if it is. */ if (ntoskrnl_is_signalled(obj, td) == TRUE) { /* Sanity check the signal state value. */ if (obj->dh_sigstate != INT32_MIN) { ntoskrnl_satisfy_wait(obj, curthread); mtx_unlock(&ntoskrnl_dispatchlock); return (STATUS_SUCCESS); } else { /* * There's a limit to how many times we can * recursively acquire a mutant. If we hit * the limit, something is very wrong. */ if (obj->dh_type == DISP_TYPE_MUTANT) { mtx_unlock(&ntoskrnl_dispatchlock); panic("mutant limit exceeded"); } } } bzero((char *)&w, sizeof(wait_block)); w.wb_object = obj; w.wb_ext = &we; w.wb_waittype = WAITTYPE_ANY; w.wb_next = &w; w.wb_waitkey = 0; w.wb_awakened = FALSE; w.wb_oldpri = td->td_priority; InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist)); /* * The timeout value is specified in 100 nanosecond units * and can be a positive or negative number. If it's positive, * then the duetime is absolute, and we need to convert it * to an absolute offset relative to now in order to use it. * If it's negative, then the duetime is relative and we * just have to convert the units. */ if (duetime != NULL) { if (*duetime < 0) { tv.tv_sec = - (*duetime) / 10000000; tv.tv_usec = (- (*duetime) / 10) - (tv.tv_sec * 1000000); } else { ntoskrnl_time(&curtime); if (*duetime < curtime) tv.tv_sec = tv.tv_usec = 0; else { tv.tv_sec = ((*duetime) - curtime) / 10000000; tv.tv_usec = ((*duetime) - curtime) / 10 - (tv.tv_sec * 1000000); } } } if (duetime == NULL) cv_wait(&we.we_cv, &ntoskrnl_dispatchlock); else error = cv_timedwait(&we.we_cv, &ntoskrnl_dispatchlock, tvtohz(&tv)); RemoveEntryList(&w.wb_waitlist); cv_destroy(&we.we_cv); /* We timed out. Leave the object alone and return status. */ if (error == EWOULDBLOCK) { mtx_unlock(&ntoskrnl_dispatchlock); return (STATUS_TIMEOUT); } mtx_unlock(&ntoskrnl_dispatchlock); return (STATUS_SUCCESS); /* return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason, mode, alertable, duetime, &w)); */ } static uint32_t KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype, uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime, wait_block *wb_array) { struct thread *td = curthread; wait_block *whead, *w; wait_block _wb_array[MAX_WAIT_OBJECTS]; nt_dispatch_header *cur; struct timeval tv; int i, wcnt = 0, error = 0; uint64_t curtime; struct timespec t1, t2; uint32_t status = STATUS_SUCCESS; wb_ext we; if (cnt > MAX_WAIT_OBJECTS) return (STATUS_INVALID_PARAMETER); if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL) return (STATUS_INVALID_PARAMETER); mtx_lock(&ntoskrnl_dispatchlock); cv_init(&we.we_cv, "KeWFM"); we.we_td = td; if (wb_array == NULL) whead = _wb_array; else whead = wb_array; bzero((char *)whead, sizeof(wait_block) * cnt); /* First pass: see if we can satisfy any waits immediately. */ wcnt = 0; w = whead; for (i = 0; i < cnt; i++) { InsertTailList((&obj[i]->dh_waitlisthead), (&w->wb_waitlist)); w->wb_ext = &we; w->wb_object = obj[i]; w->wb_waittype = wtype; w->wb_waitkey = i; w->wb_awakened = FALSE; w->wb_oldpri = td->td_priority; w->wb_next = w + 1; w++; wcnt++; if (ntoskrnl_is_signalled(obj[i], td)) { /* * There's a limit to how many times * we can recursively acquire a mutant. * If we hit the limit, something * is very wrong. */ if (obj[i]->dh_sigstate == INT32_MIN && obj[i]->dh_type == DISP_TYPE_MUTANT) { mtx_unlock(&ntoskrnl_dispatchlock); panic("mutant limit exceeded"); } /* * If this is a WAITTYPE_ANY wait, then * satisfy the waited object and exit * right now. */ if (wtype == WAITTYPE_ANY) { ntoskrnl_satisfy_wait(obj[i], td); status = STATUS_WAIT_0 + i; goto wait_done; } else { w--; wcnt--; w->wb_object = NULL; RemoveEntryList(&w->wb_waitlist); } } } /* * If this is a WAITTYPE_ALL wait and all objects are * already signalled, satisfy the waits and exit now. */ if (wtype == WAITTYPE_ALL && wcnt == 0) { for (i = 0; i < cnt; i++) ntoskrnl_satisfy_wait(obj[i], td); status = STATUS_SUCCESS; goto wait_done; } /* * Create a circular waitblock list. The waitcount * must always be non-zero when we get here. */ (w - 1)->wb_next = whead; /* Wait on any objects that aren't yet signalled. */ /* Calculate timeout, if any. */ if (duetime != NULL) { if (*duetime < 0) { tv.tv_sec = - (*duetime) / 10000000; tv.tv_usec = (- (*duetime) / 10) - (tv.tv_sec * 1000000); } else { ntoskrnl_time(&curtime); if (*duetime < curtime) tv.tv_sec = tv.tv_usec = 0; else { tv.tv_sec = ((*duetime) - curtime) / 10000000; tv.tv_usec = ((*duetime) - curtime) / 10 - (tv.tv_sec * 1000000); } } } while (wcnt) { nanotime(&t1); if (duetime == NULL) cv_wait(&we.we_cv, &ntoskrnl_dispatchlock); else error = cv_timedwait(&we.we_cv, &ntoskrnl_dispatchlock, tvtohz(&tv)); /* Wait with timeout expired. */ if (error) { status = STATUS_TIMEOUT; goto wait_done; } nanotime(&t2); /* See what's been signalled. */ w = whead; do { cur = w->wb_object; if (ntoskrnl_is_signalled(cur, td) == TRUE || w->wb_awakened == TRUE) { /* Sanity check the signal state value. */ if (cur->dh_sigstate == INT32_MIN && cur->dh_type == DISP_TYPE_MUTANT) { mtx_unlock(&ntoskrnl_dispatchlock); panic("mutant limit exceeded"); } wcnt--; if (wtype == WAITTYPE_ANY) { status = w->wb_waitkey & STATUS_WAIT_0; goto wait_done; } } w = w->wb_next; } while (w != whead); /* * If all objects have been signalled, or if this * is a WAITTYPE_ANY wait and we were woke up by * someone, we can bail. */ if (wcnt == 0) { status = STATUS_SUCCESS; goto wait_done; } /* * If this is WAITTYPE_ALL wait, and there's still * objects that haven't been signalled, deduct the * time that's elapsed so far from the timeout and * wait again (or continue waiting indefinitely if * there's no timeout). */ if (duetime != NULL) { tv.tv_sec -= (t2.tv_sec - t1.tv_sec); tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000; } } wait_done: cv_destroy(&we.we_cv); for (i = 0; i < cnt; i++) { if (whead[i].wb_object != NULL) RemoveEntryList(&whead[i].wb_waitlist); } mtx_unlock(&ntoskrnl_dispatchlock); return (status); } static void WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val) { bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val); } static uint16_t READ_REGISTER_USHORT(reg) uint16_t *reg; { return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg)); } static void WRITE_REGISTER_ULONG(reg, val) uint32_t *reg; uint32_t val; { bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val); } static uint32_t READ_REGISTER_ULONG(reg) uint32_t *reg; { return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg)); } static uint8_t READ_REGISTER_UCHAR(uint8_t *reg) { return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg)); } static void WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val) { bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val); } static int64_t _allmul(a, b) int64_t a; int64_t b; { return (a * b); } static int64_t _alldiv(a, b) int64_t a; int64_t b; { return (a / b); } static int64_t _allrem(a, b) int64_t a; int64_t b; { return (a % b); } static uint64_t _aullmul(a, b) uint64_t a; uint64_t b; { return (a * b); } static uint64_t _aulldiv(a, b) uint64_t a; uint64_t b; { return (a / b); } static uint64_t _aullrem(a, b) uint64_t a; uint64_t b; { return (a % b); } static int64_t _allshl(int64_t a, uint8_t b) { return (a << b); } static uint64_t _aullshl(uint64_t a, uint8_t b) { return (a << b); } static int64_t _allshr(int64_t a, uint8_t b) { return (a >> b); } static uint64_t _aullshr(uint64_t a, uint8_t b) { return (a >> b); } static slist_entry * ntoskrnl_pushsl(head, entry) slist_header *head; slist_entry *entry; { slist_entry *oldhead; oldhead = head->slh_list.slh_next; entry->sl_next = head->slh_list.slh_next; head->slh_list.slh_next = entry; head->slh_list.slh_depth++; head->slh_list.slh_seq++; return (oldhead); } static slist_entry * ntoskrnl_popsl(head) slist_header *head; { slist_entry *first; first = head->slh_list.slh_next; if (first != NULL) { head->slh_list.slh_next = first->sl_next; head->slh_list.slh_depth--; head->slh_list.slh_seq++; } return (first); } /* * We need this to make lookaside lists work for amd64. * We pass a pointer to ExAllocatePoolWithTag() the lookaside * list structure. For amd64 to work right, this has to be a * pointer to the wrapped version of the routine, not the * original. Letting the Windows driver invoke the original * function directly will result in a convention calling * mismatch and a pretty crash. On x86, this effectively * becomes a no-op since ipt_func and ipt_wrap are the same. */ static funcptr ntoskrnl_findwrap(func) funcptr func; { image_patch_table *patch; patch = ntoskrnl_functbl; while (patch->ipt_func != NULL) { if ((funcptr)patch->ipt_func == func) return ((funcptr)patch->ipt_wrap); patch++; } return (NULL); } static void ExInitializePagedLookasideList(paged_lookaside_list *lookaside, lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc, uint32_t flags, size_t size, uint32_t tag, uint16_t depth) { bzero((char *)lookaside, sizeof(paged_lookaside_list)); if (size < sizeof(slist_entry)) lookaside->nll_l.gl_size = sizeof(slist_entry); else lookaside->nll_l.gl_size = size; lookaside->nll_l.gl_tag = tag; if (allocfunc == NULL) lookaside->nll_l.gl_allocfunc = ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag); else lookaside->nll_l.gl_allocfunc = allocfunc; if (freefunc == NULL) lookaside->nll_l.gl_freefunc = ntoskrnl_findwrap((funcptr)ExFreePool); else lookaside->nll_l.gl_freefunc = freefunc; #ifdef __i386__ KeInitializeSpinLock(&lookaside->nll_obsoletelock); #endif lookaside->nll_l.gl_type = NonPagedPool; lookaside->nll_l.gl_depth = depth; lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH; } static void ExDeletePagedLookasideList(lookaside) paged_lookaside_list *lookaside; { void *buf; void (*freefunc)(void *); freefunc = lookaside->nll_l.gl_freefunc; while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL) MSCALL1(freefunc, buf); } static void ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside, lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc, uint32_t flags, size_t size, uint32_t tag, uint16_t depth) { bzero((char *)lookaside, sizeof(npaged_lookaside_list)); if (size < sizeof(slist_entry)) lookaside->nll_l.gl_size = sizeof(slist_entry); else lookaside->nll_l.gl_size = size; lookaside->nll_l.gl_tag = tag; if (allocfunc == NULL) lookaside->nll_l.gl_allocfunc = ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag); else lookaside->nll_l.gl_allocfunc = allocfunc; if (freefunc == NULL) lookaside->nll_l.gl_freefunc = ntoskrnl_findwrap((funcptr)ExFreePool); else lookaside->nll_l.gl_freefunc = freefunc; #ifdef __i386__ KeInitializeSpinLock(&lookaside->nll_obsoletelock); #endif lookaside->nll_l.gl_type = NonPagedPool; lookaside->nll_l.gl_depth = depth; lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH; } static void ExDeleteNPagedLookasideList(lookaside) npaged_lookaside_list *lookaside; { void *buf; void (*freefunc)(void *); freefunc = lookaside->nll_l.gl_freefunc; while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL) MSCALL1(freefunc, buf); } slist_entry * InterlockedPushEntrySList(head, entry) slist_header *head; slist_entry *entry; { slist_entry *oldhead; mtx_lock_spin(&ntoskrnl_interlock); oldhead = ntoskrnl_pushsl(head, entry); mtx_unlock_spin(&ntoskrnl_interlock); return (oldhead); } slist_entry * InterlockedPopEntrySList(head) slist_header *head; { slist_entry *first; mtx_lock_spin(&ntoskrnl_interlock); first = ntoskrnl_popsl(head); mtx_unlock_spin(&ntoskrnl_interlock); return (first); } static slist_entry * ExInterlockedPushEntrySList(head, entry, lock) slist_header *head; slist_entry *entry; kspin_lock *lock; { return (InterlockedPushEntrySList(head, entry)); } static slist_entry * ExInterlockedPopEntrySList(head, lock) slist_header *head; kspin_lock *lock; { return (InterlockedPopEntrySList(head)); } uint16_t ExQueryDepthSList(head) slist_header *head; { uint16_t depth; mtx_lock_spin(&ntoskrnl_interlock); depth = head->slh_list.slh_depth; mtx_unlock_spin(&ntoskrnl_interlock); return (depth); } void KeInitializeSpinLock(lock) kspin_lock *lock; { *lock = 0; } #ifdef __i386__ void KefAcquireSpinLockAtDpcLevel(lock) kspin_lock *lock; { #ifdef NTOSKRNL_DEBUG_SPINLOCKS int i = 0; #endif while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) { /* sit and spin */; #ifdef NTOSKRNL_DEBUG_SPINLOCKS i++; if (i > 200000000) panic("DEADLOCK!"); #endif } } void KefReleaseSpinLockFromDpcLevel(lock) kspin_lock *lock; { atomic_store_rel_int((volatile u_int *)lock, 0); } uint8_t KeAcquireSpinLockRaiseToDpc(kspin_lock *lock) { uint8_t oldirql; if (KeGetCurrentIrql() > DISPATCH_LEVEL) panic("IRQL_NOT_LESS_THAN_OR_EQUAL"); KeRaiseIrql(DISPATCH_LEVEL, &oldirql); KeAcquireSpinLockAtDpcLevel(lock); return (oldirql); } #else void KeAcquireSpinLockAtDpcLevel(kspin_lock *lock) { while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) /* sit and spin */; } void KeReleaseSpinLockFromDpcLevel(kspin_lock *lock) { atomic_store_rel_int((volatile u_int *)lock, 0); } #endif /* __i386__ */ uintptr_t InterlockedExchange(dst, val) volatile uint32_t *dst; uintptr_t val; { uintptr_t r; mtx_lock_spin(&ntoskrnl_interlock); r = *dst; *dst = val; mtx_unlock_spin(&ntoskrnl_interlock); return (r); } static uint32_t InterlockedIncrement(addend) volatile uint32_t *addend; { atomic_add_long((volatile u_long *)addend, 1); return (*addend); } static uint32_t InterlockedDecrement(addend) volatile uint32_t *addend; { atomic_subtract_long((volatile u_long *)addend, 1); return (*addend); } static void ExInterlockedAddLargeStatistic(addend, inc) uint64_t *addend; uint32_t inc; { mtx_lock_spin(&ntoskrnl_interlock); *addend += inc; mtx_unlock_spin(&ntoskrnl_interlock); }; mdl * IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf, uint8_t chargequota, irp *iopkt) { mdl *m; int zone = 0; if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE) m = ExAllocatePoolWithTag(NonPagedPool, MmSizeOfMdl(vaddr, len), 0); else { m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO); zone++; } if (m == NULL) return (NULL); MmInitializeMdl(m, vaddr, len); /* * MmInitializMdl() clears the flags field, so we * have to set this here. If the MDL came from the * MDL UMA zone, tag it so we can release it to * the right place later. */ if (zone) m->mdl_flags = MDL_ZONE_ALLOCED; if (iopkt != NULL) { if (secondarybuf == TRUE) { mdl *last; last = iopkt->irp_mdl; while (last->mdl_next != NULL) last = last->mdl_next; last->mdl_next = m; } else { if (iopkt->irp_mdl != NULL) panic("leaking an MDL in IoAllocateMdl()"); iopkt->irp_mdl = m; } } return (m); } void IoFreeMdl(m) mdl *m; { if (m == NULL) return; if (m->mdl_flags & MDL_ZONE_ALLOCED) uma_zfree(mdl_zone, m); else ExFreePool(m); } static void * MmAllocateContiguousMemory(size, highest) uint32_t size; uint64_t highest; { void *addr; size_t pagelength = roundup(size, PAGE_SIZE); addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0); return (addr); } static void * MmAllocateContiguousMemorySpecifyCache(size, lowest, highest, boundary, cachetype) uint32_t size; uint64_t lowest; uint64_t highest; uint64_t boundary; uint32_t cachetype; { void *addr; size_t pagelength = roundup(size, PAGE_SIZE); addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0); return (addr); } static void MmFreeContiguousMemory(base) void *base; { ExFreePool(base); } static void MmFreeContiguousMemorySpecifyCache(base, size, cachetype) void *base; uint32_t size; uint32_t cachetype; { ExFreePool(base); } static uint32_t MmSizeOfMdl(vaddr, len) void *vaddr; size_t len; { uint32_t l; l = sizeof(struct mdl) + (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len)); return (l); } /* * The Microsoft documentation says this routine fills in the * page array of an MDL with the _physical_ page addresses that * comprise the buffer, but we don't really want to do that here. * Instead, we just fill in the page array with the kernel virtual * addresses of the buffers. */ void MmBuildMdlForNonPagedPool(m) mdl *m; { vm_offset_t *mdl_pages; int pagecnt, i; pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount); if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *)) panic("not enough pages in MDL to describe buffer"); mdl_pages = MmGetMdlPfnArray(m); for (i = 0; i < pagecnt; i++) *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE); m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL; m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m); } static void * MmMapLockedPages(mdl *buf, uint8_t accessmode) { buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA; return (MmGetMdlVirtualAddress(buf)); } static void * MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype, void *vaddr, uint32_t bugcheck, uint32_t prio) { return (MmMapLockedPages(buf, accessmode)); } static void MmUnmapLockedPages(vaddr, buf) void *vaddr; mdl *buf; { buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA; } /* * This function has a problem in that it will break if you * compile this module without PAE and try to use it on a PAE * kernel. Unfortunately, there's no way around this at the * moment. It's slightly less broken that using pmap_kextract(). * You'd think the virtual memory subsystem would help us out * here, but it doesn't. */ uint8_t MmIsAddressValid(vaddr) void *vaddr; { if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr)) return (TRUE); return (FALSE); } void * MmMapIoSpace(paddr, len, cachetype) uint64_t paddr; uint32_t len; uint32_t cachetype; { devclass_t nexus_class; device_t *nexus_devs, devp; int nexus_count = 0; device_t matching_dev = NULL; struct resource *res; int i; vm_offset_t v; /* There will always be at least one nexus. */ nexus_class = devclass_find("nexus"); devclass_get_devices(nexus_class, &nexus_devs, &nexus_count); for (i = 0; i < nexus_count; i++) { devp = nexus_devs[i]; matching_dev = ntoskrnl_finddev(devp, paddr, &res); if (matching_dev) break; } free(nexus_devs, M_TEMP); if (matching_dev == NULL) return (NULL); v = (vm_offset_t)rman_get_virtual(res); if (paddr > rman_get_start(res)) v += paddr - rman_get_start(res); return ((void *)v); } void MmUnmapIoSpace(vaddr, len) void *vaddr; size_t len; { } static device_t ntoskrnl_finddev(dev, paddr, res) device_t dev; uint64_t paddr; struct resource **res; { device_t *children = NULL; device_t matching_dev; int childcnt; struct resource *r; struct resource_list *rl; struct resource_list_entry *rle; uint32_t flags; int i; /* We only want devices that have been successfully probed. */ if (device_is_alive(dev) == FALSE) return (NULL); rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev); if (rl != NULL) { -#if __FreeBSD_version < 600022 - SLIST_FOREACH(rle, rl, link) { -#else STAILQ_FOREACH(rle, rl, link) { -#endif r = rle->res; if (r == NULL) continue; flags = rman_get_flags(r); if (rle->type == SYS_RES_MEMORY && paddr >= rman_get_start(r) && paddr <= rman_get_end(r)) { if (!(flags & RF_ACTIVE)) bus_activate_resource(dev, SYS_RES_MEMORY, 0, r); *res = r; return (dev); } } } /* * If this device has children, do another * level of recursion to inspect them. */ device_get_children(dev, &children, &childcnt); for (i = 0; i < childcnt; i++) { matching_dev = ntoskrnl_finddev(children[i], paddr, res); if (matching_dev != NULL) { free(children, M_TEMP); return (matching_dev); } } /* Won't somebody please think of the children! */ if (children != NULL) free(children, M_TEMP); return (NULL); } /* * Workitems are unlike DPCs, in that they run in a user-mode thread * context rather than at DISPATCH_LEVEL in kernel context. In our * case we run them in kernel context anyway. */ static void ntoskrnl_workitem_thread(arg) void *arg; { kdpc_queue *kq; list_entry *l; io_workitem *iw; uint8_t irql; kq = arg; InitializeListHead(&kq->kq_disp); kq->kq_td = curthread; kq->kq_exit = 0; KeInitializeSpinLock(&kq->kq_lock); KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); while (1) { KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL); KeAcquireSpinLock(&kq->kq_lock, &irql); if (kq->kq_exit) { kq->kq_exit = 0; KeReleaseSpinLock(&kq->kq_lock, irql); break; } while (!IsListEmpty(&kq->kq_disp)) { l = RemoveHeadList(&kq->kq_disp); iw = CONTAINING_RECORD(l, io_workitem, iw_listentry); InitializeListHead((&iw->iw_listentry)); if (iw->iw_func == NULL) continue; KeReleaseSpinLock(&kq->kq_lock, irql); MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx); KeAcquireSpinLock(&kq->kq_lock, &irql); } KeReleaseSpinLock(&kq->kq_lock, irql); } -#if __FreeBSD_version < 502113 - mtx_lock(&Giant); -#endif kproc_exit(0); return; /* notreached */ } static void ntoskrnl_destroy_workitem_threads(void) { kdpc_queue *kq; int i; for (i = 0; i < WORKITEM_THREADS; i++) { kq = wq_queues + i; kq->kq_exit = 1; KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); while (kq->kq_exit) tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10); } } io_workitem * IoAllocateWorkItem(dobj) device_object *dobj; { io_workitem *iw; iw = uma_zalloc(iw_zone, M_NOWAIT); if (iw == NULL) return (NULL); InitializeListHead(&iw->iw_listentry); iw->iw_dobj = dobj; mtx_lock(&ntoskrnl_dispatchlock); iw->iw_idx = wq_idx; WORKIDX_INC(wq_idx); mtx_unlock(&ntoskrnl_dispatchlock); return (iw); } void IoFreeWorkItem(iw) io_workitem *iw; { uma_zfree(iw_zone, iw); } void IoQueueWorkItem(iw, iw_func, qtype, ctx) io_workitem *iw; io_workitem_func iw_func; uint32_t qtype; void *ctx; { kdpc_queue *kq; list_entry *l; io_workitem *cur; uint8_t irql; kq = wq_queues + iw->iw_idx; KeAcquireSpinLock(&kq->kq_lock, &irql); /* * Traverse the list and make sure this workitem hasn't * already been inserted. Queuing the same workitem * twice will hose the list but good. */ l = kq->kq_disp.nle_flink; while (l != &kq->kq_disp) { cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); if (cur == iw) { /* Already queued -- do nothing. */ KeReleaseSpinLock(&kq->kq_lock, irql); return; } l = l->nle_flink; } iw->iw_func = iw_func; iw->iw_ctx = ctx; InsertTailList((&kq->kq_disp), (&iw->iw_listentry)); KeReleaseSpinLock(&kq->kq_lock, irql); KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); } static void ntoskrnl_workitem(dobj, arg) device_object *dobj; void *arg; { io_workitem *iw; work_queue_item *w; work_item_func f; iw = arg; w = (work_queue_item *)dobj; f = (work_item_func)w->wqi_func; uma_zfree(iw_zone, iw); MSCALL2(f, w, w->wqi_ctx); } /* * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft * warns that it's unsafe and to use IoQueueWorkItem() instead. The * problem with ExQueueWorkItem() is that it can't guard against * the condition where a driver submits a job to the work queue and * is then unloaded before the job is able to run. IoQueueWorkItem() * acquires a reference to the device's device_object via the * object manager and retains it until after the job has completed, * which prevents the driver from being unloaded before the job * runs. (We don't currently support this behavior, though hopefully * that will change once the object manager API is fleshed out a bit.) * * Having said all that, the ExQueueWorkItem() API remains, because * there are still other parts of Windows that use it, including * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem(). * We fake up the ExQueueWorkItem() API on top of our implementation * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively * for ExQueueWorkItem() jobs, and we pass a pointer to the work * queue item (provided by the caller) in to IoAllocateWorkItem() * instead of the device_object. We need to save this pointer so * we can apply a sanity check: as with the DPC queue and other * workitem queues, we can't allow the same work queue item to * be queued twice. If it's already pending, we silently return */ void ExQueueWorkItem(w, qtype) work_queue_item *w; uint32_t qtype; { io_workitem *iw; io_workitem_func iwf; kdpc_queue *kq; list_entry *l; io_workitem *cur; uint8_t irql; /* * We need to do a special sanity test to make sure * the ExQueueWorkItem() API isn't used to queue * the same workitem twice. Rather than checking the * io_workitem pointer itself, we test the attached * device object, which is really a pointer to the * legacy work queue item structure. */ kq = wq_queues + WORKITEM_LEGACY_THREAD; KeAcquireSpinLock(&kq->kq_lock, &irql); l = kq->kq_disp.nle_flink; while (l != &kq->kq_disp) { cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); if (cur->iw_dobj == (device_object *)w) { /* Already queued -- do nothing. */ KeReleaseSpinLock(&kq->kq_lock, irql); return; } l = l->nle_flink; } KeReleaseSpinLock(&kq->kq_lock, irql); iw = IoAllocateWorkItem((device_object *)w); if (iw == NULL) return; iw->iw_idx = WORKITEM_LEGACY_THREAD; iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem); IoQueueWorkItem(iw, iwf, qtype, iw); } static void RtlZeroMemory(dst, len) void *dst; size_t len; { bzero(dst, len); } static void RtlCopyMemory(dst, src, len) void *dst; const void *src; size_t len; { bcopy(src, dst, len); } static size_t RtlCompareMemory(s1, s2, len) const void *s1; const void *s2; size_t len; { size_t i, total = 0; uint8_t *m1, *m2; m1 = __DECONST(char *, s1); m2 = __DECONST(char *, s2); for (i = 0; i < len; i++) { if (m1[i] == m2[i]) total++; } return (total); } void RtlInitAnsiString(dst, src) ansi_string *dst; char *src; { ansi_string *a; a = dst; if (a == NULL) return; if (src == NULL) { a->as_len = a->as_maxlen = 0; a->as_buf = NULL; } else { a->as_buf = src; a->as_len = a->as_maxlen = strlen(src); } } void RtlInitUnicodeString(dst, src) unicode_string *dst; uint16_t *src; { unicode_string *u; int i; u = dst; if (u == NULL) return; if (src == NULL) { u->us_len = u->us_maxlen = 0; u->us_buf = NULL; } else { i = 0; while(src[i] != 0) i++; u->us_buf = src; u->us_len = u->us_maxlen = i * 2; } } ndis_status RtlUnicodeStringToInteger(ustr, base, val) unicode_string *ustr; uint32_t base; uint32_t *val; { uint16_t *uchr; int len, neg = 0; char abuf[64]; char *astr; uchr = ustr->us_buf; len = ustr->us_len; bzero(abuf, sizeof(abuf)); if ((char)((*uchr) & 0xFF) == '-') { neg = 1; uchr++; len -= 2; } else if ((char)((*uchr) & 0xFF) == '+') { neg = 0; uchr++; len -= 2; } if (base == 0) { if ((char)((*uchr) & 0xFF) == 'b') { base = 2; uchr++; len -= 2; } else if ((char)((*uchr) & 0xFF) == 'o') { base = 8; uchr++; len -= 2; } else if ((char)((*uchr) & 0xFF) == 'x') { base = 16; uchr++; len -= 2; } else base = 10; } astr = abuf; if (neg) { strcpy(astr, "-"); astr++; } ntoskrnl_unicode_to_ascii(uchr, astr, len); *val = strtoul(abuf, NULL, base); return (STATUS_SUCCESS); } void RtlFreeUnicodeString(ustr) unicode_string *ustr; { if (ustr->us_buf == NULL) return; ExFreePool(ustr->us_buf); ustr->us_buf = NULL; } void RtlFreeAnsiString(astr) ansi_string *astr; { if (astr->as_buf == NULL) return; ExFreePool(astr->as_buf); astr->as_buf = NULL; } static int atoi(str) const char *str; { return (int)strtol(str, (char **)NULL, 10); } static long atol(str) const char *str; { return strtol(str, (char **)NULL, 10); } static int rand(void) { struct timeval tv; microtime(&tv); srandom(tv.tv_usec); return ((int)random()); } static void srand(seed) unsigned int seed; { srandom(seed); } static uint8_t IoIsWdmVersionAvailable(uint8_t major, uint8_t minor) { if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP) return (TRUE); return (FALSE); } static ndis_status IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj) unicode_string *name; uint32_t reqaccess; void *fileobj; device_object *devobj; { return (STATUS_SUCCESS); } static ndis_status IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen) device_object *devobj; uint32_t regprop; uint32_t buflen; void *prop; uint32_t *reslen; { driver_object *drv; uint16_t **name; drv = devobj->do_drvobj; switch (regprop) { case DEVPROP_DRIVER_KEYNAME: name = prop; *name = drv->dro_drivername.us_buf; *reslen = drv->dro_drivername.us_len; break; default: return (STATUS_INVALID_PARAMETER_2); break; } return (STATUS_SUCCESS); } static void KeInitializeMutex(kmutex, level) kmutant *kmutex; uint32_t level; { InitializeListHead((&kmutex->km_header.dh_waitlisthead)); kmutex->km_abandoned = FALSE; kmutex->km_apcdisable = 1; kmutex->km_header.dh_sigstate = 1; kmutex->km_header.dh_type = DISP_TYPE_MUTANT; kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t); kmutex->km_ownerthread = NULL; } static uint32_t KeReleaseMutex(kmutant *kmutex, uint8_t kwait) { uint32_t prevstate; mtx_lock(&ntoskrnl_dispatchlock); prevstate = kmutex->km_header.dh_sigstate; if (kmutex->km_ownerthread != curthread) { mtx_unlock(&ntoskrnl_dispatchlock); return (STATUS_MUTANT_NOT_OWNED); } kmutex->km_header.dh_sigstate++; kmutex->km_abandoned = FALSE; if (kmutex->km_header.dh_sigstate == 1) { kmutex->km_ownerthread = NULL; ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT); } mtx_unlock(&ntoskrnl_dispatchlock); return (prevstate); } static uint32_t KeReadStateMutex(kmutex) kmutant *kmutex; { return (kmutex->km_header.dh_sigstate); } void KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state) { InitializeListHead((&kevent->k_header.dh_waitlisthead)); kevent->k_header.dh_sigstate = state; if (type == EVENT_TYPE_NOTIFY) kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT; else kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT; kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t); } uint32_t KeResetEvent(kevent) nt_kevent *kevent; { uint32_t prevstate; mtx_lock(&ntoskrnl_dispatchlock); prevstate = kevent->k_header.dh_sigstate; kevent->k_header.dh_sigstate = FALSE; mtx_unlock(&ntoskrnl_dispatchlock); return (prevstate); } uint32_t KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait) { uint32_t prevstate; wait_block *w; nt_dispatch_header *dh; struct thread *td; wb_ext *we; mtx_lock(&ntoskrnl_dispatchlock); prevstate = kevent->k_header.dh_sigstate; dh = &kevent->k_header; if (IsListEmpty(&dh->dh_waitlisthead)) /* * If there's nobody in the waitlist, just set * the state to signalled. */ dh->dh_sigstate = 1; else { /* * Get the first waiter. If this is a synchronization * event, just wake up that one thread (don't bother * setting the state to signalled since we're supposed * to automatically clear synchronization events anyway). * * If it's a notification event, or the the first * waiter is doing a WAITTYPE_ALL wait, go through * the full wait satisfaction process. */ w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink, wait_block, wb_waitlist); we = w->wb_ext; td = we->we_td; if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT || w->wb_waittype == WAITTYPE_ALL) { if (prevstate == 0) { dh->dh_sigstate = 1; ntoskrnl_waittest(dh, increment); } } else { w->wb_awakened |= TRUE; cv_broadcastpri(&we->we_cv, (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ? w->wb_oldpri - (increment * 4) : PRI_MIN_KERN); } } mtx_unlock(&ntoskrnl_dispatchlock); return (prevstate); } void KeClearEvent(kevent) nt_kevent *kevent; { kevent->k_header.dh_sigstate = FALSE; } uint32_t KeReadStateEvent(kevent) nt_kevent *kevent; { return (kevent->k_header.dh_sigstate); } /* * The object manager in Windows is responsible for managing * references and access to various types of objects, including * device_objects, events, threads, timers and so on. However, * there's a difference in the way objects are handled in user * mode versus kernel mode. * * In user mode (i.e. Win32 applications), all objects are * managed by the object manager. For example, when you create * a timer or event object, you actually end up with an * object_header (for the object manager's bookkeeping * purposes) and an object body (which contains the actual object * structure, e.g. ktimer, kevent, etc...). This allows Windows * to manage resource quotas and to enforce access restrictions * on basically every kind of system object handled by the kernel. * * However, in kernel mode, you only end up using the object * manager some of the time. For example, in a driver, you create * a timer object by simply allocating the memory for a ktimer * structure and initializing it with KeInitializeTimer(). Hence, * the timer has no object_header and no reference counting or * security/resource checks are done on it. The assumption in * this case is that if you're running in kernel mode, you know * what you're doing, and you're already at an elevated privilege * anyway. * * There are some exceptions to this. The two most important ones * for our purposes are device_objects and threads. We need to use * the object manager to do reference counting on device_objects, * and for threads, you can only get a pointer to a thread's * dispatch header by using ObReferenceObjectByHandle() on the * handle returned by PsCreateSystemThread(). */ static ndis_status ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype, uint8_t accessmode, void **object, void **handleinfo) { nt_objref *nr; nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO); if (nr == NULL) return (STATUS_INSUFFICIENT_RESOURCES); InitializeListHead((&nr->no_dh.dh_waitlisthead)); nr->no_obj = handle; nr->no_dh.dh_type = DISP_TYPE_THREAD; nr->no_dh.dh_sigstate = 0; nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) / sizeof(uint32_t)); TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link); *object = nr; return (STATUS_SUCCESS); } static void ObfDereferenceObject(object) void *object; { nt_objref *nr; nr = object; TAILQ_REMOVE(&ntoskrnl_reflist, nr, link); free(nr, M_DEVBUF); } static uint32_t ZwClose(handle) ndis_handle handle; { return (STATUS_SUCCESS); } static uint32_t WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf) uint32_t traceclass; void *traceinfo; uint32_t infolen; uint32_t reqlen; void *buf; { return (STATUS_NOT_FOUND); } static uint32_t WmiTraceMessage(uint64_t loghandle, uint32_t messageflags, void *guid, uint16_t messagenum, ...) { return (STATUS_SUCCESS); } static uint32_t IoWMIRegistrationControl(dobj, action) device_object *dobj; uint32_t action; { return (STATUS_SUCCESS); } /* * This is here just in case the thread returns without calling * PsTerminateSystemThread(). */ static void ntoskrnl_thrfunc(arg) void *arg; { thread_context *thrctx; uint32_t (*tfunc)(void *); void *tctx; uint32_t rval; thrctx = arg; tfunc = thrctx->tc_thrfunc; tctx = thrctx->tc_thrctx; free(thrctx, M_TEMP); rval = MSCALL1(tfunc, tctx); PsTerminateSystemThread(rval); return; /* notreached */ } static ndis_status PsCreateSystemThread(handle, reqaccess, objattrs, phandle, clientid, thrfunc, thrctx) ndis_handle *handle; uint32_t reqaccess; void *objattrs; ndis_handle phandle; void *clientid; void *thrfunc; void *thrctx; { int error; thread_context *tc; struct proc *p; tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT); if (tc == NULL) return (STATUS_INSUFFICIENT_RESOURCES); tc->tc_thrctx = thrctx; tc->tc_thrfunc = thrfunc; error = kproc_create(ntoskrnl_thrfunc, tc, &p, RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth); if (error) { free(tc, M_TEMP); return (STATUS_INSUFFICIENT_RESOURCES); } *handle = p; ntoskrnl_kth++; return (STATUS_SUCCESS); } /* * In Windows, the exit of a thread is an event that you're allowed * to wait on, assuming you've obtained a reference to the thread using * ObReferenceObjectByHandle(). Unfortunately, the only way we can * simulate this behavior is to register each thread we create in a * reference list, and if someone holds a reference to us, we poke * them. */ static ndis_status PsTerminateSystemThread(status) ndis_status status; { struct nt_objref *nr; mtx_lock(&ntoskrnl_dispatchlock); TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) { if (nr->no_obj != curthread->td_proc) continue; nr->no_dh.dh_sigstate = 1; ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT); break; } mtx_unlock(&ntoskrnl_dispatchlock); ntoskrnl_kth--; -#if __FreeBSD_version < 502113 - mtx_lock(&Giant); -#endif kproc_exit(0); return (0); /* notreached */ } static uint32_t DbgPrint(char *fmt, ...) { va_list ap; if (bootverbose) { va_start(ap, fmt); vprintf(fmt, ap); } return (STATUS_SUCCESS); } static void DbgBreakPoint(void) { -#if __FreeBSD_version < 502113 - Debugger("DbgBreakPoint(): breakpoint"); -#else kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint"); -#endif } static void KeBugCheckEx(code, param1, param2, param3, param4) uint32_t code; u_long param1; u_long param2; u_long param3; u_long param4; { panic("KeBugCheckEx: STOP 0x%X", code); } static void ntoskrnl_timercall(arg) void *arg; { ktimer *timer; struct timeval tv; kdpc *dpc; mtx_lock(&ntoskrnl_dispatchlock); timer = arg; #ifdef NTOSKRNL_DEBUG_TIMERS ntoskrnl_timer_fires++; #endif ntoskrnl_remove_timer(timer); /* * This should never happen, but complain * if it does. */ if (timer->k_header.dh_inserted == FALSE) { mtx_unlock(&ntoskrnl_dispatchlock); printf("NTOS: timer %p fired even though " "it was canceled\n", timer); return; } /* Mark the timer as no longer being on the timer queue. */ timer->k_header.dh_inserted = FALSE; /* Now signal the object and satisfy any waits on it. */ timer->k_header.dh_sigstate = 1; ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT); /* * If this is a periodic timer, re-arm it * so it will fire again. We do this before * calling any deferred procedure calls because * it's possible the DPC might cancel the timer, * in which case it would be wrong for us to * re-arm it again afterwards. */ if (timer->k_period) { tv.tv_sec = 0; tv.tv_usec = timer->k_period * 1000; timer->k_header.dh_inserted = TRUE; ntoskrnl_insert_timer(timer, tvtohz(&tv)); #ifdef NTOSKRNL_DEBUG_TIMERS ntoskrnl_timer_reloads++; #endif } dpc = timer->k_dpc; mtx_unlock(&ntoskrnl_dispatchlock); /* If there's a DPC associated with the timer, queue it up. */ if (dpc != NULL) KeInsertQueueDpc(dpc, NULL, NULL); } #ifdef NTOSKRNL_DEBUG_TIMERS static int sysctl_show_timers(SYSCTL_HANDLER_ARGS) { int ret; ret = 0; ntoskrnl_show_timers(); return (sysctl_handle_int(oidp, &ret, 0, req)); } static void ntoskrnl_show_timers() { int i = 0; list_entry *l; mtx_lock_spin(&ntoskrnl_calllock); l = ntoskrnl_calllist.nle_flink; while(l != &ntoskrnl_calllist) { i++; l = l->nle_flink; } mtx_unlock_spin(&ntoskrnl_calllock); printf("\n"); printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS); printf("timer sets: %qu\n", ntoskrnl_timer_sets); printf("timer reloads: %qu\n", ntoskrnl_timer_reloads); printf("timer cancels: %qu\n", ntoskrnl_timer_cancels); printf("timer fires: %qu\n", ntoskrnl_timer_fires); printf("\n"); } #endif /* * Must be called with dispatcher lock held. */ static void ntoskrnl_insert_timer(timer, ticks) ktimer *timer; int ticks; { callout_entry *e; list_entry *l; struct callout *c; /* * Try and allocate a timer. */ mtx_lock_spin(&ntoskrnl_calllock); if (IsListEmpty(&ntoskrnl_calllist)) { mtx_unlock_spin(&ntoskrnl_calllock); #ifdef NTOSKRNL_DEBUG_TIMERS ntoskrnl_show_timers(); #endif panic("out of timers!"); } l = RemoveHeadList(&ntoskrnl_calllist); mtx_unlock_spin(&ntoskrnl_calllock); e = CONTAINING_RECORD(l, callout_entry, ce_list); c = &e->ce_callout; timer->k_callout = c; callout_init(c, CALLOUT_MPSAFE); callout_reset(c, ticks, ntoskrnl_timercall, timer); } static void ntoskrnl_remove_timer(timer) ktimer *timer; { callout_entry *e; e = (callout_entry *)timer->k_callout; callout_stop(timer->k_callout); mtx_lock_spin(&ntoskrnl_calllock); InsertHeadList((&ntoskrnl_calllist), (&e->ce_list)); mtx_unlock_spin(&ntoskrnl_calllock); } void KeInitializeTimer(timer) ktimer *timer; { if (timer == NULL) return; KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY); } void KeInitializeTimerEx(timer, type) ktimer *timer; uint32_t type; { if (timer == NULL) return; bzero((char *)timer, sizeof(ktimer)); InitializeListHead((&timer->k_header.dh_waitlisthead)); timer->k_header.dh_sigstate = FALSE; timer->k_header.dh_inserted = FALSE; if (type == EVENT_TYPE_NOTIFY) timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER; else timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER; timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t); } /* * DPC subsystem. A Windows Defered Procedure Call has the following * properties: * - It runs at DISPATCH_LEVEL. * - It can have one of 3 importance values that control when it * runs relative to other DPCs in the queue. * - On SMP systems, it can be set to run on a specific processor. * In order to satisfy the last property, we create a DPC thread for * each CPU in the system and bind it to that CPU. Each thread * maintains three queues with different importance levels, which * will be processed in order from lowest to highest. * * In Windows, interrupt handlers run as DPCs. (Not to be confused * with ISRs, which run in interrupt context and can preempt DPCs.) * ISRs are given the highest importance so that they'll take * precedence over timers and other things. */ static void ntoskrnl_dpc_thread(arg) void *arg; { kdpc_queue *kq; kdpc *d; list_entry *l; uint8_t irql; kq = arg; InitializeListHead(&kq->kq_disp); kq->kq_td = curthread; kq->kq_exit = 0; kq->kq_running = FALSE; KeInitializeSpinLock(&kq->kq_lock); KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE); /* * Elevate our priority. DPCs are used to run interrupt * handlers, and they should trigger as soon as possible * once scheduled by an ISR. */ thread_lock(curthread); #ifdef NTOSKRNL_MULTIPLE_DPCS -#if __FreeBSD_version >= 502102 sched_bind(curthread, kq->kq_cpu); #endif -#endif sched_prio(curthread, PRI_MIN_KERN); -#if __FreeBSD_version < 600000 - curthread->td_base_pri = PRI_MIN_KERN; -#endif thread_unlock(curthread); while (1) { KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL); KeAcquireSpinLock(&kq->kq_lock, &irql); if (kq->kq_exit) { kq->kq_exit = 0; KeReleaseSpinLock(&kq->kq_lock, irql); break; } kq->kq_running = TRUE; while (!IsListEmpty(&kq->kq_disp)) { l = RemoveHeadList((&kq->kq_disp)); d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); InitializeListHead((&d->k_dpclistentry)); KeReleaseSpinLockFromDpcLevel(&kq->kq_lock); MSCALL4(d->k_deferedfunc, d, d->k_deferredctx, d->k_sysarg1, d->k_sysarg2); KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); } kq->kq_running = FALSE; KeReleaseSpinLock(&kq->kq_lock, irql); KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE); } -#if __FreeBSD_version < 502113 - mtx_lock(&Giant); -#endif kproc_exit(0); return; /* notreached */ } static void ntoskrnl_destroy_dpc_threads(void) { kdpc_queue *kq; kdpc dpc; int i; kq = kq_queues; #ifdef NTOSKRNL_MULTIPLE_DPCS for (i = 0; i < mp_ncpus; i++) { #else for (i = 0; i < 1; i++) { #endif kq += i; kq->kq_exit = 1; KeInitializeDpc(&dpc, NULL, NULL); KeSetTargetProcessorDpc(&dpc, i); KeInsertQueueDpc(&dpc, NULL, NULL); while (kq->kq_exit) tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10); } } static uint8_t ntoskrnl_insert_dpc(head, dpc) list_entry *head; kdpc *dpc; { list_entry *l; kdpc *d; l = head->nle_flink; while (l != head) { d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); if (d == dpc) return (FALSE); l = l->nle_flink; } if (dpc->k_importance == KDPC_IMPORTANCE_LOW) InsertTailList((head), (&dpc->k_dpclistentry)); else InsertHeadList((head), (&dpc->k_dpclistentry)); return (TRUE); } void KeInitializeDpc(dpc, dpcfunc, dpcctx) kdpc *dpc; void *dpcfunc; void *dpcctx; { if (dpc == NULL) return; dpc->k_deferedfunc = dpcfunc; dpc->k_deferredctx = dpcctx; dpc->k_num = KDPC_CPU_DEFAULT; dpc->k_importance = KDPC_IMPORTANCE_MEDIUM; InitializeListHead((&dpc->k_dpclistentry)); } uint8_t KeInsertQueueDpc(dpc, sysarg1, sysarg2) kdpc *dpc; void *sysarg1; void *sysarg2; { kdpc_queue *kq; uint8_t r; uint8_t irql; if (dpc == NULL) return (FALSE); kq = kq_queues; #ifdef NTOSKRNL_MULTIPLE_DPCS KeRaiseIrql(DISPATCH_LEVEL, &irql); /* * By default, the DPC is queued to run on the same CPU * that scheduled it. */ if (dpc->k_num == KDPC_CPU_DEFAULT) kq += curthread->td_oncpu; else kq += dpc->k_num; KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); #else KeAcquireSpinLock(&kq->kq_lock, &irql); #endif r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc); if (r == TRUE) { dpc->k_sysarg1 = sysarg1; dpc->k_sysarg2 = sysarg2; } KeReleaseSpinLock(&kq->kq_lock, irql); if (r == FALSE) return (r); KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); return (r); } uint8_t KeRemoveQueueDpc(dpc) kdpc *dpc; { kdpc_queue *kq; uint8_t irql; if (dpc == NULL) return (FALSE); #ifdef NTOSKRNL_MULTIPLE_DPCS KeRaiseIrql(DISPATCH_LEVEL, &irql); kq = kq_queues + dpc->k_num; KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); #else kq = kq_queues; KeAcquireSpinLock(&kq->kq_lock, &irql); #endif if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) { KeReleaseSpinLockFromDpcLevel(&kq->kq_lock); KeLowerIrql(irql); return (FALSE); } RemoveEntryList((&dpc->k_dpclistentry)); InitializeListHead((&dpc->k_dpclistentry)); KeReleaseSpinLock(&kq->kq_lock, irql); return (TRUE); } void KeSetImportanceDpc(dpc, imp) kdpc *dpc; uint32_t imp; { if (imp != KDPC_IMPORTANCE_LOW && imp != KDPC_IMPORTANCE_MEDIUM && imp != KDPC_IMPORTANCE_HIGH) return; dpc->k_importance = (uint8_t)imp; } void KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu) { if (cpu > mp_ncpus) return; dpc->k_num = cpu; } void KeFlushQueuedDpcs(void) { kdpc_queue *kq; int i; /* * Poke each DPC queue and wait * for them to drain. */ #ifdef NTOSKRNL_MULTIPLE_DPCS for (i = 0; i < mp_ncpus; i++) { #else for (i = 0; i < 1; i++) { #endif kq = kq_queues + i; KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL); } } uint32_t KeGetCurrentProcessorNumber(void) { return ((uint32_t)curthread->td_oncpu); } uint8_t KeSetTimerEx(timer, duetime, period, dpc) ktimer *timer; int64_t duetime; uint32_t period; kdpc *dpc; { struct timeval tv; uint64_t curtime; uint8_t pending; if (timer == NULL) return (FALSE); mtx_lock(&ntoskrnl_dispatchlock); if (timer->k_header.dh_inserted == TRUE) { ntoskrnl_remove_timer(timer); #ifdef NTOSKRNL_DEBUG_TIMERS ntoskrnl_timer_cancels++; #endif timer->k_header.dh_inserted = FALSE; pending = TRUE; } else pending = FALSE; timer->k_duetime = duetime; timer->k_period = period; timer->k_header.dh_sigstate = FALSE; timer->k_dpc = dpc; if (duetime < 0) { tv.tv_sec = - (duetime) / 10000000; tv.tv_usec = (- (duetime) / 10) - (tv.tv_sec * 1000000); } else { ntoskrnl_time(&curtime); if (duetime < curtime) tv.tv_sec = tv.tv_usec = 0; else { tv.tv_sec = ((duetime) - curtime) / 10000000; tv.tv_usec = ((duetime) - curtime) / 10 - (tv.tv_sec * 1000000); } } timer->k_header.dh_inserted = TRUE; ntoskrnl_insert_timer(timer, tvtohz(&tv)); #ifdef NTOSKRNL_DEBUG_TIMERS ntoskrnl_timer_sets++; #endif mtx_unlock(&ntoskrnl_dispatchlock); return (pending); } uint8_t KeSetTimer(timer, duetime, dpc) ktimer *timer; int64_t duetime; kdpc *dpc; { return (KeSetTimerEx(timer, duetime, 0, dpc)); } /* * The Windows DDK documentation seems to say that cancelling * a timer that has a DPC will result in the DPC also being * cancelled, but this isn't really the case. */ uint8_t KeCancelTimer(timer) ktimer *timer; { uint8_t pending; if (timer == NULL) return (FALSE); mtx_lock(&ntoskrnl_dispatchlock); pending = timer->k_header.dh_inserted; if (timer->k_header.dh_inserted == TRUE) { timer->k_header.dh_inserted = FALSE; ntoskrnl_remove_timer(timer); #ifdef NTOSKRNL_DEBUG_TIMERS ntoskrnl_timer_cancels++; #endif } mtx_unlock(&ntoskrnl_dispatchlock); return (pending); } uint8_t KeReadStateTimer(timer) ktimer *timer; { return (timer->k_header.dh_sigstate); } static int32_t KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval) { ktimer timer; if (wait_mode != 0) panic("invalid wait_mode %d", wait_mode); KeInitializeTimer(&timer); KeSetTimer(&timer, *interval, NULL); KeWaitForSingleObject(&timer, 0, 0, alertable, NULL); return STATUS_SUCCESS; } static uint64_t KeQueryInterruptTime(void) { int ticks; struct timeval tv; getmicrouptime(&tv); ticks = tvtohz(&tv); return ticks * ((10000000 + hz - 1) / hz); } static struct thread * KeGetCurrentThread(void) { return curthread; } static int32_t KeSetPriorityThread(td, pri) struct thread *td; int32_t pri; { int32_t old; if (td == NULL) return LOW_REALTIME_PRIORITY; if (td->td_priority <= PRI_MIN_KERN) old = HIGH_PRIORITY; else if (td->td_priority >= PRI_MAX_KERN) old = LOW_PRIORITY; else old = LOW_REALTIME_PRIORITY; thread_lock(td); if (pri == HIGH_PRIORITY) sched_prio(td, PRI_MIN_KERN); if (pri == LOW_REALTIME_PRIORITY) sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2); if (pri == LOW_PRIORITY) sched_prio(td, PRI_MAX_KERN); thread_unlock(td); return old; } static void dummy() { printf("ntoskrnl dummy called...\n"); } image_patch_table ntoskrnl_functbl[] = { IMPORT_SFUNC(RtlZeroMemory, 2), IMPORT_SFUNC(RtlCopyMemory, 3), IMPORT_SFUNC(RtlCompareMemory, 3), IMPORT_SFUNC(RtlEqualUnicodeString, 3), IMPORT_SFUNC(RtlCopyUnicodeString, 2), IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3), IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3), IMPORT_SFUNC(RtlInitAnsiString, 2), IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2), IMPORT_SFUNC(RtlInitUnicodeString, 2), IMPORT_SFUNC(RtlFreeAnsiString, 1), IMPORT_SFUNC(RtlFreeUnicodeString, 1), IMPORT_SFUNC(RtlUnicodeStringToInteger, 3), IMPORT_CFUNC(sprintf, 0), IMPORT_CFUNC(vsprintf, 0), IMPORT_CFUNC_MAP(_snprintf, snprintf, 0), IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0), IMPORT_CFUNC(DbgPrint, 0), IMPORT_SFUNC(DbgBreakPoint, 0), IMPORT_SFUNC(KeBugCheckEx, 5), IMPORT_CFUNC(strncmp, 0), IMPORT_CFUNC(strcmp, 0), IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0), IMPORT_CFUNC(strncpy, 0), IMPORT_CFUNC(strcpy, 0), IMPORT_CFUNC(strlen, 0), IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0), IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0), IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0), IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0), IMPORT_CFUNC_MAP(strchr, index, 0), IMPORT_CFUNC_MAP(strrchr, rindex, 0), IMPORT_CFUNC(memcpy, 0), IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0), IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0), IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0), IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4), IMPORT_SFUNC(IoGetDriverObjectExtension, 2), IMPORT_FFUNC(IofCallDriver, 2), IMPORT_FFUNC(IofCompleteRequest, 2), IMPORT_SFUNC(IoAcquireCancelSpinLock, 1), IMPORT_SFUNC(IoReleaseCancelSpinLock, 1), IMPORT_SFUNC(IoCancelIrp, 1), IMPORT_SFUNC(IoConnectInterrupt, 11), IMPORT_SFUNC(IoDisconnectInterrupt, 1), IMPORT_SFUNC(IoCreateDevice, 7), IMPORT_SFUNC(IoDeleteDevice, 1), IMPORT_SFUNC(IoGetAttachedDevice, 1), IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2), IMPORT_SFUNC(IoDetachDevice, 1), IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7), IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6), IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9), IMPORT_SFUNC(IoAllocateIrp, 2), IMPORT_SFUNC(IoReuseIrp, 2), IMPORT_SFUNC(IoMakeAssociatedIrp, 2), IMPORT_SFUNC(IoFreeIrp, 1), IMPORT_SFUNC(IoInitializeIrp, 3), IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1), IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2), IMPORT_SFUNC(KeSynchronizeExecution, 3), IMPORT_SFUNC(KeWaitForSingleObject, 5), IMPORT_SFUNC(KeWaitForMultipleObjects, 8), IMPORT_SFUNC(_allmul, 4), IMPORT_SFUNC(_alldiv, 4), IMPORT_SFUNC(_allrem, 4), IMPORT_RFUNC(_allshr, 0), IMPORT_RFUNC(_allshl, 0), IMPORT_SFUNC(_aullmul, 4), IMPORT_SFUNC(_aulldiv, 4), IMPORT_SFUNC(_aullrem, 4), IMPORT_RFUNC(_aullshr, 0), IMPORT_RFUNC(_aullshl, 0), IMPORT_CFUNC(atoi, 0), IMPORT_CFUNC(atol, 0), IMPORT_CFUNC(rand, 0), IMPORT_CFUNC(srand, 0), IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2), IMPORT_SFUNC(READ_REGISTER_USHORT, 1), IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2), IMPORT_SFUNC(READ_REGISTER_ULONG, 1), IMPORT_SFUNC(READ_REGISTER_UCHAR, 1), IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2), IMPORT_SFUNC(ExInitializePagedLookasideList, 7), IMPORT_SFUNC(ExDeletePagedLookasideList, 1), IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7), IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1), IMPORT_FFUNC(InterlockedPopEntrySList, 1), IMPORT_FFUNC(InterlockedPushEntrySList, 2), IMPORT_SFUNC(ExQueryDepthSList, 1), IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList, InterlockedPopEntrySList, 1), IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList, InterlockedPushEntrySList, 2), IMPORT_FFUNC(ExInterlockedPopEntrySList, 2), IMPORT_FFUNC(ExInterlockedPushEntrySList, 3), IMPORT_SFUNC(ExAllocatePoolWithTag, 3), IMPORT_SFUNC(ExFreePool, 1), #ifdef __i386__ IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1), IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1), IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1), #else /* * For AMD64, we can get away with just mapping * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock() * because the calling conventions end up being the same. * On i386, we have to be careful because KfAcquireSpinLock() * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't. */ IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1), IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1), IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1), #endif IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1), IMPORT_FFUNC(InterlockedIncrement, 1), IMPORT_FFUNC(InterlockedDecrement, 1), IMPORT_FFUNC(InterlockedExchange, 2), IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2), IMPORT_SFUNC(IoAllocateMdl, 5), IMPORT_SFUNC(IoFreeMdl, 1), IMPORT_SFUNC(MmAllocateContiguousMemory, 2), IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5), IMPORT_SFUNC(MmFreeContiguousMemory, 1), IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3), IMPORT_SFUNC_MAP(MmGetPhysicalAddress, pmap_kextract, 1), IMPORT_SFUNC(MmSizeOfMdl, 1), IMPORT_SFUNC(MmMapLockedPages, 2), IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6), IMPORT_SFUNC(MmUnmapLockedPages, 2), IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1), IMPORT_SFUNC(MmIsAddressValid, 1), IMPORT_SFUNC(MmMapIoSpace, 3 + 1), IMPORT_SFUNC(MmUnmapIoSpace, 2), IMPORT_SFUNC(KeInitializeSpinLock, 1), IMPORT_SFUNC(IoIsWdmVersionAvailable, 2), IMPORT_SFUNC(IoGetDeviceObjectPointer, 4), IMPORT_SFUNC(IoGetDeviceProperty, 5), IMPORT_SFUNC(IoAllocateWorkItem, 1), IMPORT_SFUNC(IoFreeWorkItem, 1), IMPORT_SFUNC(IoQueueWorkItem, 4), IMPORT_SFUNC(ExQueueWorkItem, 2), IMPORT_SFUNC(ntoskrnl_workitem, 2), IMPORT_SFUNC(KeInitializeMutex, 2), IMPORT_SFUNC(KeReleaseMutex, 2), IMPORT_SFUNC(KeReadStateMutex, 1), IMPORT_SFUNC(KeInitializeEvent, 3), IMPORT_SFUNC(KeSetEvent, 3), IMPORT_SFUNC(KeResetEvent, 1), IMPORT_SFUNC(KeClearEvent, 1), IMPORT_SFUNC(KeReadStateEvent, 1), IMPORT_SFUNC(KeInitializeTimer, 1), IMPORT_SFUNC(KeInitializeTimerEx, 2), IMPORT_SFUNC(KeSetTimer, 3), IMPORT_SFUNC(KeSetTimerEx, 4), IMPORT_SFUNC(KeCancelTimer, 1), IMPORT_SFUNC(KeReadStateTimer, 1), IMPORT_SFUNC(KeInitializeDpc, 3), IMPORT_SFUNC(KeInsertQueueDpc, 3), IMPORT_SFUNC(KeRemoveQueueDpc, 1), IMPORT_SFUNC(KeSetImportanceDpc, 2), IMPORT_SFUNC(KeSetTargetProcessorDpc, 2), IMPORT_SFUNC(KeFlushQueuedDpcs, 0), IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1), IMPORT_SFUNC(ObReferenceObjectByHandle, 6), IMPORT_FFUNC(ObfDereferenceObject, 1), IMPORT_SFUNC(ZwClose, 1), IMPORT_SFUNC(PsCreateSystemThread, 7), IMPORT_SFUNC(PsTerminateSystemThread, 1), IMPORT_SFUNC(IoWMIRegistrationControl, 2), IMPORT_SFUNC(WmiQueryTraceInformation, 5), IMPORT_CFUNC(WmiTraceMessage, 0), IMPORT_SFUNC(KeQuerySystemTime, 1), IMPORT_CFUNC(KeTickCount, 0), IMPORT_SFUNC(KeDelayExecutionThread, 3), IMPORT_SFUNC(KeQueryInterruptTime, 0), IMPORT_SFUNC(KeGetCurrentThread, 0), IMPORT_SFUNC(KeSetPriorityThread, 2), /* * This last entry is a catch-all for any function we haven't * implemented yet. The PE import list patching routine will * use it for any function that doesn't have an explicit match * in this table. */ { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL }, /* End of list. */ { NULL, NULL, NULL } };