Index: head/lib/libvmmapi/vmmapi.c =================================================================== --- head/lib/libvmmapi/vmmapi.c (revision 295880) +++ head/lib/libvmmapi/vmmapi.c (revision 295881) @@ -1,1414 +1,1413 @@ /*- * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #include #include #include #include #include #include "vmmapi.h" #define MB (1024 * 1024UL) #define GB (1024 * 1024 * 1024UL) /* * Size of the guard region before and after the virtual address space * mapping the guest physical memory. This must be a multiple of the * superpage size for performance reasons. */ #define VM_MMAP_GUARD_SIZE (4 * MB) #define PROT_RW (PROT_READ | PROT_WRITE) #define PROT_ALL (PROT_READ | PROT_WRITE | PROT_EXEC) struct vmctx { int fd; uint32_t lowmem_limit; int memflags; size_t lowmem; size_t highmem; char *baseaddr; char *name; }; #define CREATE(x) sysctlbyname("hw.vmm.create", NULL, NULL, (x), strlen((x))) #define DESTROY(x) sysctlbyname("hw.vmm.destroy", NULL, NULL, (x), strlen((x))) static int vm_device_open(const char *name) { int fd, len; char *vmfile; len = strlen("/dev/vmm/") + strlen(name) + 1; vmfile = malloc(len); assert(vmfile != NULL); snprintf(vmfile, len, "/dev/vmm/%s", name); /* Open the device file */ fd = open(vmfile, O_RDWR, 0); free(vmfile); return (fd); } int vm_create(const char *name) { return (CREATE((char *)name)); } struct vmctx * vm_open(const char *name) { struct vmctx *vm; vm = malloc(sizeof(struct vmctx) + strlen(name) + 1); assert(vm != NULL); vm->fd = -1; vm->memflags = 0; vm->lowmem_limit = 3 * GB; vm->name = (char *)(vm + 1); strcpy(vm->name, name); if ((vm->fd = vm_device_open(vm->name)) < 0) goto err; return (vm); err: vm_destroy(vm); return (NULL); } void vm_destroy(struct vmctx *vm) { assert(vm != NULL); if (vm->fd >= 0) close(vm->fd); DESTROY(vm->name); free(vm); } int vm_parse_memsize(const char *optarg, size_t *ret_memsize) { char *endptr; size_t optval; int error; optval = strtoul(optarg, &endptr, 0); if (*optarg != '\0' && *endptr == '\0') { /* * For the sake of backward compatibility if the memory size * specified on the command line is less than a megabyte then * it is interpreted as being in units of MB. */ if (optval < MB) optval *= MB; *ret_memsize = optval; error = 0; } else error = expand_number(optarg, ret_memsize); return (error); } uint32_t vm_get_lowmem_limit(struct vmctx *ctx) { return (ctx->lowmem_limit); } void vm_set_lowmem_limit(struct vmctx *ctx, uint32_t limit) { ctx->lowmem_limit = limit; } void vm_set_memflags(struct vmctx *ctx, int flags) { ctx->memflags = flags; } int vm_get_memflags(struct vmctx *ctx) { return (ctx->memflags); } /* * Map segment 'segid' starting at 'off' into guest address range [gpa,gpa+len). */ int vm_mmap_memseg(struct vmctx *ctx, vm_paddr_t gpa, int segid, vm_ooffset_t off, size_t len, int prot) { struct vm_memmap memmap; int error, flags; memmap.gpa = gpa; memmap.segid = segid; memmap.segoff = off; memmap.len = len; memmap.prot = prot; memmap.flags = 0; if (ctx->memflags & VM_MEM_F_WIRED) memmap.flags |= VM_MEMMAP_F_WIRED; /* * If this mapping already exists then don't create it again. This * is the common case for SYSMEM mappings created by bhyveload(8). */ error = vm_mmap_getnext(ctx, &gpa, &segid, &off, &len, &prot, &flags); if (error == 0 && gpa == memmap.gpa) { if (segid != memmap.segid || off != memmap.segoff || prot != memmap.prot || flags != memmap.flags) { errno = EEXIST; return (-1); } else { return (0); } } error = ioctl(ctx->fd, VM_MMAP_MEMSEG, &memmap); return (error); } int vm_mmap_getnext(struct vmctx *ctx, vm_paddr_t *gpa, int *segid, vm_ooffset_t *segoff, size_t *len, int *prot, int *flags) { struct vm_memmap memmap; int error; bzero(&memmap, sizeof(struct vm_memmap)); memmap.gpa = *gpa; error = ioctl(ctx->fd, VM_MMAP_GETNEXT, &memmap); if (error == 0) { *gpa = memmap.gpa; *segid = memmap.segid; *segoff = memmap.segoff; *len = memmap.len; *prot = memmap.prot; *flags = memmap.flags; } return (error); } /* * Return 0 if the segments are identical and non-zero otherwise. * * This is slightly complicated by the fact that only device memory segments * are named. */ static int cmpseg(size_t len, const char *str, size_t len2, const char *str2) { if (len == len2) { if ((!str && !str2) || (str && str2 && !strcmp(str, str2))) return (0); } return (-1); } static int vm_alloc_memseg(struct vmctx *ctx, int segid, size_t len, const char *name) { struct vm_memseg memseg; size_t n; int error; /* * If the memory segment has already been created then just return. * This is the usual case for the SYSMEM segment created by userspace * loaders like bhyveload(8). */ error = vm_get_memseg(ctx, segid, &memseg.len, memseg.name, sizeof(memseg.name)); if (error) return (error); if (memseg.len != 0) { if (cmpseg(len, name, memseg.len, VM_MEMSEG_NAME(&memseg))) { errno = EINVAL; return (-1); } else { return (0); } } bzero(&memseg, sizeof(struct vm_memseg)); memseg.segid = segid; memseg.len = len; if (name != NULL) { n = strlcpy(memseg.name, name, sizeof(memseg.name)); if (n >= sizeof(memseg.name)) { errno = ENAMETOOLONG; return (-1); } } error = ioctl(ctx->fd, VM_ALLOC_MEMSEG, &memseg); return (error); } int vm_get_memseg(struct vmctx *ctx, int segid, size_t *lenp, char *namebuf, size_t bufsize) { struct vm_memseg memseg; size_t n; int error; memseg.segid = segid; error = ioctl(ctx->fd, VM_GET_MEMSEG, &memseg); if (error == 0) { *lenp = memseg.len; n = strlcpy(namebuf, memseg.name, bufsize); if (n >= bufsize) { errno = ENAMETOOLONG; error = -1; } } return (error); } static int setup_memory_segment(struct vmctx *ctx, vm_paddr_t gpa, size_t len, char *base) { char *ptr; int error, flags; /* Map 'len' bytes starting at 'gpa' in the guest address space */ error = vm_mmap_memseg(ctx, gpa, VM_SYSMEM, gpa, len, PROT_ALL); if (error) return (error); flags = MAP_SHARED | MAP_FIXED; if ((ctx->memflags & VM_MEM_F_INCORE) == 0) flags |= MAP_NOCORE; /* mmap into the process address space on the host */ ptr = mmap(base + gpa, len, PROT_RW, flags, ctx->fd, gpa); if (ptr == MAP_FAILED) return (-1); return (0); } int vm_setup_memory(struct vmctx *ctx, size_t memsize, enum vm_mmap_style vms) { size_t objsize, len; vm_paddr_t gpa; char *baseaddr, *ptr; int error, flags; assert(vms == VM_MMAP_ALL); /* * If 'memsize' cannot fit entirely in the 'lowmem' segment then * create another 'highmem' segment above 4GB for the remainder. */ if (memsize > ctx->lowmem_limit) { ctx->lowmem = ctx->lowmem_limit; ctx->highmem = memsize - ctx->lowmem_limit; objsize = 4*GB + ctx->highmem; } else { ctx->lowmem = memsize; ctx->highmem = 0; objsize = ctx->lowmem; } error = vm_alloc_memseg(ctx, VM_SYSMEM, objsize, NULL); if (error) return (error); /* * Stake out a contiguous region covering the guest physical memory * and the adjoining guard regions. */ len = VM_MMAP_GUARD_SIZE + objsize + VM_MMAP_GUARD_SIZE; flags = MAP_PRIVATE | MAP_ANON | MAP_NOCORE | MAP_ALIGNED_SUPER; ptr = mmap(NULL, len, PROT_NONE, flags, -1, 0); if (ptr == MAP_FAILED) return (-1); baseaddr = ptr + VM_MMAP_GUARD_SIZE; if (ctx->highmem > 0) { gpa = 4*GB; len = ctx->highmem; error = setup_memory_segment(ctx, gpa, len, baseaddr); if (error) return (error); } if (ctx->lowmem > 0) { gpa = 0; len = ctx->lowmem; error = setup_memory_segment(ctx, gpa, len, baseaddr); if (error) return (error); } ctx->baseaddr = baseaddr; return (0); } /* * Returns a non-NULL pointer if [gaddr, gaddr+len) is entirely contained in * the lowmem or highmem regions. * * In particular return NULL if [gaddr, gaddr+len) falls in guest MMIO region. * The instruction emulation code depends on this behavior. */ void * vm_map_gpa(struct vmctx *ctx, vm_paddr_t gaddr, size_t len) { if (ctx->lowmem > 0) { if (gaddr < ctx->lowmem && gaddr + len <= ctx->lowmem) return (ctx->baseaddr + gaddr); } if (ctx->highmem > 0) { if (gaddr >= 4*GB && gaddr + len <= 4*GB + ctx->highmem) return (ctx->baseaddr + gaddr); } return (NULL); } size_t vm_get_lowmem_size(struct vmctx *ctx) { return (ctx->lowmem); } size_t vm_get_highmem_size(struct vmctx *ctx) { return (ctx->highmem); } void * vm_create_devmem(struct vmctx *ctx, int segid, const char *name, size_t len) { char pathname[MAXPATHLEN]; size_t len2; char *base, *ptr; int fd, error, flags; fd = -1; ptr = MAP_FAILED; if (name == NULL || strlen(name) == 0) { errno = EINVAL; goto done; } error = vm_alloc_memseg(ctx, segid, len, name); if (error) goto done; strlcpy(pathname, "/dev/vmm.io/", sizeof(pathname)); strlcat(pathname, ctx->name, sizeof(pathname)); strlcat(pathname, ".", sizeof(pathname)); strlcat(pathname, name, sizeof(pathname)); fd = open(pathname, O_RDWR); if (fd < 0) goto done; /* * Stake out a contiguous region covering the device memory and the * adjoining guard regions. */ len2 = VM_MMAP_GUARD_SIZE + len + VM_MMAP_GUARD_SIZE; flags = MAP_PRIVATE | MAP_ANON | MAP_NOCORE | MAP_ALIGNED_SUPER; base = mmap(NULL, len2, PROT_NONE, flags, -1, 0); if (base == MAP_FAILED) goto done; flags = MAP_SHARED | MAP_FIXED; if ((ctx->memflags & VM_MEM_F_INCORE) == 0) flags |= MAP_NOCORE; /* mmap the devmem region in the host address space */ ptr = mmap(base + VM_MMAP_GUARD_SIZE, len, PROT_RW, flags, fd, 0); done: if (fd >= 0) close(fd); return (ptr); } int vm_set_desc(struct vmctx *ctx, int vcpu, int reg, uint64_t base, uint32_t limit, uint32_t access) { int error; struct vm_seg_desc vmsegdesc; bzero(&vmsegdesc, sizeof(vmsegdesc)); vmsegdesc.cpuid = vcpu; vmsegdesc.regnum = reg; vmsegdesc.desc.base = base; vmsegdesc.desc.limit = limit; vmsegdesc.desc.access = access; error = ioctl(ctx->fd, VM_SET_SEGMENT_DESCRIPTOR, &vmsegdesc); return (error); } int vm_get_desc(struct vmctx *ctx, int vcpu, int reg, uint64_t *base, uint32_t *limit, uint32_t *access) { int error; struct vm_seg_desc vmsegdesc; bzero(&vmsegdesc, sizeof(vmsegdesc)); vmsegdesc.cpuid = vcpu; vmsegdesc.regnum = reg; error = ioctl(ctx->fd, VM_GET_SEGMENT_DESCRIPTOR, &vmsegdesc); if (error == 0) { *base = vmsegdesc.desc.base; *limit = vmsegdesc.desc.limit; *access = vmsegdesc.desc.access; } return (error); } int vm_get_seg_desc(struct vmctx *ctx, int vcpu, int reg, struct seg_desc *seg_desc) { int error; error = vm_get_desc(ctx, vcpu, reg, &seg_desc->base, &seg_desc->limit, &seg_desc->access); return (error); } int vm_set_register(struct vmctx *ctx, int vcpu, int reg, uint64_t val) { int error; struct vm_register vmreg; bzero(&vmreg, sizeof(vmreg)); vmreg.cpuid = vcpu; vmreg.regnum = reg; vmreg.regval = val; error = ioctl(ctx->fd, VM_SET_REGISTER, &vmreg); return (error); } int vm_get_register(struct vmctx *ctx, int vcpu, int reg, uint64_t *ret_val) { int error; struct vm_register vmreg; bzero(&vmreg, sizeof(vmreg)); vmreg.cpuid = vcpu; vmreg.regnum = reg; error = ioctl(ctx->fd, VM_GET_REGISTER, &vmreg); *ret_val = vmreg.regval; return (error); } int vm_run(struct vmctx *ctx, int vcpu, struct vm_exit *vmexit) { int error; struct vm_run vmrun; bzero(&vmrun, sizeof(vmrun)); vmrun.cpuid = vcpu; error = ioctl(ctx->fd, VM_RUN, &vmrun); bcopy(&vmrun.vm_exit, vmexit, sizeof(struct vm_exit)); return (error); } int vm_suspend(struct vmctx *ctx, enum vm_suspend_how how) { struct vm_suspend vmsuspend; bzero(&vmsuspend, sizeof(vmsuspend)); vmsuspend.how = how; return (ioctl(ctx->fd, VM_SUSPEND, &vmsuspend)); } int vm_reinit(struct vmctx *ctx) { return (ioctl(ctx->fd, VM_REINIT, 0)); } int vm_inject_exception(struct vmctx *ctx, int vcpu, int vector, int errcode_valid, uint32_t errcode, int restart_instruction) { struct vm_exception exc; exc.cpuid = vcpu; exc.vector = vector; exc.error_code = errcode; exc.error_code_valid = errcode_valid; exc.restart_instruction = restart_instruction; return (ioctl(ctx->fd, VM_INJECT_EXCEPTION, &exc)); } int vm_apicid2vcpu(struct vmctx *ctx, int apicid) { /* * The apic id associated with the 'vcpu' has the same numerical value * as the 'vcpu' itself. */ return (apicid); } int vm_lapic_irq(struct vmctx *ctx, int vcpu, int vector) { struct vm_lapic_irq vmirq; bzero(&vmirq, sizeof(vmirq)); vmirq.cpuid = vcpu; vmirq.vector = vector; return (ioctl(ctx->fd, VM_LAPIC_IRQ, &vmirq)); } int vm_lapic_local_irq(struct vmctx *ctx, int vcpu, int vector) { struct vm_lapic_irq vmirq; bzero(&vmirq, sizeof(vmirq)); vmirq.cpuid = vcpu; vmirq.vector = vector; return (ioctl(ctx->fd, VM_LAPIC_LOCAL_IRQ, &vmirq)); } int vm_lapic_msi(struct vmctx *ctx, uint64_t addr, uint64_t msg) { struct vm_lapic_msi vmmsi; bzero(&vmmsi, sizeof(vmmsi)); vmmsi.addr = addr; vmmsi.msg = msg; return (ioctl(ctx->fd, VM_LAPIC_MSI, &vmmsi)); } int vm_ioapic_assert_irq(struct vmctx *ctx, int irq) { struct vm_ioapic_irq ioapic_irq; bzero(&ioapic_irq, sizeof(struct vm_ioapic_irq)); ioapic_irq.irq = irq; return (ioctl(ctx->fd, VM_IOAPIC_ASSERT_IRQ, &ioapic_irq)); } int vm_ioapic_deassert_irq(struct vmctx *ctx, int irq) { struct vm_ioapic_irq ioapic_irq; bzero(&ioapic_irq, sizeof(struct vm_ioapic_irq)); ioapic_irq.irq = irq; return (ioctl(ctx->fd, VM_IOAPIC_DEASSERT_IRQ, &ioapic_irq)); } int vm_ioapic_pulse_irq(struct vmctx *ctx, int irq) { struct vm_ioapic_irq ioapic_irq; bzero(&ioapic_irq, sizeof(struct vm_ioapic_irq)); ioapic_irq.irq = irq; return (ioctl(ctx->fd, VM_IOAPIC_PULSE_IRQ, &ioapic_irq)); } int vm_ioapic_pincount(struct vmctx *ctx, int *pincount) { return (ioctl(ctx->fd, VM_IOAPIC_PINCOUNT, pincount)); } int vm_isa_assert_irq(struct vmctx *ctx, int atpic_irq, int ioapic_irq) { struct vm_isa_irq isa_irq; bzero(&isa_irq, sizeof(struct vm_isa_irq)); isa_irq.atpic_irq = atpic_irq; isa_irq.ioapic_irq = ioapic_irq; return (ioctl(ctx->fd, VM_ISA_ASSERT_IRQ, &isa_irq)); } int vm_isa_deassert_irq(struct vmctx *ctx, int atpic_irq, int ioapic_irq) { struct vm_isa_irq isa_irq; bzero(&isa_irq, sizeof(struct vm_isa_irq)); isa_irq.atpic_irq = atpic_irq; isa_irq.ioapic_irq = ioapic_irq; return (ioctl(ctx->fd, VM_ISA_DEASSERT_IRQ, &isa_irq)); } int vm_isa_pulse_irq(struct vmctx *ctx, int atpic_irq, int ioapic_irq) { struct vm_isa_irq isa_irq; bzero(&isa_irq, sizeof(struct vm_isa_irq)); isa_irq.atpic_irq = atpic_irq; isa_irq.ioapic_irq = ioapic_irq; return (ioctl(ctx->fd, VM_ISA_PULSE_IRQ, &isa_irq)); } int vm_isa_set_irq_trigger(struct vmctx *ctx, int atpic_irq, enum vm_intr_trigger trigger) { struct vm_isa_irq_trigger isa_irq_trigger; bzero(&isa_irq_trigger, sizeof(struct vm_isa_irq_trigger)); isa_irq_trigger.atpic_irq = atpic_irq; isa_irq_trigger.trigger = trigger; return (ioctl(ctx->fd, VM_ISA_SET_IRQ_TRIGGER, &isa_irq_trigger)); } int vm_inject_nmi(struct vmctx *ctx, int vcpu) { struct vm_nmi vmnmi; bzero(&vmnmi, sizeof(vmnmi)); vmnmi.cpuid = vcpu; return (ioctl(ctx->fd, VM_INJECT_NMI, &vmnmi)); } static struct { const char *name; int type; } capstrmap[] = { { "hlt_exit", VM_CAP_HALT_EXIT }, { "mtrap_exit", VM_CAP_MTRAP_EXIT }, { "pause_exit", VM_CAP_PAUSE_EXIT }, { "unrestricted_guest", VM_CAP_UNRESTRICTED_GUEST }, { "enable_invpcid", VM_CAP_ENABLE_INVPCID }, { 0 } }; int vm_capability_name2type(const char *capname) { int i; for (i = 0; capstrmap[i].name != NULL && capname != NULL; i++) { if (strcmp(capstrmap[i].name, capname) == 0) return (capstrmap[i].type); } return (-1); } const char * vm_capability_type2name(int type) { int i; for (i = 0; capstrmap[i].name != NULL; i++) { if (capstrmap[i].type == type) return (capstrmap[i].name); } return (NULL); } int vm_get_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap, int *retval) { int error; struct vm_capability vmcap; bzero(&vmcap, sizeof(vmcap)); vmcap.cpuid = vcpu; vmcap.captype = cap; error = ioctl(ctx->fd, VM_GET_CAPABILITY, &vmcap); *retval = vmcap.capval; return (error); } int vm_set_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap, int val) { struct vm_capability vmcap; bzero(&vmcap, sizeof(vmcap)); vmcap.cpuid = vcpu; vmcap.captype = cap; vmcap.capval = val; return (ioctl(ctx->fd, VM_SET_CAPABILITY, &vmcap)); } int vm_assign_pptdev(struct vmctx *ctx, int bus, int slot, int func) { struct vm_pptdev pptdev; bzero(&pptdev, sizeof(pptdev)); pptdev.bus = bus; pptdev.slot = slot; pptdev.func = func; return (ioctl(ctx->fd, VM_BIND_PPTDEV, &pptdev)); } int vm_unassign_pptdev(struct vmctx *ctx, int bus, int slot, int func) { struct vm_pptdev pptdev; bzero(&pptdev, sizeof(pptdev)); pptdev.bus = bus; pptdev.slot = slot; pptdev.func = func; return (ioctl(ctx->fd, VM_UNBIND_PPTDEV, &pptdev)); } int vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) { struct vm_pptdev_mmio pptmmio; bzero(&pptmmio, sizeof(pptmmio)); pptmmio.bus = bus; pptmmio.slot = slot; pptmmio.func = func; pptmmio.gpa = gpa; pptmmio.len = len; pptmmio.hpa = hpa; return (ioctl(ctx->fd, VM_MAP_PPTDEV_MMIO, &pptmmio)); } int vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int bus, int slot, int func, uint64_t addr, uint64_t msg, int numvec) { struct vm_pptdev_msi pptmsi; bzero(&pptmsi, sizeof(pptmsi)); pptmsi.vcpu = vcpu; pptmsi.bus = bus; pptmsi.slot = slot; pptmsi.func = func; pptmsi.msg = msg; pptmsi.addr = addr; pptmsi.numvec = numvec; return (ioctl(ctx->fd, VM_PPTDEV_MSI, &pptmsi)); } int vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int bus, int slot, int func, int idx, uint64_t addr, uint64_t msg, uint32_t vector_control) { struct vm_pptdev_msix pptmsix; bzero(&pptmsix, sizeof(pptmsix)); pptmsix.vcpu = vcpu; pptmsix.bus = bus; pptmsix.slot = slot; pptmsix.func = func; pptmsix.idx = idx; pptmsix.msg = msg; pptmsix.addr = addr; pptmsix.vector_control = vector_control; return ioctl(ctx->fd, VM_PPTDEV_MSIX, &pptmsix); } uint64_t * vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv, int *ret_entries) { int error; static struct vm_stats vmstats; vmstats.cpuid = vcpu; error = ioctl(ctx->fd, VM_STATS, &vmstats); if (error == 0) { if (ret_entries) *ret_entries = vmstats.num_entries; if (ret_tv) *ret_tv = vmstats.tv; return (vmstats.statbuf); } else return (NULL); } const char * vm_get_stat_desc(struct vmctx *ctx, int index) { static struct vm_stat_desc statdesc; statdesc.index = index; if (ioctl(ctx->fd, VM_STAT_DESC, &statdesc) == 0) return (statdesc.desc); else return (NULL); } int vm_get_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state *state) { int error; struct vm_x2apic x2apic; bzero(&x2apic, sizeof(x2apic)); x2apic.cpuid = vcpu; error = ioctl(ctx->fd, VM_GET_X2APIC_STATE, &x2apic); *state = x2apic.state; return (error); } int vm_set_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state state) { int error; struct vm_x2apic x2apic; bzero(&x2apic, sizeof(x2apic)); x2apic.cpuid = vcpu; x2apic.state = state; error = ioctl(ctx->fd, VM_SET_X2APIC_STATE, &x2apic); return (error); } /* * From Intel Vol 3a: * Table 9-1. IA-32 Processor States Following Power-up, Reset or INIT */ int vcpu_reset(struct vmctx *vmctx, int vcpu) { int error; uint64_t rflags, rip, cr0, cr4, zero, desc_base, rdx; uint32_t desc_access, desc_limit; uint16_t sel; zero = 0; rflags = 0x2; error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RFLAGS, rflags); if (error) goto done; rip = 0xfff0; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RIP, rip)) != 0) goto done; cr0 = CR0_NE; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR0, cr0)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR3, zero)) != 0) goto done; cr4 = 0; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR4, cr4)) != 0) goto done; /* * CS: present, r/w, accessed, 16-bit, byte granularity, usable */ desc_base = 0xffff0000; desc_limit = 0xffff; desc_access = 0x0093; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_CS, desc_base, desc_limit, desc_access); if (error) goto done; sel = 0xf000; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CS, sel)) != 0) goto done; /* * SS,DS,ES,FS,GS: present, r/w, accessed, 16-bit, byte granularity */ desc_base = 0; desc_limit = 0xffff; desc_access = 0x0093; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_SS, desc_base, desc_limit, desc_access); if (error) goto done; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_DS, desc_base, desc_limit, desc_access); if (error) goto done; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_ES, desc_base, desc_limit, desc_access); if (error) goto done; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_FS, desc_base, desc_limit, desc_access); if (error) goto done; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GS, desc_base, desc_limit, desc_access); if (error) goto done; sel = 0; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_SS, sel)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_DS, sel)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_ES, sel)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_FS, sel)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_GS, sel)) != 0) goto done; /* General purpose registers */ rdx = 0xf00; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RAX, zero)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RBX, zero)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RCX, zero)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RDX, rdx)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSI, zero)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RDI, zero)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RBP, zero)) != 0) goto done; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSP, zero)) != 0) goto done; /* GDTR, IDTR */ desc_base = 0; desc_limit = 0xffff; desc_access = 0; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GDTR, desc_base, desc_limit, desc_access); if (error != 0) goto done; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_IDTR, desc_base, desc_limit, desc_access); if (error != 0) goto done; /* TR */ desc_base = 0; desc_limit = 0xffff; desc_access = 0x0000008b; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_TR, 0, 0, desc_access); if (error) goto done; sel = 0; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_TR, sel)) != 0) goto done; /* LDTR */ desc_base = 0; desc_limit = 0xffff; desc_access = 0x00000082; error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_LDTR, desc_base, desc_limit, desc_access); if (error) goto done; sel = 0; if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_LDTR, 0)) != 0) goto done; /* XXX cr2, debug registers */ error = 0; done: return (error); } int vm_get_gpa_pmap(struct vmctx *ctx, uint64_t gpa, uint64_t *pte, int *num) { int error, i; struct vm_gpa_pte gpapte; bzero(&gpapte, sizeof(gpapte)); gpapte.gpa = gpa; error = ioctl(ctx->fd, VM_GET_GPA_PMAP, &gpapte); if (error == 0) { *num = gpapte.ptenum; for (i = 0; i < gpapte.ptenum; i++) pte[i] = gpapte.pte[i]; } return (error); } int vm_get_hpet_capabilities(struct vmctx *ctx, uint32_t *capabilities) { int error; struct vm_hpet_cap cap; bzero(&cap, sizeof(struct vm_hpet_cap)); error = ioctl(ctx->fd, VM_GET_HPET_CAPABILITIES, &cap); if (capabilities != NULL) *capabilities = cap.capabilities; return (error); } int vm_gla2gpa(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging, uint64_t gla, int prot, uint64_t *gpa, int *fault) { struct vm_gla2gpa gg; int error; bzero(&gg, sizeof(struct vm_gla2gpa)); gg.vcpuid = vcpu; gg.prot = prot; gg.gla = gla; gg.paging = *paging; error = ioctl(ctx->fd, VM_GLA2GPA, &gg); if (error == 0) { *fault = gg.fault; *gpa = gg.gpa; } return (error); } #ifndef min #define min(a,b) (((a) < (b)) ? (a) : (b)) #endif int vm_copy_setup(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging, uint64_t gla, size_t len, int prot, struct iovec *iov, int iovcnt, int *fault) { void *va; uint64_t gpa; int error, i, n, off; for (i = 0; i < iovcnt; i++) { iov[i].iov_base = 0; iov[i].iov_len = 0; } while (len) { assert(iovcnt > 0); error = vm_gla2gpa(ctx, vcpu, paging, gla, prot, &gpa, fault); if (error || *fault) return (error); off = gpa & PAGE_MASK; n = min(len, PAGE_SIZE - off); va = vm_map_gpa(ctx, gpa, n); if (va == NULL) return (EFAULT); iov->iov_base = va; iov->iov_len = n; iov++; iovcnt--; gla += n; len -= n; } return (0); } void vm_copy_teardown(struct vmctx *ctx, int vcpu, struct iovec *iov, int iovcnt) { return; } void vm_copyin(struct vmctx *ctx, int vcpu, struct iovec *iov, void *vp, size_t len) { const char *src; char *dst; size_t n; dst = vp; while (len) { assert(iov->iov_len); n = min(len, iov->iov_len); src = iov->iov_base; bcopy(src, dst, n); iov++; dst += n; len -= n; } } void vm_copyout(struct vmctx *ctx, int vcpu, const void *vp, struct iovec *iov, size_t len) { const char *src; char *dst; size_t n; src = vp; while (len) { assert(iov->iov_len); n = min(len, iov->iov_len); dst = iov->iov_base; bcopy(src, dst, n); iov++; src += n; len -= n; } } static int vm_get_cpus(struct vmctx *ctx, int which, cpuset_t *cpus) { struct vm_cpuset vm_cpuset; int error; bzero(&vm_cpuset, sizeof(struct vm_cpuset)); vm_cpuset.which = which; vm_cpuset.cpusetsize = sizeof(cpuset_t); vm_cpuset.cpus = cpus; error = ioctl(ctx->fd, VM_GET_CPUS, &vm_cpuset); return (error); } int vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus) { return (vm_get_cpus(ctx, VM_ACTIVE_CPUS, cpus)); } int vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus) { return (vm_get_cpus(ctx, VM_SUSPENDED_CPUS, cpus)); } int vm_activate_cpu(struct vmctx *ctx, int vcpu) { struct vm_activate_cpu ac; int error; bzero(&ac, sizeof(struct vm_activate_cpu)); ac.vcpuid = vcpu; error = ioctl(ctx->fd, VM_ACTIVATE_CPU, &ac); return (error); } int vm_get_intinfo(struct vmctx *ctx, int vcpu, uint64_t *info1, uint64_t *info2) { struct vm_intinfo vmii; int error; bzero(&vmii, sizeof(struct vm_intinfo)); vmii.vcpuid = vcpu; error = ioctl(ctx->fd, VM_GET_INTINFO, &vmii); if (error == 0) { *info1 = vmii.info1; *info2 = vmii.info2; } return (error); } int vm_set_intinfo(struct vmctx *ctx, int vcpu, uint64_t info1) { struct vm_intinfo vmii; int error; bzero(&vmii, sizeof(struct vm_intinfo)); vmii.vcpuid = vcpu; vmii.info1 = info1; error = ioctl(ctx->fd, VM_SET_INTINFO, &vmii); return (error); } int vm_rtc_write(struct vmctx *ctx, int offset, uint8_t value) { struct vm_rtc_data rtcdata; int error; bzero(&rtcdata, sizeof(struct vm_rtc_data)); rtcdata.offset = offset; rtcdata.value = value; error = ioctl(ctx->fd, VM_RTC_WRITE, &rtcdata); return (error); } int vm_rtc_read(struct vmctx *ctx, int offset, uint8_t *retval) { struct vm_rtc_data rtcdata; int error; bzero(&rtcdata, sizeof(struct vm_rtc_data)); rtcdata.offset = offset; error = ioctl(ctx->fd, VM_RTC_READ, &rtcdata); if (error == 0) *retval = rtcdata.value; return (error); } int vm_rtc_settime(struct vmctx *ctx, time_t secs) { struct vm_rtc_time rtctime; int error; bzero(&rtctime, sizeof(struct vm_rtc_time)); rtctime.secs = secs; error = ioctl(ctx->fd, VM_RTC_SETTIME, &rtctime); return (error); } int vm_rtc_gettime(struct vmctx *ctx, time_t *secs) { struct vm_rtc_time rtctime; int error; bzero(&rtctime, sizeof(struct vm_rtc_time)); error = ioctl(ctx->fd, VM_RTC_GETTIME, &rtctime); if (error == 0) *secs = rtctime.secs; return (error); } int vm_restart_instruction(void *arg, int vcpu) { struct vmctx *ctx = arg; return (ioctl(ctx->fd, VM_RESTART_INSTRUCTION, &vcpu)); } Index: head/share/man/man9/bios.9 =================================================================== --- head/share/man/man9/bios.9 (revision 295880) +++ head/share/man/man9/bios.9 (revision 295881) @@ -1,181 +1,180 @@ .\" $FreeBSD$ .\" .\" Copyright (c) 1997 Michael Smith .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR .\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES .\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. .\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, .\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, .\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; .\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED .\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, .\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .Dd August 9, 2005 .Dt BIOS 9 .Os .Sh NAME .Nm bios_sigsearch , .Nm bios32_SDlookup , .Nm bios32 , .Nm bios_oem_strings .Nd interact with PC BIOS .Sh SYNOPSIS .In sys/param.h .In vm/vm.h .In vm/pmap.h -.In machine/param.h .In machine/pc/bios.h .Ft uint32_t .Fn bios_sigsearch "uint32_t start" "u_char *sig" "int siglen" "int paralen" "int sigofs" .Ft int .Fn bios32_SDlookup "struct bios32_SDentry *ent" .Ft int .Fn bios32 "struct bios_regs *br" "u_int offset" "u_short segment" .Fn BIOS_PADDRTOVADDR "addr" .Fn BIOS_VADDRTOPADDR "addr" .Vt extern struct bios32_SDentry PCIbios ; .Vt extern struct SMBIOS_table SMBIOStable ; .Vt extern struct DMI_table DMItable ; .Ft int .Fn bios_oem_strings "struct bios_oem *oem" "u_char *buffer" "size_t maxlen" .Bd -literal struct bios_oem_signature { char * anchor; /* search anchor string in BIOS memory */ size_t offset; /* offset from anchor (may be negative) */ size_t totlen; /* total length of BIOS string to copy */ }; struct bios_oem_range { u_int from; /* shouldn't be below 0xe0000 */ u_int to; /* shouldn't be above 0xfffff */ }; struct bios_oem { struct bios_oem_range range; struct bios_oem_signature signature[]; }; .Ed .Sh DESCRIPTION These functions provide a general-purpose interface for dealing with the BIOS functions and data encountered on x86 PC-architecture systems. .Bl -tag -width 20n .It Fn bios_sigsearch Searches the BIOS address space for a service signature, usually an uppercase ASCII sequence surrounded by underscores. The search begins at .Fa start , or at the beginning of the BIOS if .Fa start is zero. .Fa siglen bytes of the BIOS image and .Fa sig are compared at .Fa sigofs bytes offset from the current location. If no match is found, the current location is incremented by .Fa paralen bytes and the search repeated. If the signature is found, its effective physical address is returned. If no signature is found, zero is returned. .It Fn bios_oem_strings Searches a given BIOS memory range for one or more strings, and composes a printable concatenation of those found. The routine expects a structure describing the BIOS address .Fa range (within .Li 0xe0000 - .Li 0xfffff ) , and a { .Dv NULL , Li 0 , 0 } -terminated array of .Vt bios_oem_signature structures which define the .Va anchor string, an .Va offset from the beginning of the match (which may be negative), and .Va totlen number of bytes to be collected from BIOS memory starting at that offset. Unmatched anchors are ignored, whereas matches are copied from BIOS memory starting at their corresponding .Vt offset with unprintable characters being replaced with space, and consecutive spaces being suppressed. This composed string is stored in .Fa buffer up to the given .Fa maxlen bytes (including trailing .Ql \e0 , and any trailing space suppressed). If an error is encountered, i.e.\& trying to read out of said BIOS range, other invalid input, or .Fa buffer overflow, a negative integer is returned, otherwise the length of the composed string is returned. In particular, a return value of 0 means that none of the given anchor strings were found in the specified BIOS memory range. .It Fn BIOS_VADDRTOPADDR Returns the effective physical address which corresponds to the kernel virtual address .Fa addr . .It Fn BIOS_PADDRTOVADDR Returns the kernel virtual address which corresponds to the effective physical address .Fa addr . .It SMBIOStable If not NULL, points to a .Ft struct SMBIOS_table structure containing information read from the System Management BIOS table during system startup. .It DMItable If not NULL, points to a .Ft struct DMI_table structure containing information read from the Desktop Management Interface parameter table during system startup. .El .Sh BIOS32 At system startup, the BIOS is scanned for the BIOS32 Service Directory (part of the PCI specification), and the existence of the directory is recorded. This can then be used to locate other services. .Bl -tag -width 20n .It Fn bios32_SDlookup Attempts to locate the BIOS32 service matching the 4-byte identifier passed in the .Fa ident field of the .Fa ent argument. .It Fn bios32 Calls a bios32 function. This presumes that the function is capable of working within the kernel segment (normally the case). The virtual address of the entrypoint is supplied in .Fa entry and the register arguments to the function are supplied in .Fa args . .It PCIbios If not NULL, points to a .Ft struct bios32_SDentry structure describing the PCI BIOS entrypoint which was found during system startup. .El Index: head/sys/arm/arm/debug_monitor.c =================================================================== --- head/sys/arm/arm/debug_monitor.c (revision 295880) +++ head/sys/arm/arm/debug_monitor.c (revision 295881) @@ -1,1072 +1,1071 @@ /* * Copyright (c) 2015 Juniper Networks Inc. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include enum dbg_t { DBG_TYPE_BREAKPOINT = 0, DBG_TYPE_WATCHPOINT = 1, }; struct dbg_wb_conf { enum dbg_t type; enum dbg_access_t access; db_addr_t address; db_expr_t size; u_int slot; }; static int dbg_reset_state(void); static int dbg_setup_breakpoint(db_expr_t, db_expr_t, u_int); static int dbg_remove_breakpoint(u_int); static u_int dbg_find_slot(enum dbg_t, db_expr_t); static boolean_t dbg_check_slot_free(enum dbg_t, u_int); static int dbg_remove_xpoint(struct dbg_wb_conf *); static int dbg_setup_xpoint(struct dbg_wb_conf *); static boolean_t dbg_capable; /* Indicates that machine is capable of using HW watchpoints/breakpoints */ static boolean_t dbg_ready[MAXCPU]; /* Debug arch. reset performed on this CPU */ static uint32_t dbg_model; /* Debug Arch. Model */ static boolean_t dbg_ossr; /* OS Save and Restore implemented */ static uint32_t dbg_watchpoint_num; static uint32_t dbg_breakpoint_num; static int dbg_ref_count_mme; /* Times monitor mode was enabled */ /* ID_DFR0 - Debug Feature Register 0 */ #define ID_DFR0_CP_DEBUG_M_SHIFT 0 #define ID_DFR0_CP_DEBUG_M_MASK (0xF << ID_DFR0_CP_DEBUG_M_SHIFT) #define ID_DFR0_CP_DEBUG_M_NS (0x0) /* Not supported */ #define ID_DFR0_CP_DEBUG_M_V6 (0x2) /* v6 Debug arch. CP14 access */ #define ID_DFR0_CP_DEBUG_M_V6_1 (0x3) /* v6.1 Debug arch. CP14 access */ #define ID_DFR0_CP_DEBUG_M_V7 (0x4) /* v7 Debug arch. CP14 access */ #define ID_DFR0_CP_DEBUG_M_V7_1 (0x5) /* v7.1 Debug arch. CP14 access */ /* DBGDIDR - Debug ID Register */ #define DBGDIDR_WRPS_SHIFT 28 #define DBGDIDR_WRPS_MASK (0xF << DBGDIDR_WRPS_SHIFT) #define DBGDIDR_WRPS_NUM(reg) \ ((((reg) & DBGDIDR_WRPS_MASK) >> DBGDIDR_WRPS_SHIFT) + 1) #define DBGDIDR_BRPS_SHIFT 24 #define DBGDIDR_BRPS_MASK (0xF << DBGDIDR_BRPS_SHIFT) #define DBGDIDR_BRPS_NUM(reg) \ ((((reg) & DBGDIDR_BRPS_MASK) >> DBGDIDR_BRPS_SHIFT) + 1) /* DBGPRSR - Device Powerdown and Reset Status Register */ #define DBGPRSR_PU (1 << 0) /* Powerup status */ /* DBGOSLSR - OS Lock Status Register */ #define DBGOSLSR_OSLM0 (1 << 0) /* DBGOSDLR - OS Double Lock Register */ #define DBGPRSR_DLK (1 << 0) /* OS Double Lock set */ /* DBGDSCR - Debug Status and Control Register */ #define DBGSCR_MDBG_EN (1 << 15) /* Monitor debug-mode enable */ /* DBGWVR - Watchpoint Value Register */ #define DBGWVR_ADDR_MASK (~0x3U) /* Watchpoints/breakpoints control register bitfields */ #define DBG_WB_CTRL_LEN_1 (0x1 << 5) #define DBG_WB_CTRL_LEN_2 (0x3 << 5) #define DBG_WB_CTRL_LEN_4 (0xf << 5) #define DBG_WB_CTRL_LEN_8 (0xff << 5) #define DBG_WB_CTRL_LEN_MASK(x) ((x) & (0xff << 5)) #define DBG_WB_CTRL_EXEC (0x0 << 3) #define DBG_WB_CTRL_LOAD (0x1 << 3) #define DBG_WB_CTRL_STORE (0x2 << 3) #define DBG_WB_CTRL_ACCESS_MASK(x) ((x) & (0x3 << 3)) /* Common for breakpoint and watchpoint */ #define DBG_WB_CTRL_PL1 (0x1 << 1) #define DBG_WB_CTRL_PL0 (0x2 << 1) #define DBG_WB_CTRL_PLX_MASK(x) ((x) & (0x3 << 1)) #define DBG_WB_CTRL_E (0x1 << 0) /* * Watchpoint/breakpoint helpers */ #define DBG_BKPT_BT_SLOT 0 /* Slot for branch taken */ #define DBG_BKPT_BNT_SLOT 1 /* Slot for branch not taken */ #define OP2_SHIFT 4 /* Opc2 numbers for coprocessor instructions */ #define DBG_WB_BVR 4 #define DBG_WB_BCR 5 #define DBG_WB_WVR 6 #define DBG_WB_WCR 7 #define DBG_REG_BASE_BVR (DBG_WB_BVR << OP2_SHIFT) #define DBG_REG_BASE_BCR (DBG_WB_BCR << OP2_SHIFT) #define DBG_REG_BASE_WVR (DBG_WB_WVR << OP2_SHIFT) #define DBG_REG_BASE_WCR (DBG_WB_WCR << OP2_SHIFT) #define DBG_WB_READ(cn, cm, op2, val) do { \ __asm __volatile("mrc p14, 0, %0, " #cn "," #cm "," #op2 : "=r" (val)); \ } while (0) #define DBG_WB_WRITE(cn, cm, op2, val) do { \ __asm __volatile("mcr p14, 0, %0, " #cn "," #cm "," #op2 :: "r" (val)); \ } while (0) #define READ_WB_REG_CASE(op2, m, val) \ case (((op2) << OP2_SHIFT) + m): \ DBG_WB_READ(c0, c ## m, op2, val); \ break #define WRITE_WB_REG_CASE(op2, m, val) \ case (((op2) << OP2_SHIFT) + m): \ DBG_WB_WRITE(c0, c ## m, op2, val); \ break #define SWITCH_CASES_READ_WB_REG(op2, val) \ READ_WB_REG_CASE(op2, 0, val); \ READ_WB_REG_CASE(op2, 1, val); \ READ_WB_REG_CASE(op2, 2, val); \ READ_WB_REG_CASE(op2, 3, val); \ READ_WB_REG_CASE(op2, 4, val); \ READ_WB_REG_CASE(op2, 5, val); \ READ_WB_REG_CASE(op2, 6, val); \ READ_WB_REG_CASE(op2, 7, val); \ READ_WB_REG_CASE(op2, 8, val); \ READ_WB_REG_CASE(op2, 9, val); \ READ_WB_REG_CASE(op2, 10, val); \ READ_WB_REG_CASE(op2, 11, val); \ READ_WB_REG_CASE(op2, 12, val); \ READ_WB_REG_CASE(op2, 13, val); \ READ_WB_REG_CASE(op2, 14, val); \ READ_WB_REG_CASE(op2, 15, val) #define SWITCH_CASES_WRITE_WB_REG(op2, val) \ WRITE_WB_REG_CASE(op2, 0, val); \ WRITE_WB_REG_CASE(op2, 1, val); \ WRITE_WB_REG_CASE(op2, 2, val); \ WRITE_WB_REG_CASE(op2, 3, val); \ WRITE_WB_REG_CASE(op2, 4, val); \ WRITE_WB_REG_CASE(op2, 5, val); \ WRITE_WB_REG_CASE(op2, 6, val); \ WRITE_WB_REG_CASE(op2, 7, val); \ WRITE_WB_REG_CASE(op2, 8, val); \ WRITE_WB_REG_CASE(op2, 9, val); \ WRITE_WB_REG_CASE(op2, 10, val); \ WRITE_WB_REG_CASE(op2, 11, val); \ WRITE_WB_REG_CASE(op2, 12, val); \ WRITE_WB_REG_CASE(op2, 13, val); \ WRITE_WB_REG_CASE(op2, 14, val); \ WRITE_WB_REG_CASE(op2, 15, val) static uint32_t dbg_wb_read_reg(int reg, int n) { uint32_t val; val = 0; switch (reg + n) { SWITCH_CASES_READ_WB_REG(DBG_WB_WVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_WCR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BCR, val); default: db_printf( "trying to read from CP14 reg. using wrong opc2 %d\n", reg >> OP2_SHIFT); } return (val); } static void dbg_wb_write_reg(int reg, int n, uint32_t val) { switch (reg + n) { SWITCH_CASES_WRITE_WB_REG(DBG_WB_WVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_WCR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BCR, val); default: db_printf( "trying to write to CP14 reg. using wrong opc2 %d\n", reg >> OP2_SHIFT); } isb(); } boolean_t kdb_cpu_pc_is_singlestep(db_addr_t pc) { if (dbg_find_slot(DBG_TYPE_BREAKPOINT, pc) != ~0U) return (TRUE); return (FALSE); } void kdb_cpu_set_singlestep(void) { db_expr_t inst; db_addr_t pc, brpc; uint32_t wcr; u_int i; /* * Disable watchpoints, e.g. stepping over watched instruction will * trigger break exception instead of single-step exception and locks * CPU on that instruction for ever. */ for (i = 0; i < dbg_watchpoint_num; i++) { wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i); if ((wcr & DBG_WB_CTRL_E) != 0) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, (wcr & ~DBG_WB_CTRL_E)); } } pc = PC_REGS(); inst = db_get_value(pc, sizeof(pc), FALSE); if (inst_branch(inst) || inst_call(inst) || inst_return(inst)) { brpc = branch_taken(inst, pc); dbg_setup_breakpoint(brpc, INSN_SIZE, DBG_BKPT_BT_SLOT); } pc = next_instr_address(pc, 0); dbg_setup_breakpoint(pc, INSN_SIZE, DBG_BKPT_BNT_SLOT); } void kdb_cpu_clear_singlestep(void) { uint32_t wvr, wcr; u_int i; dbg_remove_breakpoint(DBG_BKPT_BT_SLOT); dbg_remove_breakpoint(DBG_BKPT_BNT_SLOT); /* Restore all watchpoints */ for (i = 0; i < dbg_watchpoint_num; i++) { wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i); wvr = dbg_wb_read_reg(DBG_REG_BASE_WVR, i); /* Watchpoint considered not empty if address value is not 0 */ if ((wvr & DBGWVR_ADDR_MASK) != 0) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, (wcr | DBG_WB_CTRL_E)); } } } int dbg_setup_watchpoint(db_expr_t addr, db_expr_t size, enum dbg_access_t access) { struct dbg_wb_conf conf; if (access == HW_BREAKPOINT_X) { db_printf("Invalid access type for watchpoint: %d\n", access); return (EINVAL); } conf.address = addr; conf.size = size; conf.access = access; conf.type = DBG_TYPE_WATCHPOINT; return (dbg_setup_xpoint(&conf)); } int dbg_remove_watchpoint(db_expr_t addr, db_expr_t size __unused) { struct dbg_wb_conf conf; conf.address = addr; conf.type = DBG_TYPE_WATCHPOINT; return (dbg_remove_xpoint(&conf)); } static int dbg_setup_breakpoint(db_expr_t addr, db_expr_t size, u_int slot) { struct dbg_wb_conf conf; conf.address = addr; conf.size = size; conf.access = HW_BREAKPOINT_X; conf.type = DBG_TYPE_BREAKPOINT; conf.slot = slot; return (dbg_setup_xpoint(&conf)); } static int dbg_remove_breakpoint(u_int slot) { struct dbg_wb_conf conf; /* Slot already cleared. Don't recurse */ if (dbg_check_slot_free(DBG_TYPE_BREAKPOINT, slot)) return (0); conf.slot = slot; conf.type = DBG_TYPE_BREAKPOINT; return (dbg_remove_xpoint(&conf)); } static const char * dbg_watchtype_str(uint32_t type) { switch (type) { case DBG_WB_CTRL_EXEC: return ("execute"); case DBG_WB_CTRL_STORE: return ("write"); case DBG_WB_CTRL_LOAD: return ("read"); case DBG_WB_CTRL_LOAD | DBG_WB_CTRL_STORE: return ("read/write"); default: return ("invalid"); } } static int dbg_watchtype_len(uint32_t len) { switch (len) { case DBG_WB_CTRL_LEN_1: return (1); case DBG_WB_CTRL_LEN_2: return (2); case DBG_WB_CTRL_LEN_4: return (4); case DBG_WB_CTRL_LEN_8: return (8); default: return (0); } } void dbg_show_watchpoint(void) { uint32_t wcr, len, type; uint32_t addr; boolean_t is_enabled; int i; if (!dbg_capable) { db_printf("Architecture does not support HW " "breakpoints/watchpoints\n"); return; } db_printf("\nhardware watchpoints:\n"); db_printf(" watch status type len address symbol\n"); db_printf(" ----- -------- ---------- --- ---------- ------------------\n"); for (i = 0; i < dbg_watchpoint_num; i++) { wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i); if ((wcr & DBG_WB_CTRL_E) != 0) is_enabled = TRUE; else is_enabled = FALSE; type = DBG_WB_CTRL_ACCESS_MASK(wcr); len = DBG_WB_CTRL_LEN_MASK(wcr); addr = dbg_wb_read_reg(DBG_REG_BASE_WVR, i) & DBGWVR_ADDR_MASK; db_printf(" %-5d %-8s %10s %3d 0x%08x ", i, is_enabled ? "enabled" : "disabled", is_enabled ? dbg_watchtype_str(type) : "", is_enabled ? dbg_watchtype_len(len) : 0, addr); db_printsym((db_addr_t)addr, DB_STGY_ANY); db_printf("\n"); } } static boolean_t dbg_check_slot_free(enum dbg_t type, u_int slot) { uint32_t cr, vr; uint32_t max; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; cr = DBG_REG_BASE_BCR; vr = DBG_REG_BASE_BVR; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; cr = DBG_REG_BASE_WCR; vr = DBG_REG_BASE_WVR; break; default: db_printf("%s: Unsupported event type %d\n", __func__, type); return (FALSE); } if (slot >= max) { db_printf("%s: Invalid slot number %d, max %d\n", __func__, slot, max - 1); return (FALSE); } if ((dbg_wb_read_reg(cr, slot) & DBG_WB_CTRL_E) == 0 && (dbg_wb_read_reg(vr, slot) & DBGWVR_ADDR_MASK) == 0) return (TRUE); return (FALSE); } static u_int dbg_find_free_slot(enum dbg_t type) { u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; break; default: db_printf("Unsupported debug type\n"); return (~0U); } for (i = 0; i < max; i++) { if (dbg_check_slot_free(type, i)) return (i); } return (~0U); } static u_int dbg_find_slot(enum dbg_t type, db_expr_t addr) { uint32_t reg_addr, reg_ctrl; u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg_addr = DBG_REG_BASE_BVR; reg_ctrl = DBG_REG_BASE_BCR; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg_addr = DBG_REG_BASE_WVR; reg_ctrl = DBG_REG_BASE_WCR; break; default: db_printf("Unsupported debug type\n"); return (~0U); } for (i = 0; i < max; i++) { if ((dbg_wb_read_reg(reg_addr, i) == addr) && ((dbg_wb_read_reg(reg_ctrl, i) & DBG_WB_CTRL_E) != 0)) return (i); } return (~0U); } static __inline boolean_t dbg_monitor_is_enabled(void) { return ((cp14_dbgdscrint_get() & DBGSCR_MDBG_EN) != 0); } static int dbg_enable_monitor(void) { uint32_t dbg_dscr; /* Already enabled? Just return */ if (dbg_monitor_is_enabled()) return (0); dbg_dscr = cp14_dbgdscrint_get(); switch (dbg_model) { case ID_DFR0_CP_DEBUG_M_V6: case ID_DFR0_CP_DEBUG_M_V6_1: /* fall through */ cp14_dbgdscr_v6_set(dbg_dscr | DBGSCR_MDBG_EN); break; case ID_DFR0_CP_DEBUG_M_V7: /* fall through */ case ID_DFR0_CP_DEBUG_M_V7_1: cp14_dbgdscr_v7_set(dbg_dscr | DBGSCR_MDBG_EN); break; default: break; } isb(); /* Verify that Monitor mode is set */ if (dbg_monitor_is_enabled()) return (0); return (ENXIO); } static int dbg_disable_monitor(void) { uint32_t dbg_dscr; if (!dbg_monitor_is_enabled()) return (0); dbg_dscr = cp14_dbgdscrint_get(); switch (dbg_model) { case ID_DFR0_CP_DEBUG_M_V6: case ID_DFR0_CP_DEBUG_M_V6_1: /* fall through */ dbg_dscr &= ~DBGSCR_MDBG_EN; cp14_dbgdscr_v6_set(dbg_dscr); break; case ID_DFR0_CP_DEBUG_M_V7: /* fall through */ case ID_DFR0_CP_DEBUG_M_V7_1: dbg_dscr &= ~DBGSCR_MDBG_EN; cp14_dbgdscr_v7_set(dbg_dscr); break; default: return (ENXIO); } isb(); return (0); } static int dbg_setup_xpoint(struct dbg_wb_conf *conf) { struct pcpu *pcpu; struct dbreg *d; const char *typestr; uint32_t cr_size, cr_priv, cr_access; uint32_t reg_ctrl, reg_addr, ctrl, addr; boolean_t is_bkpt; u_int cpuid, cpu; u_int i; int err; if (!dbg_capable) return (ENXIO); is_bkpt = (conf->type == DBG_TYPE_BREAKPOINT); typestr = is_bkpt ? "breakpoint" : "watchpoint"; cpuid = PCPU_GET(cpuid); if (!dbg_ready[cpuid]) { err = dbg_reset_state(); if (err != 0) return (err); dbg_ready[cpuid] = TRUE; } if (is_bkpt) { if (dbg_breakpoint_num == 0) { db_printf("Breakpoints not supported on this architecture\n"); return (ENXIO); } i = conf->slot; if (!dbg_check_slot_free(DBG_TYPE_BREAKPOINT, i)) { /* * This should never happen. If it does it means that * there is an erroneus scenario somewhere. Still, it can * be done but let's inform the user. */ db_printf("ERROR: Breakpoint already set. Replacing...\n"); } } else { i = dbg_find_free_slot(DBG_TYPE_WATCHPOINT); if (i == ~0U) { db_printf("Can not find slot for %s, max %d slots supported\n", typestr, dbg_watchpoint_num); return (ENXIO); } } /* Kernel access only */ cr_priv = DBG_WB_CTRL_PL1; switch(conf->size) { case 1: cr_size = DBG_WB_CTRL_LEN_1; break; case 2: cr_size = DBG_WB_CTRL_LEN_2; break; case 4: cr_size = DBG_WB_CTRL_LEN_4; break; case 8: cr_size = DBG_WB_CTRL_LEN_8; break; default: db_printf("Unsupported address size for %s\n", typestr); return (EINVAL); } if (is_bkpt) { cr_access = DBG_WB_CTRL_EXEC; reg_ctrl = DBG_REG_BASE_BCR; reg_addr = DBG_REG_BASE_BVR; /* Always unlinked BKPT */ ctrl = (cr_size | cr_access | cr_priv | DBG_WB_CTRL_E); } else { switch(conf->access) { case HW_WATCHPOINT_R: cr_access = DBG_WB_CTRL_LOAD; break; case HW_WATCHPOINT_W: cr_access = DBG_WB_CTRL_STORE; break; case HW_WATCHPOINT_RW: cr_access = DBG_WB_CTRL_LOAD | DBG_WB_CTRL_STORE; break; default: db_printf("Unsupported exception level for %s\n", typestr); return (EINVAL); } reg_ctrl = DBG_REG_BASE_WCR; reg_addr = DBG_REG_BASE_WVR; ctrl = (cr_size | cr_access | cr_priv | DBG_WB_CTRL_E); } addr = conf->address; dbg_wb_write_reg(reg_addr, i, addr); dbg_wb_write_reg(reg_ctrl, i, ctrl); err = dbg_enable_monitor(); if (err != 0) return (err); /* Increment monitor enable counter */ dbg_ref_count_mme++; /* * Save watchpoint settings for all CPUs. * We don't need to do the same with breakpoints since HW breakpoints * are only used to perform single stepping. */ if (!is_bkpt) { CPU_FOREACH(cpu) { pcpu = pcpu_find(cpu); /* Fill out the settings for watchpoint */ d = (struct dbreg *)pcpu->pc_dbreg; d->dbg_wvr[i] = addr; d->dbg_wcr[i] = ctrl; /* Skip update command for the current CPU */ if (cpu != cpuid) pcpu->pc_dbreg_cmd = PC_DBREG_CMD_LOAD; } } /* Ensure all data is written before waking other CPUs */ atomic_thread_fence_rel(); return (0); } static int dbg_remove_xpoint(struct dbg_wb_conf *conf) { struct pcpu *pcpu; struct dbreg *d; uint32_t reg_ctrl, reg_addr, addr; boolean_t is_bkpt; u_int cpuid, cpu; u_int i; int err; if (!dbg_capable) return (ENXIO); is_bkpt = (conf->type == DBG_TYPE_BREAKPOINT); cpuid = PCPU_GET(cpuid); if (!dbg_ready[cpuid]) { err = dbg_reset_state(); if (err != 0) return (err); dbg_ready[cpuid] = TRUE; } addr = conf->address; if (is_bkpt) { i = conf->slot; reg_ctrl = DBG_REG_BASE_BCR; reg_addr = DBG_REG_BASE_BVR; } else { i = dbg_find_slot(DBG_TYPE_WATCHPOINT, addr); if (i == ~0U) { db_printf("Can not find watchpoint for address 0%x\n", addr); return (EINVAL); } reg_ctrl = DBG_REG_BASE_WCR; reg_addr = DBG_REG_BASE_WVR; } dbg_wb_write_reg(reg_ctrl, i, 0); dbg_wb_write_reg(reg_addr, i, 0); /* Decrement monitor enable counter */ dbg_ref_count_mme--; if (dbg_ref_count_mme < 0) dbg_ref_count_mme = 0; atomic_thread_fence_rel(); if (dbg_ref_count_mme == 0) { err = dbg_disable_monitor(); if (err != 0) return (err); } /* * Save watchpoint settings for all CPUs. * We don't need to do the same with breakpoints since HW breakpoints * are only used to perform single stepping. */ if (!is_bkpt) { CPU_FOREACH(cpu) { pcpu = pcpu_find(cpu); /* Fill out the settings for watchpoint */ d = (struct dbreg *)pcpu->pc_dbreg; d->dbg_wvr[i] = 0; d->dbg_wcr[i] = 0; /* Skip update command for the current CPU */ if (cpu != cpuid) pcpu->pc_dbreg_cmd = PC_DBREG_CMD_LOAD; } /* Ensure all data is written before waking other CPUs */ atomic_thread_fence_rel(); } return (0); } static __inline uint32_t dbg_get_debug_model(void) { uint32_t dbg_m; dbg_m = ((cpuinfo.id_dfr0 & ID_DFR0_CP_DEBUG_M_MASK) >> ID_DFR0_CP_DEBUG_M_SHIFT); return (dbg_m); } static __inline boolean_t dbg_get_ossr(void) { switch (dbg_model) { case ID_DFR0_CP_DEBUG_M_V6_1: if ((cp14_dbgoslsr_get() & DBGOSLSR_OSLM0) != 0) return (TRUE); return (FALSE); case ID_DFR0_CP_DEBUG_M_V7_1: return (TRUE); default: return (FALSE); } } static __inline boolean_t dbg_arch_supported(void) { switch (dbg_model) { #ifdef not_yet case ID_DFR0_CP_DEBUG_M_V6: case ID_DFR0_CP_DEBUG_M_V6_1: #endif case ID_DFR0_CP_DEBUG_M_V7: case ID_DFR0_CP_DEBUG_M_V7_1: /* fall through */ return (TRUE); default: /* We only support valid v6.x/v7.x modes through CP14 */ return (FALSE); } } static __inline uint32_t dbg_get_wrp_num(void) { uint32_t dbg_didr; dbg_didr = cp14_dbgdidr_get(); return (DBGDIDR_WRPS_NUM(dbg_didr)); } static __inline uint32_t dgb_get_brp_num(void) { uint32_t dbg_didr; dbg_didr = cp14_dbgdidr_get(); return (DBGDIDR_BRPS_NUM(dbg_didr)); } static int dbg_reset_state(void) { u_int cpuid; size_t i; int err; cpuid = PCPU_GET(cpuid); err = 0; switch (dbg_model) { case ID_DFR0_CP_DEBUG_M_V6: /* v6 Debug logic reset upon power-up */ return (0); case ID_DFR0_CP_DEBUG_M_V6_1: /* Is core power domain powered up? */ if ((cp14_dbgprsr_get() & DBGPRSR_PU) == 0) err = ENXIO; if (err != 0) break; if (dbg_ossr) goto vectr_clr; break; case ID_DFR0_CP_DEBUG_M_V7: break; case ID_DFR0_CP_DEBUG_M_V7_1: /* Is double lock set? */ if ((cp14_dbgosdlr_get() & DBGPRSR_DLK) != 0) err = ENXIO; break; default: break; } if (err != 0) { db_printf("Debug facility locked (CPU%d)\n", cpuid); return (err); } /* * DBGOSLAR is always implemented for v7.1 Debug Arch. however is * optional for v7 (depends on OS save and restore support). */ if (((dbg_model & ID_DFR0_CP_DEBUG_M_V7_1) != 0) || dbg_ossr) { /* * Clear OS lock. * Writing any other value than 0xC5ACCESS will unlock. */ cp14_dbgoslar_set(0); isb(); } vectr_clr: /* * After reset we must ensure that DBGVCR has a defined value. * Disable all vector catch events. Safe to use - required in all * implementations. */ cp14_dbgvcr_set(0); isb(); /* * We have limited number of {watch,break}points, each consists of * two registers: * - wcr/bcr regsiter configurates corresponding {watch,break}point * behaviour * - wvr/bvr register keeps address we are hunting for * * Reset all breakpoints and watchpoints. */ for (i = 0; i < dbg_watchpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } return (0); } void dbg_monitor_init(void) { int err; /* Fetch ARM Debug Architecture model */ dbg_model = dbg_get_debug_model(); if (!dbg_arch_supported()) { db_printf("ARM Debug Architecture not supported\n"); return; } if (bootverbose) { db_printf("ARM Debug Architecture %s\n", (dbg_model == ID_DFR0_CP_DEBUG_M_V6) ? "v6" : (dbg_model == ID_DFR0_CP_DEBUG_M_V6_1) ? "v6.1" : (dbg_model == ID_DFR0_CP_DEBUG_M_V7) ? "v7" : (dbg_model == ID_DFR0_CP_DEBUG_M_V7_1) ? "v7.1" : "unknown"); } /* Do we have OS Save and Restore mechanism? */ dbg_ossr = dbg_get_ossr(); /* Find out many breakpoints and watchpoints we can use */ dbg_watchpoint_num = dbg_get_wrp_num(); dbg_breakpoint_num = dgb_get_brp_num(); if (bootverbose) { db_printf("%d watchpoints and %d breakpoints supported\n", dbg_watchpoint_num, dbg_breakpoint_num); } err = dbg_reset_state(); if (err == 0) { dbg_capable = TRUE; return; } db_printf("HW Breakpoints/Watchpoints not enabled on CPU%d\n", PCPU_GET(cpuid)); } CTASSERT(sizeof(struct dbreg) == sizeof(((struct pcpu *)NULL)->pc_dbreg)); void dbg_resume_dbreg(void) { struct dbreg *d; u_int cpuid; u_int i; int err; /* * This flag is set on the primary CPU * and its meaning is valid for other CPUs too. */ if (!dbg_capable) return; atomic_thread_fence_acq(); switch (PCPU_GET(dbreg_cmd)) { case PC_DBREG_CMD_LOAD: d = (struct dbreg *)PCPU_PTR(dbreg); cpuid = PCPU_GET(cpuid); /* Reset Debug Architecture State if not done earlier */ if (!dbg_ready[cpuid]) { err = dbg_reset_state(); if (err != 0) { /* * Something is very wrong. * WPs/BPs will not work correctly in this CPU. */ panic("%s: Failed to reset Debug Architecture " "state on CPU%d", __func__, cpuid); } dbg_ready[cpuid] = TRUE; } /* Restore watchpoints */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WVR, i, d->dbg_wvr[i]); dbg_wb_write_reg(DBG_REG_BASE_WCR, i, d->dbg_wcr[i]); } if ((dbg_ref_count_mme > 0) && !dbg_monitor_is_enabled()) { err = dbg_enable_monitor(); if (err != 0) { panic("%s: Failed to enable Debug Monitor " "on CPU%d", __func__, cpuid); } } if ((dbg_ref_count_mme == 0) && dbg_monitor_is_enabled()) { err = dbg_disable_monitor(); if (err != 0) { panic("%s: Failed to disable Debug Monitor " "on CPU%d", __func__, cpuid); } } PCPU_SET(dbreg_cmd, PC_DBREG_CMD_NONE); break; } } Index: head/sys/arm64/arm64/debug_monitor.c =================================================================== --- head/sys/arm64/arm64/debug_monitor.c (revision 295880) +++ head/sys/arm64/arm64/debug_monitor.c (revision 295881) @@ -1,487 +1,486 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include -#include #include #include enum dbg_t { DBG_TYPE_BREAKPOINT = 0, DBG_TYPE_WATCHPOINT = 1, }; static int dbg_watchpoint_num; static int dbg_breakpoint_num; static int dbg_ref_count_mde[MAXCPU]; static int dbg_ref_count_kde[MAXCPU]; /* Watchpoints/breakpoints control register bitfields */ #define DBG_WATCH_CTRL_LEN_1 (0x1 << 5) #define DBG_WATCH_CTRL_LEN_2 (0x3 << 5) #define DBG_WATCH_CTRL_LEN_4 (0xf << 5) #define DBG_WATCH_CTRL_LEN_8 (0xff << 5) #define DBG_WATCH_CTRL_LEN_MASK(x) ((x) & (0xff << 5)) #define DBG_WATCH_CTRL_EXEC (0x0 << 3) #define DBG_WATCH_CTRL_LOAD (0x1 << 3) #define DBG_WATCH_CTRL_STORE (0x2 << 3) #define DBG_WATCH_CTRL_ACCESS_MASK(x) ((x) & (0x3 << 3)) /* Common for breakpoint and watchpoint */ #define DBG_WB_CTRL_EL1 (0x1 << 1) #define DBG_WB_CTRL_EL0 (0x2 << 1) #define DBG_WB_CTRL_ELX_MASK(x) ((x) & (0x3 << 1)) #define DBG_WB_CTRL_E (0x1 << 0) #define DBG_REG_BASE_BVR 0 #define DBG_REG_BASE_BCR (DBG_REG_BASE_BVR + 16) #define DBG_REG_BASE_WVR (DBG_REG_BASE_BCR + 16) #define DBG_REG_BASE_WCR (DBG_REG_BASE_WVR + 16) /* Watchpoint/breakpoint helpers */ #define DBG_WB_WVR "wvr" #define DBG_WB_WCR "wcr" #define DBG_WB_BVR "bvr" #define DBG_WB_BCR "bcr" #define DBG_WB_READ(reg, num, val) do { \ __asm __volatile("mrs %0, dbg" reg #num "_el1" : "=r" (val)); \ } while (0) #define DBG_WB_WRITE(reg, num, val) do { \ __asm __volatile("msr dbg" reg #num "_el1, %0" :: "r" (val)); \ } while (0) #define READ_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_READ(reg, num, val); \ break #define WRITE_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_WRITE(reg, num, val); \ break #define SWITCH_CASES_READ_WB_REG(reg, offset, val) \ READ_WB_REG_CASE(reg, 0, offset, val); \ READ_WB_REG_CASE(reg, 1, offset, val); \ READ_WB_REG_CASE(reg, 2, offset, val); \ READ_WB_REG_CASE(reg, 3, offset, val); \ READ_WB_REG_CASE(reg, 4, offset, val); \ READ_WB_REG_CASE(reg, 5, offset, val); \ READ_WB_REG_CASE(reg, 6, offset, val); \ READ_WB_REG_CASE(reg, 7, offset, val); \ READ_WB_REG_CASE(reg, 8, offset, val); \ READ_WB_REG_CASE(reg, 9, offset, val); \ READ_WB_REG_CASE(reg, 10, offset, val); \ READ_WB_REG_CASE(reg, 11, offset, val); \ READ_WB_REG_CASE(reg, 12, offset, val); \ READ_WB_REG_CASE(reg, 13, offset, val); \ READ_WB_REG_CASE(reg, 14, offset, val); \ READ_WB_REG_CASE(reg, 15, offset, val) #define SWITCH_CASES_WRITE_WB_REG(reg, offset, val) \ WRITE_WB_REG_CASE(reg, 0, offset, val); \ WRITE_WB_REG_CASE(reg, 1, offset, val); \ WRITE_WB_REG_CASE(reg, 2, offset, val); \ WRITE_WB_REG_CASE(reg, 3, offset, val); \ WRITE_WB_REG_CASE(reg, 4, offset, val); \ WRITE_WB_REG_CASE(reg, 5, offset, val); \ WRITE_WB_REG_CASE(reg, 6, offset, val); \ WRITE_WB_REG_CASE(reg, 7, offset, val); \ WRITE_WB_REG_CASE(reg, 8, offset, val); \ WRITE_WB_REG_CASE(reg, 9, offset, val); \ WRITE_WB_REG_CASE(reg, 10, offset, val); \ WRITE_WB_REG_CASE(reg, 11, offset, val); \ WRITE_WB_REG_CASE(reg, 12, offset, val); \ WRITE_WB_REG_CASE(reg, 13, offset, val); \ WRITE_WB_REG_CASE(reg, 14, offset, val); \ WRITE_WB_REG_CASE(reg, 15, offset, val) static uint64_t dbg_wb_read_reg(int reg, int n) { uint64_t val = 0; switch (reg + n) { SWITCH_CASES_READ_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: db_printf("trying to read from wrong debug register %d\n", n); } return val; } static void dbg_wb_write_reg(int reg, int n, uint64_t val) { switch (reg + n) { SWITCH_CASES_WRITE_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: db_printf("trying to write to wrong debug register %d\n", n); } isb(); } void kdb_cpu_set_singlestep(void) { kdb_frame->tf_spsr |= DBG_SPSR_SS; WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) | DBG_MDSCR_SS | DBG_MDSCR_KDE); /* * Disable breakpoints and watchpoints, e.g. stepping * over watched instruction will trigger break exception instead of * single-step exception and locks CPU on that instruction for ever. */ if (dbg_ref_count_mde[PCPU_GET(cpuid)] > 0) { WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) & ~DBG_MDSCR_MDE); } } void kdb_cpu_clear_singlestep(void) { WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) & ~(DBG_MDSCR_SS | DBG_MDSCR_KDE)); /* Restore breakpoints and watchpoints */ if (dbg_ref_count_mde[PCPU_GET(cpuid)] > 0) { WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) | DBG_MDSCR_MDE); } if (dbg_ref_count_kde[PCPU_GET(cpuid)] > 0) { WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) | DBG_MDSCR_KDE); } } static const char * dbg_watchtype_str(uint32_t type) { switch (type) { case DBG_WATCH_CTRL_EXEC: return ("execute"); case DBG_WATCH_CTRL_STORE: return ("write"); case DBG_WATCH_CTRL_LOAD: return ("read"); case DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE: return ("read/write"); default: return ("invalid"); } } static int dbg_watchtype_len(uint32_t len) { switch (len) { case DBG_WATCH_CTRL_LEN_1: return (1); case DBG_WATCH_CTRL_LEN_2: return (2); case DBG_WATCH_CTRL_LEN_4: return (4); case DBG_WATCH_CTRL_LEN_8: return (8); default: return (0); } } void dbg_show_watchpoint(void) { uint32_t wcr, len, type; uint64_t addr; int i; db_printf("\nhardware watchpoints:\n"); db_printf(" watch status type len address symbol\n"); db_printf(" ----- -------- ---------- --- ------------------ ------------------\n"); for (i = 0; i < dbg_watchpoint_num; i++) { wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i); if ((wcr & DBG_WB_CTRL_E) != 0) { type = DBG_WATCH_CTRL_ACCESS_MASK(wcr); len = DBG_WATCH_CTRL_LEN_MASK(wcr); addr = dbg_wb_read_reg(DBG_REG_BASE_WVR, i); db_printf(" %-5d %-8s %10s %3d 0x%16lx ", i, "enabled", dbg_watchtype_str(type), dbg_watchtype_len(len), addr); db_printsym((db_addr_t)addr, DB_STGY_ANY); db_printf("\n"); } else { db_printf(" %-5d disabled\n", i); } } } static int dbg_find_free_slot(enum dbg_t type) { u_int max, reg, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg = DBG_REG_BASE_BCR; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg = DBG_REG_BASE_WCR; break; default: db_printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if ((dbg_wb_read_reg(reg, i) & DBG_WB_CTRL_E) == 0) return (i); } return (-1); } static int dbg_find_slot(enum dbg_t type, db_expr_t addr) { u_int max, reg_addr, reg_ctrl, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg_addr = DBG_REG_BASE_BVR; reg_ctrl = DBG_REG_BASE_BCR; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg_addr = DBG_REG_BASE_WVR; reg_ctrl = DBG_REG_BASE_WCR; break; default: db_printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if ((dbg_wb_read_reg(reg_addr, i) == addr) && ((dbg_wb_read_reg(reg_ctrl, i) & DBG_WB_CTRL_E) != 0)) return (i); } return (-1); } static void dbg_enable_monitor(enum dbg_el_t el) { uint64_t reg_mdcr = 0; /* * There is no need to have debug monitor on permanently, thus we are * refcounting and turn it on only if any of CPU is going to use that. */ if (atomic_fetchadd_int(&dbg_ref_count_mde[PCPU_GET(cpuid)], 1) == 0) reg_mdcr = DBG_MDSCR_MDE; if ((el == DBG_FROM_EL1) && atomic_fetchadd_int(&dbg_ref_count_kde[PCPU_GET(cpuid)], 1) == 0) reg_mdcr |= DBG_MDSCR_KDE; if (reg_mdcr) WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) | reg_mdcr); } static void dbg_disable_monitor(enum dbg_el_t el) { uint64_t reg_mdcr = 0; if (atomic_fetchadd_int(&dbg_ref_count_mde[PCPU_GET(cpuid)], -1) == 1) reg_mdcr = DBG_MDSCR_MDE; if ((el == DBG_FROM_EL1) && atomic_fetchadd_int(&dbg_ref_count_kde[PCPU_GET(cpuid)], -1) == 1) reg_mdcr |= DBG_MDSCR_KDE; if (reg_mdcr) WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) & ~reg_mdcr); } int dbg_setup_watchpoint(db_expr_t addr, db_expr_t size, enum dbg_el_t el, enum dbg_access_t access) { uint64_t wcr_size, wcr_priv, wcr_access; u_int i; i = dbg_find_free_slot(DBG_TYPE_WATCHPOINT); if (i == -1) { db_printf("Can not find slot for watchpoint, max %d" " watchpoints supported\n", dbg_watchpoint_num); return (i); } switch(size) { case 1: wcr_size = DBG_WATCH_CTRL_LEN_1; break; case 2: wcr_size = DBG_WATCH_CTRL_LEN_2; break; case 4: wcr_size = DBG_WATCH_CTRL_LEN_4; break; case 8: wcr_size = DBG_WATCH_CTRL_LEN_8; break; default: db_printf("Unsupported address size for watchpoint\n"); return (-1); } switch(el) { case DBG_FROM_EL0: wcr_priv = DBG_WB_CTRL_EL0; break; case DBG_FROM_EL1: wcr_priv = DBG_WB_CTRL_EL1; break; default: db_printf("Unsupported exception level for watchpoint\n"); return (-1); } switch(access) { case HW_BREAKPOINT_X: wcr_access = DBG_WATCH_CTRL_EXEC; break; case HW_BREAKPOINT_R: wcr_access = DBG_WATCH_CTRL_LOAD; break; case HW_BREAKPOINT_W: wcr_access = DBG_WATCH_CTRL_STORE; break; case HW_BREAKPOINT_RW: wcr_access = DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE; break; default: db_printf("Unsupported exception level for watchpoint\n"); return (-1); } dbg_wb_write_reg(DBG_REG_BASE_WVR, i, addr); dbg_wb_write_reg(DBG_REG_BASE_WCR, i, wcr_size | wcr_access | wcr_priv | DBG_WB_CTRL_E); dbg_enable_monitor(el); return (0); } int dbg_remove_watchpoint(db_expr_t addr, db_expr_t size, enum dbg_el_t el) { u_int i; i = dbg_find_slot(DBG_TYPE_WATCHPOINT, addr); if (i == -1) { db_printf("Can not find watchpoint for address 0%lx\n", addr); return (i); } dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_disable_monitor(el); return (0); } void dbg_monitor_init(void) { u_int i; /* Clear OS lock */ WRITE_SPECIALREG(OSLAR_EL1, 0); /* Find out many breakpoints and watchpoints we can use */ dbg_watchpoint_num = ((READ_SPECIALREG(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1; dbg_breakpoint_num = ((READ_SPECIALREG(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1; if (bootverbose && PCPU_GET(cpuid) == 0) { db_printf("%d watchpoints and %d breakpoints supported\n", dbg_watchpoint_num, dbg_breakpoint_num); } /* * We have limited number of {watch,break}points, each consists of * two registers: * - wcr/bcr regsiter configurates corresponding {watch,break}point * behaviour * - wvr/bvr register keeps address we are hunting for * * Reset all breakpoints and watchpoints. */ for (i = 0; i < dbg_watchpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } dbg_enable(); } Index: head/sys/dev/drm/drmP.h =================================================================== --- head/sys/dev/drm/drmP.h (revision 295880) +++ head/sys/dev/drm/drmP.h (revision 295881) @@ -1,1011 +1,1010 @@ /* drmP.h -- Private header for Direct Rendering Manager -*- linux-c -*- * Created: Mon Jan 4 10:05:05 1999 by faith@precisioninsight.com */ /*- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas. * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. * All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: * Rickard E. (Rik) Faith * Gareth Hughes * */ #include __FBSDID("$FreeBSD$"); #ifndef _DRM_P_H_ #define _DRM_P_H_ #if defined(_KERNEL) || defined(__KERNEL__) struct drm_device; struct drm_file; #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #if defined(__i386__) || defined(__amd64__) #include #endif #include #include #if _BYTE_ORDER == _BIG_ENDIAN #define __BIG_ENDIAN 1 #else #define __LITTLE_ENDIAN 1 #endif #include #include #include #include #include #include #include #include #include #include #include "dev/drm/drm.h" #include "dev/drm/drm_atomic.h" #include "dev/drm/drm_internal.h" #include "dev/drm/drm_linux_list.h" #include #ifdef DRM_DEBUG #undef DRM_DEBUG #define DRM_DEBUG_DEFAULT_ON 1 #endif /* DRM_DEBUG */ #if defined(DRM_LINUX) && DRM_LINUX && !defined(__amd64__) #include #include #include #include #else /* Either it was defined when it shouldn't be (FreeBSD amd64) or it isn't * supported on this OS yet. */ #undef DRM_LINUX #define DRM_LINUX 0 #endif /* driver capabilities and requirements mask */ #define DRIVER_USE_AGP 0x1 #define DRIVER_REQUIRE_AGP 0x2 #define DRIVER_USE_MTRR 0x4 #define DRIVER_PCI_DMA 0x8 #define DRIVER_SG 0x10 #define DRIVER_HAVE_DMA 0x20 #define DRIVER_HAVE_IRQ 0x40 #define DRIVER_DMA_QUEUE 0x100 #define DRM_HASH_SIZE 16 /* Size of key hash table */ #define DRM_KERNEL_CONTEXT 0 /* Change drm_resctx if changed */ #define DRM_RESERVED_CONTEXTS 1 /* Change drm_resctx if changed */ MALLOC_DECLARE(DRM_MEM_DMA); MALLOC_DECLARE(DRM_MEM_SAREA); MALLOC_DECLARE(DRM_MEM_DRIVER); MALLOC_DECLARE(DRM_MEM_MAGIC); MALLOC_DECLARE(DRM_MEM_IOCTLS); MALLOC_DECLARE(DRM_MEM_MAPS); MALLOC_DECLARE(DRM_MEM_BUFS); MALLOC_DECLARE(DRM_MEM_SEGS); MALLOC_DECLARE(DRM_MEM_PAGES); MALLOC_DECLARE(DRM_MEM_FILES); MALLOC_DECLARE(DRM_MEM_QUEUES); MALLOC_DECLARE(DRM_MEM_CMDS); MALLOC_DECLARE(DRM_MEM_MAPPINGS); MALLOC_DECLARE(DRM_MEM_BUFLISTS); MALLOC_DECLARE(DRM_MEM_AGPLISTS); MALLOC_DECLARE(DRM_MEM_CTXBITMAP); MALLOC_DECLARE(DRM_MEM_SGLISTS); MALLOC_DECLARE(DRM_MEM_DRAWABLE); MALLOC_DECLARE(DRM_MEM_MM); MALLOC_DECLARE(DRM_MEM_HASHTAB); SYSCTL_DECL(_hw_drm); #define DRM_MAX_CTXBITMAP (PAGE_SIZE * 8) /* Internal types and structures */ #define DRM_ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) #define DRM_MIN(a,b) ((a)<(b)?(a):(b)) #define DRM_MAX(a,b) ((a)>(b)?(a):(b)) #define DRM_IF_VERSION(maj, min) (maj << 16 | min) #define __OS_HAS_AGP 1 #define DRM_DEV_MODE (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP) #define DRM_DEV_UID UID_ROOT #define DRM_DEV_GID GID_VIDEO #define wait_queue_head_t atomic_t #define DRM_WAKEUP(w) wakeup((void *)w) #define DRM_WAKEUP_INT(w) wakeup(w) #define DRM_INIT_WAITQUEUE(queue) do {(void)(queue);} while (0) #define DRM_CURPROC curthread #define DRM_STRUCTPROC struct thread #define DRM_SPINTYPE struct mtx #define DRM_SPININIT(l,name) mtx_init(l, name, NULL, MTX_DEF) #define DRM_SPINUNINIT(l) mtx_destroy(l) #define DRM_SPINLOCK(l) mtx_lock(l) #define DRM_SPINUNLOCK(u) mtx_unlock(u) #define DRM_SPINLOCK_IRQSAVE(l, irqflags) do { \ mtx_lock(l); \ (void)irqflags; \ } while (0) #define DRM_SPINUNLOCK_IRQRESTORE(u, irqflags) mtx_unlock(u) #define DRM_SPINLOCK_ASSERT(l) mtx_assert(l, MA_OWNED) #define DRM_CURRENTPID curthread->td_proc->p_pid #define DRM_LOCK() mtx_lock(&dev->dev_lock) #define DRM_UNLOCK() mtx_unlock(&dev->dev_lock) #define DRM_SYSCTL_HANDLER_ARGS (SYSCTL_HANDLER_ARGS) #define DRM_IRQ_ARGS void *arg typedef void irqreturn_t; #define IRQ_HANDLED /* nothing */ #define IRQ_NONE /* nothing */ #define unlikely(x) __builtin_expect(!!(x), 0) #define container_of(ptr, type, member) ({ \ __typeof( ((type *)0)->member ) *__mptr = (ptr); \ (type *)( (char *)__mptr - offsetof(type,member) );}) enum { DRM_IS_NOT_AGP, DRM_IS_AGP, DRM_MIGHT_BE_AGP }; #define DRM_AGP_MEM struct agp_memory_info #define drm_get_device_from_kdev(_kdev) (_kdev->si_drv1) #define PAGE_ALIGN(addr) round_page(addr) /* DRM_SUSER returns true if the user is superuser */ #define DRM_SUSER(p) (priv_check(p, PRIV_DRIVER) == 0) #define DRM_AGP_FIND_DEVICE() agp_find_device() #define DRM_MTRR_WC MDF_WRITECOMBINE #define jiffies ticks typedef vm_paddr_t dma_addr_t; typedef u_int64_t u64; typedef u_int32_t u32; typedef u_int16_t u16; typedef u_int8_t u8; /* DRM_READMEMORYBARRIER() prevents reordering of reads. * DRM_WRITEMEMORYBARRIER() prevents reordering of writes. * DRM_MEMORYBARRIER() prevents reordering of reads and writes. */ #define DRM_READMEMORYBARRIER() rmb() #define DRM_WRITEMEMORYBARRIER() wmb() #define DRM_MEMORYBARRIER() mb() #define DRM_READ8(map, offset) \ *(volatile u_int8_t *)(((vm_offset_t)(map)->virtual) + \ (vm_offset_t)(offset)) #define DRM_READ16(map, offset) \ le16toh(*(volatile u_int16_t *)(((vm_offset_t)(map)->virtual) + \ (vm_offset_t)(offset))) #define DRM_READ32(map, offset) \ le32toh(*(volatile u_int32_t *)(((vm_offset_t)(map)->virtual) + \ (vm_offset_t)(offset))) #define DRM_WRITE8(map, offset, val) \ *(volatile u_int8_t *)(((vm_offset_t)(map)->virtual) + \ (vm_offset_t)(offset)) = val #define DRM_WRITE16(map, offset, val) \ *(volatile u_int16_t *)(((vm_offset_t)(map)->virtual) + \ (vm_offset_t)(offset)) = htole16(val) #define DRM_WRITE32(map, offset, val) \ *(volatile u_int32_t *)(((vm_offset_t)(map)->virtual) + \ (vm_offset_t)(offset)) = htole32(val) #define DRM_VERIFYAREA_READ( uaddr, size ) \ (!useracc(__DECONST(caddr_t, uaddr), size, VM_PROT_READ)) #define DRM_COPY_TO_USER(user, kern, size) \ copyout(kern, user, size) #define DRM_COPY_FROM_USER(kern, user, size) \ copyin(user, kern, size) #define DRM_COPY_FROM_USER_UNCHECKED(arg1, arg2, arg3) \ copyin(arg2, arg1, arg3) #define DRM_COPY_TO_USER_UNCHECKED(arg1, arg2, arg3) \ copyout(arg2, arg1, arg3) #define DRM_GET_USER_UNCHECKED(val, uaddr) \ ((val) = fuword32(uaddr), 0) #define cpu_to_le32(x) htole32(x) #define le32_to_cpu(x) le32toh(x) #define DRM_HZ hz #define DRM_UDELAY(udelay) DELAY(udelay) #define DRM_TIME_SLICE (hz/20) /* Time slice for GLXContexts */ #define DRM_GET_PRIV_SAREA(_dev, _ctx, _map) do { \ (_map) = (_dev)->context_sareas[_ctx]; \ } while(0) #define LOCK_TEST_WITH_RETURN(dev, file_priv) \ do { \ if (!_DRM_LOCK_IS_HELD(dev->lock.hw_lock->lock) || \ dev->lock.file_priv != file_priv) { \ DRM_ERROR("%s called without lock held\n", \ __FUNCTION__); \ return EINVAL; \ } \ } while (0) /* Returns -errno to shared code */ #define DRM_WAIT_ON( ret, queue, timeout, condition ) \ for ( ret = 0 ; !ret && !(condition) ; ) { \ DRM_UNLOCK(); \ mtx_lock(&dev->irq_lock); \ if (!(condition)) \ ret = -mtx_sleep(&(queue), &dev->irq_lock, \ PCATCH, "drmwtq", (timeout)); \ mtx_unlock(&dev->irq_lock); \ DRM_LOCK(); \ } #define DRM_ERROR(fmt, ...) \ printf("error: [" DRM_NAME ":pid%d:%s] *ERROR* " fmt, \ DRM_CURRENTPID, __func__ , ##__VA_ARGS__) #define DRM_INFO(fmt, ...) printf("info: [" DRM_NAME "] " fmt , ##__VA_ARGS__) #define DRM_DEBUG(fmt, ...) do { \ if (drm_debug_flag) \ printf("[" DRM_NAME ":pid%d:%s] " fmt, DRM_CURRENTPID, \ __func__ , ##__VA_ARGS__); \ } while (0) typedef struct drm_pci_id_list { int vendor; int device; long driver_private; char *name; } drm_pci_id_list_t; struct drm_msi_blacklist_entry { int vendor; int device; }; #define DRM_AUTH 0x1 #define DRM_MASTER 0x2 #define DRM_ROOT_ONLY 0x4 typedef struct drm_ioctl_desc { unsigned long cmd; int (*func)(struct drm_device *dev, void *data, struct drm_file *file_priv); int flags; } drm_ioctl_desc_t; /** * Creates a driver or general drm_ioctl_desc array entry for the given * ioctl, for use by drm_ioctl(). */ #define DRM_IOCTL_DEF(ioctl, func, flags) \ [DRM_IOCTL_NR(ioctl)] = {ioctl, func, flags} typedef struct drm_magic_entry { drm_magic_t magic; struct drm_file *priv; struct drm_magic_entry *next; } drm_magic_entry_t; typedef struct drm_magic_head { struct drm_magic_entry *head; struct drm_magic_entry *tail; } drm_magic_head_t; typedef struct drm_buf { int idx; /* Index into master buflist */ int total; /* Buffer size */ int order; /* log-base-2(total) */ int used; /* Amount of buffer in use (for DMA) */ unsigned long offset; /* Byte offset (used internally) */ void *address; /* Address of buffer */ unsigned long bus_address; /* Bus address of buffer */ struct drm_buf *next; /* Kernel-only: used for free list */ __volatile__ int pending; /* On hardware DMA queue */ struct drm_file *file_priv; /* Unique identifier of holding process */ int context; /* Kernel queue for this buffer */ enum { DRM_LIST_NONE = 0, DRM_LIST_FREE = 1, DRM_LIST_WAIT = 2, DRM_LIST_PEND = 3, DRM_LIST_PRIO = 4, DRM_LIST_RECLAIM = 5 } list; /* Which list we're on */ int dev_priv_size; /* Size of buffer private stoarge */ void *dev_private; /* Per-buffer private storage */ } drm_buf_t; typedef struct drm_freelist { int initialized; /* Freelist in use */ atomic_t count; /* Number of free buffers */ drm_buf_t *next; /* End pointer */ int low_mark; /* Low water mark */ int high_mark; /* High water mark */ } drm_freelist_t; typedef struct drm_dma_handle { void *vaddr; bus_addr_t busaddr; bus_dma_tag_t tag; bus_dmamap_t map; } drm_dma_handle_t; typedef struct drm_buf_entry { int buf_size; int buf_count; drm_buf_t *buflist; int seg_count; drm_dma_handle_t **seglist; int page_order; drm_freelist_t freelist; } drm_buf_entry_t; typedef TAILQ_HEAD(drm_file_list, drm_file) drm_file_list_t; struct drm_file { TAILQ_ENTRY(drm_file) link; struct drm_device *dev; int authenticated; int master; pid_t pid; uid_t uid; drm_magic_t magic; unsigned long ioctl_count; void *driver_priv; }; typedef struct drm_lock_data { struct drm_hw_lock *hw_lock; /* Hardware lock */ struct drm_file *file_priv; /* Unique identifier of holding process (NULL is kernel)*/ int lock_queue; /* Queue of blocked processes */ unsigned long lock_time; /* Time of last lock in jiffies */ } drm_lock_data_t; /* This structure, in the struct drm_device, is always initialized while the * device * is open. dev->dma_lock protects the incrementing of dev->buf_use, which * when set marks that no further bufs may be allocated until device teardown * occurs (when the last open of the device has closed). The high/low * watermarks of bufs are only touched by the X Server, and thus not * concurrently accessed, so no locking is needed. */ typedef struct drm_device_dma { drm_buf_entry_t bufs[DRM_MAX_ORDER+1]; int buf_count; drm_buf_t **buflist; /* Vector of pointers info bufs */ int seg_count; int page_count; unsigned long *pagelist; unsigned long byte_count; enum { _DRM_DMA_USE_AGP = 0x01, _DRM_DMA_USE_SG = 0x02 } flags; } drm_device_dma_t; typedef struct drm_agp_mem { void *handle; unsigned long bound; /* address */ int pages; struct drm_agp_mem *prev; struct drm_agp_mem *next; } drm_agp_mem_t; typedef struct drm_agp_head { device_t agpdev; struct agp_info info; const char *chipset; drm_agp_mem_t *memory; unsigned long mode; int enabled; int acquired; unsigned long base; int mtrr; int cant_use_aperture; unsigned long page_mask; } drm_agp_head_t; typedef struct drm_sg_mem { vm_offset_t vaddr; vm_paddr_t *busaddr; vm_pindex_t pages; } drm_sg_mem_t; #define DRM_MAP_HANDLE_BITS (sizeof(void *) == 4 ? 4 : 24) #define DRM_MAP_HANDLE_SHIFT (sizeof(void *) * 8 - DRM_MAP_HANDLE_BITS) typedef TAILQ_HEAD(drm_map_list, drm_local_map) drm_map_list_t; typedef struct drm_local_map { unsigned long offset; /* Physical address (0 for SAREA) */ unsigned long size; /* Physical size (bytes) */ enum drm_map_type type; /* Type of memory mapped */ enum drm_map_flags flags; /* Flags */ void *handle; /* User-space: "Handle" to pass to mmap */ /* Kernel-space: kernel-virtual address */ int mtrr; /* Boolean: MTRR used */ /* Private data */ int rid; /* PCI resource ID for bus_space */ void *virtual; /* Kernel-space: kernel-virtual address */ struct resource *bsr; bus_space_tag_t bst; bus_space_handle_t bsh; drm_dma_handle_t *dmah; TAILQ_ENTRY(drm_local_map) link; } drm_local_map_t; struct drm_vblank_info { wait_queue_head_t queue; /* vblank wait queue */ atomic_t count; /* number of VBLANK interrupts */ /* (driver must alloc the right number of counters) */ atomic_t refcount; /* number of users of vblank interrupts */ u32 last; /* protected by dev->vbl_lock, used */ /* for wraparound handling */ int enabled; /* so we don't call enable more than */ /* once per disable */ int inmodeset; /* Display driver is setting mode */ }; /* location of GART table */ #define DRM_ATI_GART_MAIN 1 #define DRM_ATI_GART_FB 2 #define DRM_ATI_GART_PCI 1 #define DRM_ATI_GART_PCIE 2 #define DRM_ATI_GART_IGP 3 struct drm_ati_pcigart_info { int gart_table_location; int gart_reg_if; void *addr; dma_addr_t bus_addr; dma_addr_t table_mask; dma_addr_t member_mask; struct drm_dma_handle *table_handle; drm_local_map_t mapping; int table_size; struct drm_dma_handle *dmah; /* handle for ATI PCIGART table */ }; #ifndef DMA_BIT_MASK #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : (1ULL<<(n)) - 1) #endif #define upper_32_bits(n) ((u32)(((n) >> 16) >> 16)) struct drm_driver_info { int (*load)(struct drm_device *, unsigned long flags); int (*firstopen)(struct drm_device *); int (*open)(struct drm_device *, struct drm_file *); void (*preclose)(struct drm_device *, struct drm_file *file_priv); void (*postclose)(struct drm_device *, struct drm_file *); void (*lastclose)(struct drm_device *); int (*unload)(struct drm_device *); void (*reclaim_buffers_locked)(struct drm_device *, struct drm_file *file_priv); int (*dma_ioctl)(struct drm_device *dev, void *data, struct drm_file *file_priv); void (*dma_ready)(struct drm_device *); int (*dma_quiescent)(struct drm_device *); int (*dma_flush_block_and_flush)(struct drm_device *, int context, enum drm_lock_flags flags); int (*dma_flush_unblock)(struct drm_device *, int context, enum drm_lock_flags flags); int (*context_ctor)(struct drm_device *dev, int context); int (*context_dtor)(struct drm_device *dev, int context); int (*kernel_context_switch)(struct drm_device *dev, int old, int new); int (*kernel_context_switch_unlock)(struct drm_device *dev); void (*irq_preinstall)(struct drm_device *dev); int (*irq_postinstall)(struct drm_device *dev); void (*irq_uninstall)(struct drm_device *dev); void (*irq_handler)(DRM_IRQ_ARGS); u32 (*get_vblank_counter)(struct drm_device *dev, int crtc); int (*enable_vblank)(struct drm_device *dev, int crtc); void (*disable_vblank)(struct drm_device *dev, int crtc); drm_pci_id_list_t *id_entry; /* PCI ID, name, and chipset private */ /** * Called by \c drm_device_is_agp. Typically used to determine if a * card is really attached to AGP or not. * * \param dev DRM device handle * * \returns * One of three values is returned depending on whether or not the * card is absolutely \b not AGP (return of 0), absolutely \b is AGP * (return of 1), or may or may not be AGP (return of 2). */ int (*device_is_agp) (struct drm_device * dev); drm_ioctl_desc_t *ioctls; int max_ioctl; int buf_priv_size; int major; int minor; int patchlevel; const char *name; /* Simple driver name */ const char *desc; /* Longer driver name */ const char *date; /* Date of last major changes. */ u32 driver_features; }; /* Length for the array of resource pointers for drm_get_resource_*. */ #define DRM_MAX_PCI_RESOURCE 6 /** * DRM device functions structure */ struct drm_device { struct drm_driver_info *driver; drm_pci_id_list_t *id_entry; /* PCI ID, name, and chipset private */ u_int16_t pci_device; /* PCI device id */ u_int16_t pci_vendor; /* PCI vendor id */ char *unique; /* Unique identifier: e.g., busid */ int unique_len; /* Length of unique field */ device_t device; /* Device instance from newbus */ struct cdev *devnode; /* Device number for mknod */ int if_version; /* Highest interface version set */ int flags; /* Flags to open(2) */ /* Locks */ struct mtx vbl_lock; /* protects vblank operations */ struct mtx dma_lock; /* protects dev->dma */ struct mtx irq_lock; /* protects irq condition checks */ struct mtx dev_lock; /* protects everything else */ DRM_SPINTYPE drw_lock; /* Usage Counters */ int open_count; /* Outstanding files open */ int buf_use; /* Buffers in use -- cannot alloc */ /* Performance counters */ unsigned long counters; enum drm_stat_type types[15]; atomic_t counts[15]; /* Authentication */ drm_file_list_t files; drm_magic_head_t magiclist[DRM_HASH_SIZE]; /* Linked list of mappable regions. Protected by dev_lock */ drm_map_list_t maplist; struct unrhdr *map_unrhdr; drm_local_map_t **context_sareas; int max_context; drm_lock_data_t lock; /* Information on hardware lock */ /* DMA queues (contexts) */ drm_device_dma_t *dma; /* Optional pointer for DMA support */ /* Context support */ int irq; /* Interrupt used by board */ int irq_enabled; /* True if the irq handler is enabled */ int msi_enabled; /* MSI enabled */ int irqrid; /* Interrupt used by board */ struct resource *irqr; /* Resource for interrupt used by board */ void *irqh; /* Handle from bus_setup_intr */ /* Storage of resource pointers for drm_get_resource_* */ struct resource *pcir[DRM_MAX_PCI_RESOURCE]; int pcirid[DRM_MAX_PCI_RESOURCE]; int pci_domain; int pci_bus; int pci_slot; int pci_func; atomic_t context_flag; /* Context swapping flag */ int last_context; /* Last current context */ int vblank_disable_allowed; struct callout vblank_disable_timer; u32 max_vblank_count; /* size of vblank counter register */ struct drm_vblank_info *vblank; /* per crtc vblank info */ int num_crtcs; struct sigio *buf_sigio; /* Processes waiting for SIGIO */ /* Sysctl support */ struct drm_sysctl_info *sysctl; drm_agp_head_t *agp; drm_sg_mem_t *sg; /* Scatter gather memory */ atomic_t *ctx_bitmap; void *dev_private; unsigned int agp_buffer_token; drm_local_map_t *agp_buffer_map; struct unrhdr *drw_unrhdr; /* RB tree of drawable infos */ RB_HEAD(drawable_tree, bsd_drm_drawable_info) drw_head; }; static __inline__ int drm_core_check_feature(struct drm_device *dev, int feature) { return ((dev->driver->driver_features & feature) ? 1 : 0); } #if __OS_HAS_AGP static inline int drm_core_has_AGP(struct drm_device *dev) { return drm_core_check_feature(dev, DRIVER_USE_AGP); } #else #define drm_core_has_AGP(dev) (0) #endif extern int drm_debug_flag; /* Device setup support (drm_drv.c) */ int drm_probe(device_t kdev, drm_pci_id_list_t *idlist); int drm_attach(device_t kdev, drm_pci_id_list_t *idlist); void drm_close(void *data); int drm_detach(device_t kdev); d_ioctl_t drm_ioctl; d_open_t drm_open; d_read_t drm_read; d_poll_t drm_poll; d_mmap_t drm_mmap; extern drm_local_map_t *drm_getsarea(struct drm_device *dev); /* File operations helpers (drm_fops.c) */ extern int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p, struct drm_device *dev); /* Memory management support (drm_memory.c) */ void drm_mem_init(void); void drm_mem_uninit(void); void *drm_ioremap_wc(struct drm_device *dev, drm_local_map_t *map); void *drm_ioremap(struct drm_device *dev, drm_local_map_t *map); void drm_ioremapfree(drm_local_map_t *map); int drm_mtrr_add(unsigned long offset, size_t size, int flags); int drm_mtrr_del(int handle, unsigned long offset, size_t size, int flags); int drm_context_switch(struct drm_device *dev, int old, int new); int drm_context_switch_complete(struct drm_device *dev, int new); int drm_ctxbitmap_init(struct drm_device *dev); void drm_ctxbitmap_cleanup(struct drm_device *dev); void drm_ctxbitmap_free(struct drm_device *dev, int ctx_handle); int drm_ctxbitmap_next(struct drm_device *dev); /* Locking IOCTL support (drm_lock.c) */ int drm_lock_take(struct drm_lock_data *lock_data, unsigned int context); int drm_lock_transfer(struct drm_lock_data *lock_data, unsigned int context); int drm_lock_free(struct drm_lock_data *lock_data, unsigned int context); /* Buffer management support (drm_bufs.c) */ unsigned long drm_get_resource_start(struct drm_device *dev, unsigned int resource); unsigned long drm_get_resource_len(struct drm_device *dev, unsigned int resource); void drm_rmmap(struct drm_device *dev, drm_local_map_t *map); int drm_order(unsigned long size); int drm_addmap(struct drm_device *dev, unsigned long offset, unsigned long size, enum drm_map_type type, enum drm_map_flags flags, drm_local_map_t **map_ptr); int drm_addbufs_pci(struct drm_device *dev, struct drm_buf_desc *request); int drm_addbufs_sg(struct drm_device *dev, struct drm_buf_desc *request); int drm_addbufs_agp(struct drm_device *dev, struct drm_buf_desc *request); /* DMA support (drm_dma.c) */ int drm_dma_setup(struct drm_device *dev); void drm_dma_takedown(struct drm_device *dev); void drm_free_buffer(struct drm_device *dev, drm_buf_t *buf); void drm_reclaim_buffers(struct drm_device *dev, struct drm_file *file_priv); #define drm_core_reclaim_buffers drm_reclaim_buffers /* IRQ support (drm_irq.c) */ int drm_irq_install(struct drm_device *dev); int drm_irq_uninstall(struct drm_device *dev); irqreturn_t drm_irq_handler(DRM_IRQ_ARGS); void drm_driver_irq_preinstall(struct drm_device *dev); void drm_driver_irq_postinstall(struct drm_device *dev); void drm_driver_irq_uninstall(struct drm_device *dev); void drm_handle_vblank(struct drm_device *dev, int crtc); u32 drm_vblank_count(struct drm_device *dev, int crtc); int drm_vblank_get(struct drm_device *dev, int crtc); void drm_vblank_put(struct drm_device *dev, int crtc); void drm_vblank_cleanup(struct drm_device *dev); int drm_vblank_wait(struct drm_device *dev, unsigned int *vbl_seq); int drm_vblank_init(struct drm_device *dev, int num_crtcs); int drm_modeset_ctl(struct drm_device *dev, void *data, struct drm_file *file_priv); /* AGP/PCI Express/GART support (drm_agpsupport.c) */ int drm_device_is_agp(struct drm_device *dev); int drm_device_is_pcie(struct drm_device *dev); drm_agp_head_t *drm_agp_init(void); int drm_agp_acquire(struct drm_device *dev); int drm_agp_release(struct drm_device *dev); int drm_agp_info(struct drm_device * dev, struct drm_agp_info *info); int drm_agp_enable(struct drm_device *dev, struct drm_agp_mode mode); void *drm_agp_allocate_memory(size_t pages, u32 type); int drm_agp_free_memory(void *handle); int drm_agp_bind_memory(void *handle, off_t start); int drm_agp_unbind_memory(void *handle); int drm_agp_alloc(struct drm_device *dev, struct drm_agp_buffer *request); int drm_agp_free(struct drm_device *dev, struct drm_agp_buffer *request); int drm_agp_bind(struct drm_device *dev, struct drm_agp_binding *request); int drm_agp_unbind(struct drm_device *dev, struct drm_agp_binding *request); /* Scatter Gather Support (drm_scatter.c) */ void drm_sg_cleanup(drm_sg_mem_t *entry); int drm_sg_alloc(struct drm_device *dev, struct drm_scatter_gather * request); /* sysctl support (drm_sysctl.h) */ extern int drm_sysctl_init(struct drm_device *dev); extern int drm_sysctl_cleanup(struct drm_device *dev); /* ATI PCIGART support (ati_pcigart.c) */ int drm_ati_pcigart_init(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info); int drm_ati_pcigart_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info); /* Locking IOCTL support (drm_drv.c) */ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_version(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_setversion(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Misc. IOCTL support (drm_ioctl.c) */ int drm_irq_by_busid(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_getunique(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_setunique(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_getmap(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_getclient(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_getstats(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_noop(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Context IOCTL support (drm_context.c) */ int drm_resctx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_addctx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_modctx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_getctx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_switchctx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_newctx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_rmctx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_setsareactx(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_getsareactx(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Drawable IOCTL support (drm_drawable.c) */ int drm_adddraw(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_rmdraw(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_update_draw(struct drm_device *dev, void *data, struct drm_file *file_priv); struct drm_drawable_info *drm_get_drawable_info(struct drm_device *dev, int handle); /* Drawable support (drm_drawable.c) */ void drm_drawable_free_all(struct drm_device *dev); /* Authentication IOCTL support (drm_auth.c) */ int drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_authmagic(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Buffer management support (drm_bufs.c) */ int drm_addmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_rmmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_addbufs(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_infobufs(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_markbufs(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_freebufs(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_mapbufs(struct drm_device *dev, void *data, struct drm_file *file_priv); /* DMA support (drm_dma.c) */ int drm_dma(struct drm_device *dev, void *data, struct drm_file *file_priv); /* IRQ support (drm_irq.c) */ int drm_control(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_priv); /* AGP/GART support (drm_agpsupport.c) */ int drm_agp_acquire_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_agp_release_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_agp_enable_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_agp_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_agp_alloc_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_agp_free_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_agp_unbind_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_agp_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Scatter Gather Support (drm_scatter.c) */ int drm_sg_alloc_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_sg_free(struct drm_device *dev, void *data, struct drm_file *file_priv); /* consistent PCI memory functions (drm_pci.c) */ drm_dma_handle_t *drm_pci_alloc(struct drm_device *dev, size_t size, size_t align, dma_addr_t maxaddr); void drm_pci_free(struct drm_device *dev, drm_dma_handle_t *dmah); /* Inline replacements for drm_alloc and friends */ static __inline__ void * drm_alloc(size_t size, struct malloc_type *area) { return malloc(size, area, M_NOWAIT); } static __inline__ void * drm_calloc(size_t nmemb, size_t size, struct malloc_type *area) { return malloc(size * nmemb, area, M_NOWAIT | M_ZERO); } static __inline__ void * drm_realloc(void *oldpt, size_t oldsize, size_t size, struct malloc_type *area) { return reallocf(oldpt, size, area, M_NOWAIT); } static __inline__ void drm_free(void *pt, size_t size, struct malloc_type *area) { free(pt, area); } /* Inline replacements for DRM_IOREMAP macros */ static __inline__ void drm_core_ioremap_wc(struct drm_local_map *map, struct drm_device *dev) { map->virtual = drm_ioremap_wc(dev, map); } static __inline__ void drm_core_ioremap(struct drm_local_map *map, struct drm_device *dev) { map->virtual = drm_ioremap(dev, map); } static __inline__ void drm_core_ioremapfree(struct drm_local_map *map, struct drm_device *dev) { if ( map->virtual && map->size ) drm_ioremapfree(map); } static __inline__ struct drm_local_map * drm_core_findmap(struct drm_device *dev, unsigned long offset) { drm_local_map_t *map; DRM_SPINLOCK_ASSERT(&dev->dev_lock); TAILQ_FOREACH(map, &dev->maplist, link) { if (offset == (unsigned long)map->handle) return map; } return NULL; } static __inline__ void drm_core_dropmap(struct drm_map *map) { } #endif /* __KERNEL__ */ #endif /* _DRM_P_H_ */ Index: head/sys/dev/drm2/drmP.h =================================================================== --- head/sys/dev/drm2/drmP.h (revision 295880) +++ head/sys/dev/drm2/drmP.h (revision 295881) @@ -1,1806 +1,1805 @@ /** * \file drmP.h * Private header for Direct Rendering Manager * * \author Rickard E. (Rik) Faith * \author Gareth Hughes */ /* * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas. * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. * Copyright (c) 2009-2010, Code Aurora Forum. * All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #ifndef _DRM_P_H_ #define _DRM_P_H_ #if defined(_KERNEL) || defined(__KERNEL__) #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #if defined(__i386__) || defined(__amd64__) #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define __OS_HAS_AGP (defined(CONFIG_AGP) || (defined(CONFIG_AGP_MODULE) && defined(MODULE))) #define __OS_HAS_MTRR (defined(CONFIG_MTRR)) struct drm_file; struct drm_device; #include #include #include "opt_compat.h" #include "opt_drm.h" #include "opt_syscons.h" #ifdef DRM_DEBUG #undef DRM_DEBUG #define DRM_DEBUG_DEFAULT_ON 1 #endif /* DRM_DEBUG */ #define DRM_DEBUGBITS_DEBUG 0x1 #define DRM_DEBUGBITS_KMS 0x2 #define DRM_DEBUGBITS_FAILED_IOCTL 0x4 #undef DRM_LINUX #define DRM_LINUX 0 /***********************************************************************/ /** \name DRM template customization defaults */ /*@{*/ /* driver capabilities and requirements mask */ #define DRIVER_USE_AGP 0x1 #define DRIVER_REQUIRE_AGP 0x2 #define DRIVER_USE_MTRR 0x4 #define DRIVER_PCI_DMA 0x8 #define DRIVER_SG 0x10 #define DRIVER_HAVE_DMA 0x20 #define DRIVER_HAVE_IRQ 0x40 #define DRIVER_IRQ_SHARED 0x80 #define DRIVER_IRQ_VBL 0x100 #define DRIVER_DMA_QUEUE 0x200 #define DRIVER_FB_DMA 0x400 #define DRIVER_IRQ_VBL2 0x800 #define DRIVER_GEM 0x1000 #define DRIVER_MODESET 0x2000 #define DRIVER_PRIME 0x4000 #define DRIVER_BUS_PCI 0x1 #define DRIVER_BUS_PLATFORM 0x2 #define DRIVER_BUS_USB 0x3 /***********************************************************************/ /** \name Begin the DRM... */ /*@{*/ #define DRM_DEBUG_CODE 2 /**< Include debugging code if > 1, then also include looping detection. */ #define DRM_MAGIC_HASH_ORDER 4 /**< Size of key hash table. Must be power of 2. */ #define DRM_KERNEL_CONTEXT 0 /**< Change drm_resctx if changed */ #define DRM_RESERVED_CONTEXTS 1 /**< Change drm_resctx if changed */ #define DRM_LOOPING_LIMIT 5000000 #define DRM_TIME_SLICE (HZ/20) /**< Time slice for GLXContexts */ #define DRM_LOCK_SLICE 1 /**< Time slice for lock, in jiffies */ #define DRM_FLAG_DEBUG 0x01 #define DRM_MAX_CTXBITMAP (PAGE_SIZE * 8) #define DRM_MAP_HASH_OFFSET 0x10000000 /*@}*/ /***********************************************************************/ /** \name Macros to make printk easier */ /*@{*/ /** * Error output. * * \param fmt printf() like format string. * \param arg arguments */ #define DRM_ERROR(fmt, ...) \ printf("error: [" DRM_NAME ":pid%d:%s] *ERROR* " fmt, \ DRM_CURRENTPID, __func__ , ##__VA_ARGS__) #define DRM_WARNING(fmt, ...) printf("warning: [" DRM_NAME "] " fmt , ##__VA_ARGS__) #define DRM_INFO(fmt, ...) printf("info: [" DRM_NAME "] " fmt , ##__VA_ARGS__) /** * Debug output. * * \param fmt printf() like format string. * \param arg arguments */ #define DRM_DEBUG(fmt, ...) do { \ if ((drm_debug & DRM_DEBUGBITS_DEBUG) != 0) \ printf("[" DRM_NAME ":pid%d:%s] " fmt, DRM_CURRENTPID, \ __func__ , ##__VA_ARGS__); \ } while (0) #define DRM_DEBUG_DRIVER(fmt, ...) do { \ if ((drm_debug & DRM_DEBUGBITS_KMS) != 0) \ printf("[" DRM_NAME ":KMS:pid%d:%s] " fmt, DRM_CURRENTPID,\ __func__ , ##__VA_ARGS__); \ } while (0) #define DRM_DEBUG_KMS(fmt, ...) do { \ if ((drm_debug & DRM_DEBUGBITS_KMS) != 0) \ printf("[" DRM_NAME ":KMS:pid%d:%s] " fmt, DRM_CURRENTPID,\ __func__ , ##__VA_ARGS__); \ } while (0) #define DRM_LOG(fmt, ...) do { \ if ((drm_debug & DRM_DEBUGBITS_KMS) != 0) \ printf("[" DRM_NAME "]:pid%d:%s]" fmt, DRM_CURRENTPID, \ __func__ , ##__VA_ARGS__); \ } while (0) #define DRM_LOG_KMS(fmt, ...) do { \ if ((drm_debug & DRM_DEBUGBITS_KMS) != 0) \ printf("[" DRM_NAME "]:KMS:pid%d:%s]" fmt, DRM_CURRENTPID,\ __func__ , ##__VA_ARGS__); \ } while (0) #define DRM_LOG_MODE(fmt, ...) do { \ if ((drm_debug & DRM_DEBUGBITS_KMS) != 0) \ printf("[" DRM_NAME "]:pid%d:%s]" fmt, DRM_CURRENTPID, \ __func__ , ##__VA_ARGS__); \ } while (0) #define DRM_LOG_DRIVER(fmt, ...) do { \ if ((drm_debug & DRM_DEBUGBITS_KMS) != 0) \ printf("[" DRM_NAME "]:KMS:pid%d:%s]" fmt, DRM_CURRENTPID,\ __func__ , ##__VA_ARGS__); \ } while (0) /*@}*/ /***********************************************************************/ /** \name Internal types and structures */ /*@{*/ #define DRM_ARRAY_SIZE(x) ARRAY_SIZE(x) #define DRM_LEFTCOUNT(x) (((x)->rp + (x)->count - (x)->wp) % ((x)->count + 1)) #define DRM_BUFCOUNT(x) ((x)->count - DRM_LEFTCOUNT(x)) #define DRM_IF_VERSION(maj, min) (maj << 16 | min) /** * Test that the hardware lock is held by the caller, returning otherwise. * * \param dev DRM device. * \param filp file pointer of the caller. */ #define LOCK_TEST_WITH_RETURN( dev, _file_priv ) \ do { \ if (!_DRM_LOCK_IS_HELD(_file_priv->master->lock.hw_lock->lock) || \ _file_priv->master->lock.file_priv != _file_priv) { \ DRM_ERROR( "%s called without lock held, held %d owner %p %p\n",\ __func__, _DRM_LOCK_IS_HELD(_file_priv->master->lock.hw_lock->lock),\ _file_priv->master->lock.file_priv, _file_priv); \ return -EINVAL; \ } \ } while (0) /** * Ioctl function type. * * \param inode device inode. * \param file_priv DRM file private pointer. * \param cmd command. * \param arg argument. */ typedef int drm_ioctl_t(struct drm_device *dev, void *data, struct drm_file *file_priv); #define DRM_IOCTL_NR(n) ((n) & 0xff) #define DRM_MAJOR 226 #define DRM_AUTH 0x1 #define DRM_MASTER 0x2 #define DRM_ROOT_ONLY 0x4 #define DRM_CONTROL_ALLOW 0x8 #define DRM_UNLOCKED 0x10 struct drm_ioctl_desc { unsigned long cmd; int flags; drm_ioctl_t *func; unsigned int cmd_drv; }; /** * Creates a driver or general drm_ioctl_desc array entry for the given * ioctl, for use by drm_ioctl(). */ #define DRM_IOCTL_DEF(ioctl, _func, _flags) \ [DRM_IOCTL_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0} #define DRM_IOCTL_DEF_DRV(ioctl, _func, _flags) \ [DRM_IOCTL_NR(DRM_##ioctl)] = {.cmd = DRM_##ioctl, .func = _func, .flags = _flags, .cmd_drv = DRM_IOCTL_##ioctl} struct drm_magic_entry { struct list_head head; struct drm_hash_item hash_item; struct drm_file *priv; }; /** * DMA buffer. */ struct drm_buf { int idx; /**< Index into master buflist */ int total; /**< Buffer size */ int order; /**< log-base-2(total) */ int used; /**< Amount of buffer in use (for DMA) */ unsigned long offset; /**< Byte offset (used internally) */ void *address; /**< Address of buffer */ unsigned long bus_address; /**< Bus address of buffer */ struct drm_buf *next; /**< Kernel-only: used for free list */ __volatile__ int waiting; /**< On kernel DMA queue */ __volatile__ int pending; /**< On hardware DMA queue */ struct drm_file *file_priv; /**< Private of holding file descr */ int context; /**< Kernel queue for this buffer */ int while_locked; /**< Dispatch this buffer while locked */ enum { DRM_LIST_NONE = 0, DRM_LIST_FREE = 1, DRM_LIST_WAIT = 2, DRM_LIST_PEND = 3, DRM_LIST_PRIO = 4, DRM_LIST_RECLAIM = 5 } list; /**< Which list we're on */ int dev_priv_size; /**< Size of buffer private storage */ void *dev_private; /**< Per-buffer private storage */ }; struct drm_freelist { int initialized; /**< Freelist in use */ atomic_t count; /**< Number of free buffers */ struct drm_buf *next; /**< End pointer */ #ifdef FREEBSD_NOTYET wait_queue_head_t waiting; /**< Processes waiting on free bufs */ #endif /* defined(FREEBSD_NOTYET) */ int low_mark; /**< Low water mark */ int high_mark; /**< High water mark */ #ifdef FREEBSD_NOTYET atomic_t wfh; /**< If waiting for high mark */ spinlock_t lock; #endif /* defined(FREEBSD_NOTYET) */ }; typedef struct drm_dma_handle { void *vaddr; bus_addr_t busaddr; bus_dma_tag_t tag; bus_dmamap_t map; } drm_dma_handle_t; /** * Buffer entry. There is one of this for each buffer size order. */ struct drm_buf_entry { int buf_size; /**< size */ int buf_count; /**< number of buffers */ struct drm_buf *buflist; /**< buffer list */ int seg_count; int page_order; struct drm_dma_handle **seglist; struct drm_freelist freelist; }; /* Event queued up for userspace to read */ struct drm_pending_event { struct drm_event *event; struct list_head link; struct drm_file *file_priv; pid_t pid; /* pid of requester, no guarantee it's valid by the time we deliver the event, for tracing only */ void (*destroy)(struct drm_pending_event *event); }; /* initial implementaton using a linked list - todo hashtab */ struct drm_prime_file_private { struct list_head head; struct mtx lock; }; struct drm_file { int authenticated; pid_t pid; uid_t uid; drm_magic_t magic; unsigned long ioctl_count; struct list_head lhead; struct drm_minor *minor; unsigned long lock_count; void *driver_priv; struct drm_gem_names object_names; int is_master; /* this file private is a master for a minor */ struct drm_master *master; /* master this node is currently associated with N.B. not always minor->master */ struct list_head fbs; struct selinfo event_poll; struct list_head event_list; int event_space; struct drm_prime_file_private prime; }; /** * Lock data. */ struct drm_lock_data { struct drm_hw_lock *hw_lock; /**< Hardware lock */ /** Private of lock holder's file (NULL=kernel) */ struct drm_file *file_priv; wait_queue_head_t lock_queue; /**< Queue of blocked processes */ unsigned long lock_time; /**< Time of last lock in jiffies */ struct mtx spinlock; uint32_t kernel_waiters; uint32_t user_waiters; int idle_has_lock; }; /** * DMA data. */ struct drm_device_dma { struct drm_buf_entry bufs[DRM_MAX_ORDER + 1]; /**< buffers, grouped by their size order */ int buf_count; /**< total number of buffers */ struct drm_buf **buflist; /**< Vector of pointers into drm_device_dma::bufs */ int seg_count; int page_count; /**< number of pages */ unsigned long *pagelist; /**< page list */ unsigned long byte_count; enum { _DRM_DMA_USE_AGP = 0x01, _DRM_DMA_USE_SG = 0x02, _DRM_DMA_USE_FB = 0x04, _DRM_DMA_USE_PCI_RO = 0x08 } flags; }; /** * AGP memory entry. Stored as a doubly linked list. */ struct drm_agp_mem { unsigned long handle; /**< handle */ DRM_AGP_MEM *memory; unsigned long bound; /**< address */ int pages; struct list_head head; }; /** * AGP data. * * \sa drm_agp_init() and drm_device::agp. */ struct drm_agp_head { DRM_AGP_KERN agp_info; /**< AGP device information */ struct list_head memory; unsigned long mode; /**< AGP mode */ device_t bridge; int enabled; /**< whether the AGP bus as been enabled */ int acquired; /**< whether the AGP device has been acquired */ unsigned long base; int agp_mtrr; int cant_use_aperture; }; /** * Scatter-gather memory. */ struct drm_sg_mem { vm_offset_t vaddr; vm_paddr_t *busaddr; vm_pindex_t pages; }; struct drm_sigdata { int context; struct drm_hw_lock *lock; }; /** * Kernel side of a mapping */ #define DRM_MAP_HANDLE_BITS (sizeof(void *) == 4 ? 4 : 24) #define DRM_MAP_HANDLE_SHIFT (sizeof(void *) * 8 - DRM_MAP_HANDLE_BITS) struct drm_local_map { resource_size_t offset; /**< Requested physical address (0 for SAREA)*/ unsigned long size; /**< Requested physical size (bytes) */ enum drm_map_type type; /**< Type of memory to map */ enum drm_map_flags flags; /**< Flags */ void *handle; /**< User-space: "Handle" to pass to mmap() */ /**< Kernel-space: kernel-virtual address */ int mtrr; /**< MTRR slot used */ /* Private data */ drm_dma_handle_t *dmah; }; typedef struct drm_local_map drm_local_map_t; /** * Mappings list */ struct drm_map_list { struct list_head head; /**< list head */ struct drm_hash_item hash; struct drm_local_map *map; /**< mapping */ uint64_t user_token; struct drm_master *master; struct drm_mm_node *file_offset_node; /**< fake offset */ }; /** * Context handle list */ struct drm_ctx_list { struct list_head head; /**< list head */ drm_context_t handle; /**< context handle */ struct drm_file *tag; /**< associated fd private data */ }; /* location of GART table */ #define DRM_ATI_GART_MAIN 1 #define DRM_ATI_GART_FB 2 #define DRM_ATI_GART_PCI 1 #define DRM_ATI_GART_PCIE 2 #define DRM_ATI_GART_IGP 3 struct drm_ati_pcigart_info { int gart_table_location; int gart_reg_if; void *addr; dma_addr_t bus_addr; dma_addr_t table_mask; struct drm_dma_handle *table_handle; struct drm_local_map mapping; int table_size; struct drm_dma_handle *dmah; /* handle for ATI PCIGART table FIXME */ }; /** * GEM specific mm private for tracking GEM objects */ struct drm_gem_mm { struct unrhdr *idxunr; struct drm_open_hash offset_hash; /**< User token hash table for maps */ }; /** * This structure defines the drm_mm memory object, which will be used by the * DRM for its buffer objects. */ struct drm_gem_object { /** Reference count of this object */ u_int refcount; /** Handle count of this object. Each handle also holds a reference */ atomic_t handle_count; /* number of handles on this object */ /** Related drm device */ struct drm_device *dev; /** File representing the shmem storage: filp in Linux parlance */ vm_object_t vm_obj; /* Mapping info for this object */ bool on_map; struct drm_hash_item map_list; /** * Size of the object, in bytes. Immutable over the object's * lifetime. */ size_t size; /** * Global name for this object, starts at 1. 0 means unnamed. * Access is covered by the object_name_lock in the related drm_device */ int name; /** * Memory domains. These monitor which caches contain read/write data * related to the object. When transitioning from one set of domains * to another, the driver is called to ensure that caches are suitably * flushed and invalidated */ uint32_t read_domains; uint32_t write_domain; /** * While validating an exec operation, the * new read/write domain values are computed here. * They will be transferred to the above values * at the point that any cache flushing occurs */ uint32_t pending_read_domains; uint32_t pending_write_domain; void *driver_private; #ifdef FREEBSD_NOTYET /* dma buf exported from this GEM object */ struct dma_buf *export_dma_buf; /* dma buf attachment backing this object */ struct dma_buf_attachment *import_attach; #endif /* FREEBSD_NOTYET */ }; #include /* per-master structure */ struct drm_master { u_int refcount; /* refcount for this master */ struct list_head head; /**< each minor contains a list of masters */ struct drm_minor *minor; /**< link back to minor we are a master for */ char *unique; /**< Unique identifier: e.g., busid */ int unique_len; /**< Length of unique field */ int unique_size; /**< amount allocated */ int blocked; /**< Blocked due to VC switch? */ /** \name Authentication */ /*@{ */ struct drm_open_hash magiclist; struct list_head magicfree; /*@} */ struct drm_lock_data lock; /**< Information on hardware lock */ void *driver_priv; /**< Private structure for driver to use */ }; /* Size of ringbuffer for vblank timestamps. Just double-buffer * in initial implementation. */ #define DRM_VBLANKTIME_RBSIZE 2 /* Flags and return codes for get_vblank_timestamp() driver function. */ #define DRM_CALLED_FROM_VBLIRQ 1 #define DRM_VBLANKTIME_SCANOUTPOS_METHOD (1 << 0) #define DRM_VBLANKTIME_INVBL (1 << 1) /* get_scanout_position() return flags */ #define DRM_SCANOUTPOS_VALID (1 << 0) #define DRM_SCANOUTPOS_INVBL (1 << 1) #define DRM_SCANOUTPOS_ACCURATE (1 << 2) struct drm_bus { int bus_type; int (*get_irq)(struct drm_device *dev); void (*free_irq)(struct drm_device *dev); const char *(*get_name)(struct drm_device *dev); int (*set_busid)(struct drm_device *dev, struct drm_master *master); int (*set_unique)(struct drm_device *dev, struct drm_master *master, struct drm_unique *unique); int (*irq_by_busid)(struct drm_device *dev, struct drm_irq_busid *p); /* hooks that are for PCI */ int (*agp_init)(struct drm_device *dev); }; /** * DRM driver structure. This structure represent the common code for * a family of cards. There will one drm_device for each card present * in this family */ struct drm_driver { int (*load) (struct drm_device *, unsigned long flags); int (*firstopen) (struct drm_device *); int (*open) (struct drm_device *, struct drm_file *); void (*preclose) (struct drm_device *, struct drm_file *file_priv); void (*postclose) (struct drm_device *, struct drm_file *); void (*lastclose) (struct drm_device *); int (*unload) (struct drm_device *); int (*dma_ioctl) (struct drm_device *dev, void *data, struct drm_file *file_priv); int (*dma_quiescent) (struct drm_device *); int (*context_dtor) (struct drm_device *dev, int context); /** * get_vblank_counter - get raw hardware vblank counter * @dev: DRM device * @crtc: counter to fetch * * Driver callback for fetching a raw hardware vblank counter for @crtc. * If a device doesn't have a hardware counter, the driver can simply * return the value of drm_vblank_count. The DRM core will account for * missed vblank events while interrupts where disabled based on system * timestamps. * * Wraparound handling and loss of events due to modesetting is dealt * with in the DRM core code. * * RETURNS * Raw vblank counter value. */ u32 (*get_vblank_counter) (struct drm_device *dev, int crtc); /** * enable_vblank - enable vblank interrupt events * @dev: DRM device * @crtc: which irq to enable * * Enable vblank interrupts for @crtc. If the device doesn't have * a hardware vblank counter, this routine should be a no-op, since * interrupts will have to stay on to keep the count accurate. * * RETURNS * Zero on success, appropriate errno if the given @crtc's vblank * interrupt cannot be enabled. */ int (*enable_vblank) (struct drm_device *dev, int crtc); /** * disable_vblank - disable vblank interrupt events * @dev: DRM device * @crtc: which irq to enable * * Disable vblank interrupts for @crtc. If the device doesn't have * a hardware vblank counter, this routine should be a no-op, since * interrupts will have to stay on to keep the count accurate. */ void (*disable_vblank) (struct drm_device *dev, int crtc); /** * Called by \c drm_device_is_agp. Typically used to determine if a * card is really attached to AGP or not. * * \param dev DRM device handle * * \returns * One of three values is returned depending on whether or not the * card is absolutely \b not AGP (return of 0), absolutely \b is AGP * (return of 1), or may or may not be AGP (return of 2). */ int (*device_is_agp) (struct drm_device *dev); /** * Called by vblank timestamping code. * * Return the current display scanout position from a crtc. * * \param dev DRM device. * \param crtc Id of the crtc to query. * \param *vpos Target location for current vertical scanout position. * \param *hpos Target location for current horizontal scanout position. * * Returns vpos as a positive number while in active scanout area. * Returns vpos as a negative number inside vblank, counting the number * of scanlines to go until end of vblank, e.g., -1 means "one scanline * until start of active scanout / end of vblank." * * \return Flags, or'ed together as follows: * * DRM_SCANOUTPOS_VALID = Query successful. * DRM_SCANOUTPOS_INVBL = Inside vblank. * DRM_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of * this flag means that returned position may be offset by a constant * but unknown small number of scanlines wrt. real scanout position. * */ int (*get_scanout_position) (struct drm_device *dev, int crtc, int *vpos, int *hpos); /** * Called by \c drm_get_last_vbltimestamp. Should return a precise * timestamp when the most recent VBLANK interval ended or will end. * * Specifically, the timestamp in @vblank_time should correspond as * closely as possible to the time when the first video scanline of * the video frame after the end of VBLANK will start scanning out, * the time immediately after end of the VBLANK interval. If the * @crtc is currently inside VBLANK, this will be a time in the future. * If the @crtc is currently scanning out a frame, this will be the * past start time of the current scanout. This is meant to adhere * to the OpenML OML_sync_control extension specification. * * \param dev dev DRM device handle. * \param crtc crtc for which timestamp should be returned. * \param *max_error Maximum allowable timestamp error in nanoseconds. * Implementation should strive to provide timestamp * with an error of at most *max_error nanoseconds. * Returns true upper bound on error for timestamp. * \param *vblank_time Target location for returned vblank timestamp. * \param flags 0 = Defaults, no special treatment needed. * \param DRM_CALLED_FROM_VBLIRQ = Function is called from vblank * irq handler. Some drivers need to apply some workarounds * for gpu-specific vblank irq quirks if flag is set. * * \returns * Zero if timestamping isn't supported in current display mode or a * negative number on failure. A positive status code on success, * which describes how the vblank_time timestamp was computed. */ int (*get_vblank_timestamp) (struct drm_device *dev, int crtc, int *max_error, struct timeval *vblank_time, unsigned flags); /* these have to be filled in */ irqreturn_t(*irq_handler) (DRM_IRQ_ARGS); void (*irq_preinstall) (struct drm_device *dev); int (*irq_postinstall) (struct drm_device *dev); void (*irq_uninstall) (struct drm_device *dev); void (*set_version) (struct drm_device *dev, struct drm_set_version *sv); /* Master routines */ int (*master_create)(struct drm_device *dev, struct drm_master *master); void (*master_destroy)(struct drm_device *dev, struct drm_master *master); /** * master_set is called whenever the minor master is set. * master_drop is called whenever the minor master is dropped. */ int (*master_set)(struct drm_device *dev, struct drm_file *file_priv, bool from_open); void (*master_drop)(struct drm_device *dev, struct drm_file *file_priv, bool from_release); /** * Driver-specific constructor for drm_gem_objects, to set up * obj->driver_private. * * Returns 0 on success. */ int (*gem_init_object) (struct drm_gem_object *obj); void (*gem_free_object) (struct drm_gem_object *obj); int (*gem_open_object) (struct drm_gem_object *, struct drm_file *); void (*gem_close_object) (struct drm_gem_object *, struct drm_file *); #ifdef FREEBSD_NOTYET /* prime: */ /* export handle -> fd (see drm_gem_prime_handle_to_fd() helper) */ int (*prime_handle_to_fd)(struct drm_device *dev, struct drm_file *file_priv, uint32_t handle, uint32_t flags, int *prime_fd); /* import fd -> handle (see drm_gem_prime_fd_to_handle() helper) */ int (*prime_fd_to_handle)(struct drm_device *dev, struct drm_file *file_priv, int prime_fd, uint32_t *handle); /* export GEM -> dmabuf */ struct dma_buf * (*gem_prime_export)(struct drm_device *dev, struct drm_gem_object *obj, int flags); /* import dmabuf -> GEM */ struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev, struct dma_buf *dma_buf); #endif /* defined(FREEBSD_NOTYET) */ /* dumb alloc support */ int (*dumb_create)(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); int (*dumb_map_offset)(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle, uint64_t *offset); int (*dumb_destroy)(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle); /* Driver private ops for this object */ struct cdev_pager_ops *gem_pager_ops; int (*sysctl_init)(struct drm_device *dev, struct sysctl_ctx_list *ctx, struct sysctl_oid *top); void (*sysctl_cleanup)(struct drm_device *dev); int major; int minor; int patchlevel; char *name; char *desc; char *date; u32 driver_features; int dev_priv_size; struct drm_ioctl_desc *ioctls; int num_ioctls; struct drm_bus *bus; #ifdef COMPAT_FREEBSD32 struct drm_ioctl_desc *compat_ioctls; int *num_compat_ioctls; #endif int buf_priv_size; }; #define DRM_MINOR_UNASSIGNED 0 #define DRM_MINOR_LEGACY 1 #define DRM_MINOR_CONTROL 2 #define DRM_MINOR_RENDER 3 /** * DRM minor structure. This structure represents a drm minor number. */ struct drm_minor { int index; /**< Minor device number */ int type; /**< Control or render */ struct cdev *device; /**< Device number for mknod */ device_t kdev; /**< OS device */ struct drm_device *dev; struct drm_master *master; /* currently active master for this node */ struct list_head master_list; struct drm_mode_group mode_group; struct sigio *buf_sigio; /* Processes waiting for SIGIO */ }; /* mode specified on the command line */ struct drm_cmdline_mode { bool specified; bool refresh_specified; bool bpp_specified; int xres, yres; int bpp; int refresh; bool rb; bool interlace; bool cvt; bool margins; enum drm_connector_force force; }; struct drm_pending_vblank_event { struct drm_pending_event base; int pipe; struct drm_event_vblank event; }; /** * DRM device structure. This structure represent a complete card that * may contain multiple heads. */ struct drm_device { int if_version; /**< Highest interface version set */ /** \name Locks */ /*@{ */ struct mtx count_lock; /**< For inuse, drm_device::open_count, drm_device::buf_use */ struct sx dev_struct_lock; /**< For others */ /*@} */ /** \name Usage Counters */ /*@{ */ int open_count; /**< Outstanding files open */ atomic_t ioctl_count; /**< Outstanding IOCTLs pending */ atomic_t vma_count; /**< Outstanding vma areas open */ int buf_use; /**< Buffers in use -- cannot alloc */ atomic_t buf_alloc; /**< Buffer allocation in progress */ /*@} */ /** \name Performance counters */ /*@{ */ unsigned long counters; enum drm_stat_type types[15]; atomic_t counts[15]; /*@} */ struct list_head filelist; /** \name Memory management */ /*@{ */ struct list_head maplist; /**< Linked list of regions */ int map_count; /**< Number of mappable regions */ struct drm_open_hash map_hash; /**< User token hash table for maps */ /** \name Context handle management */ /*@{ */ struct list_head ctxlist; /**< Linked list of context handles */ int ctx_count; /**< Number of context handles */ struct mtx ctxlist_mutex; /**< For ctxlist */ drm_local_map_t **context_sareas; int max_context; unsigned long *ctx_bitmap; /*@} */ /** \name DMA support */ /*@{ */ struct drm_device_dma *dma; /**< Optional pointer for DMA support */ /*@} */ /** \name Context support */ /*@{ */ int irq_enabled; /**< True if irq handler is enabled */ atomic_t context_flag; /**< Context swapping flag */ atomic_t interrupt_flag; /**< Interruption handler flag */ atomic_t dma_flag; /**< DMA dispatch flag */ wait_queue_head_t context_wait; /**< Processes waiting on ctx switch */ int last_checked; /**< Last context checked for DMA */ int last_context; /**< Last current context */ unsigned long last_switch; /**< jiffies at last context switch */ /*@} */ /** \name VBLANK IRQ support */ /*@{ */ /* * At load time, disabling the vblank interrupt won't be allowed since * old clients may not call the modeset ioctl and therefore misbehave. * Once the modeset ioctl *has* been called though, we can safely * disable them when unused. */ int vblank_disable_allowed; atomic_t *_vblank_count; /**< number of VBLANK interrupts (driver must alloc the right number of counters) */ struct timeval *_vblank_time; /**< timestamp of current vblank_count (drivers must alloc right number of fields) */ struct mtx vblank_time_lock; /**< Protects vblank count and time updates during vblank enable/disable */ struct mtx vbl_lock; atomic_t *vblank_refcount; /* number of users of vblank interruptsper crtc */ u32 *last_vblank; /* protected by dev->vbl_lock, used */ /* for wraparound handling */ int *vblank_enabled; /* so we don't call enable more than once per disable */ int *vblank_inmodeset; /* Display driver is setting mode */ u32 *last_vblank_wait; /* Last vblank seqno waited per CRTC */ struct callout vblank_disable_callout; u32 max_vblank_count; /**< size of vblank counter register */ /** * List of events */ struct list_head vblank_event_list; struct mtx event_lock; /*@} */ struct drm_agp_head *agp; /**< AGP data */ device_t dev; /* Device instance from newbus */ uint16_t pci_device; /* PCI device id */ uint16_t pci_vendor; /* PCI vendor id */ uint16_t pci_subdevice; /* PCI subsystem device id */ uint16_t pci_subvendor; /* PCI subsystem vendor id */ struct drm_sg_mem *sg; /**< Scatter gather memory */ unsigned int num_crtcs; /**< Number of CRTCs on this device */ void *dev_private; /**< device private data */ void *mm_private; struct drm_sigdata sigdata; /**< For block_all_signals */ sigset_t sigmask; struct drm_driver *driver; struct drm_local_map *agp_buffer_map; unsigned int agp_buffer_token; struct drm_minor *control; /**< Control node for card */ struct drm_minor *primary; /**< render type primary screen head */ struct drm_mode_config mode_config; /**< Current mode config */ /** \name GEM information */ /*@{ */ struct sx object_name_lock; struct drm_gem_names object_names; /*@} */ int switch_power_state; atomic_t unplugged; /* device has been unplugged or gone away */ /* Locks */ struct mtx dma_lock; /* protects dev->dma */ struct mtx irq_lock; /* protects irq condition checks */ /* Context support */ int irq; /* Interrupt used by board */ int msi_enabled; /* MSI enabled */ int irqrid; /* Interrupt used by board */ struct resource *irqr; /* Resource for interrupt used by board */ void *irqh; /* Handle from bus_setup_intr */ /* Storage of resource pointers for drm_get_resource_* */ #define DRM_MAX_PCI_RESOURCE 6 struct resource *pcir[DRM_MAX_PCI_RESOURCE]; int pcirid[DRM_MAX_PCI_RESOURCE]; struct mtx pcir_lock; int pci_domain; int pci_bus; int pci_slot; int pci_func; /* Sysctl support */ struct drm_sysctl_info *sysctl; int sysctl_node_idx; void *drm_ttm_bdev; void *sysctl_private; char busid_str[128]; int modesetting; drm_pci_id_list_t *id_entry; /* PCI ID, name, and chipset private */ }; #define DRM_SWITCH_POWER_ON 0 #define DRM_SWITCH_POWER_OFF 1 #define DRM_SWITCH_POWER_CHANGING 2 static __inline__ int drm_core_check_feature(struct drm_device *dev, int feature) { return ((dev->driver->driver_features & feature) ? 1 : 0); } static inline int drm_dev_to_irq(struct drm_device *dev) { return dev->driver->bus->get_irq(dev); } #if __OS_HAS_AGP static inline int drm_core_has_AGP(struct drm_device *dev) { return drm_core_check_feature(dev, DRIVER_USE_AGP); } #else #define drm_core_has_AGP(dev) (0) #endif #if __OS_HAS_MTRR static inline int drm_core_has_MTRR(struct drm_device *dev) { return drm_core_check_feature(dev, DRIVER_USE_MTRR); } #define DRM_MTRR_WC MDF_WRITECOMBINE int drm_mtrr_add(unsigned long offset, unsigned long size, unsigned int flags); int drm_mtrr_del(int handle, unsigned long offset, unsigned long size, unsigned int flags); #else #define drm_core_has_MTRR(dev) (0) #define DRM_MTRR_WC 0 static inline int drm_mtrr_add(unsigned long offset, unsigned long size, unsigned int flags) { return 0; } static inline int drm_mtrr_del(int handle, unsigned long offset, unsigned long size, unsigned int flags) { return 0; } #endif /******************************************************************/ /** \name Internal function definitions */ /*@{*/ /* Driver support (drm_drv.h) */ d_ioctl_t drm_ioctl; extern int drm_lastclose(struct drm_device *dev); /* Device support (drm_fops.h) */ extern struct sx drm_global_mutex; d_open_t drm_open; d_read_t drm_read; extern void drm_release(void *data); /* Mapping support (drm_vm.h) */ d_mmap_t drm_mmap; int drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot); d_poll_t drm_poll; /* Memory management support (drm_memory.h) */ extern void drm_free_agp(DRM_AGP_MEM * handle, int pages); extern int drm_bind_agp(DRM_AGP_MEM * handle, unsigned int start); #ifdef FREEBSD_NOTYET extern DRM_AGP_MEM *drm_agp_bind_pages(struct drm_device *dev, struct page **pages, unsigned long num_pages, uint32_t gtt_offset, uint32_t type); #endif /* FREEBSD_NOTYET */ extern int drm_unbind_agp(DRM_AGP_MEM * handle); /* Misc. IOCTL support (drm_ioctl.h) */ extern int drm_irq_by_busid(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_getunique(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_setunique(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_getmap(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_getclient(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_getstats(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_setversion(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_noop(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Context IOCTL support (drm_context.h) */ extern int drm_resctx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_addctx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_modctx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_getctx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_switchctx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_newctx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_rmctx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_ctxbitmap_init(struct drm_device *dev); extern void drm_ctxbitmap_cleanup(struct drm_device *dev); extern void drm_ctxbitmap_free(struct drm_device *dev, int ctx_handle); extern int drm_setsareactx(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_getsareactx(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Authentication IOCTL support (drm_auth.h) */ extern int drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_authmagic(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_remove_magic(struct drm_master *master, drm_magic_t magic); /* Cache management (drm_cache.c) */ void drm_clflush_pages(vm_page_t *pages, unsigned long num_pages); void drm_clflush_virt_range(char *addr, unsigned long length); /* Locking IOCTL support (drm_lock.h) */ extern int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_lock_free(struct drm_lock_data *lock_data, unsigned int context); extern void drm_idlelock_take(struct drm_lock_data *lock_data); extern void drm_idlelock_release(struct drm_lock_data *lock_data); /* * These are exported to drivers so that they can implement fencing using * DMA quiscent + idle. DMA quiescent usually requires the hardware lock. */ extern int drm_i_have_hw_lock(struct drm_device *dev, struct drm_file *file_priv); /* Buffer management support (drm_bufs.h) */ extern int drm_addbufs_agp(struct drm_device *dev, struct drm_buf_desc * request); extern int drm_addbufs_pci(struct drm_device *dev, struct drm_buf_desc * request); extern int drm_addmap(struct drm_device *dev, resource_size_t offset, unsigned int size, enum drm_map_type type, enum drm_map_flags flags, struct drm_local_map **map_ptr); extern int drm_addmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_rmmap(struct drm_device *dev, struct drm_local_map *map); extern int drm_rmmap_locked(struct drm_device *dev, struct drm_local_map *map); extern int drm_rmmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_addbufs(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_infobufs(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_markbufs(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_freebufs(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_mapbufs(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_order(unsigned long size); /* DMA support (drm_dma.h) */ extern int drm_dma_setup(struct drm_device *dev); extern void drm_dma_takedown(struct drm_device *dev); extern void drm_free_buffer(struct drm_device *dev, struct drm_buf * buf); extern void drm_core_reclaim_buffers(struct drm_device *dev, struct drm_file *filp); /* IRQ support (drm_irq.h) */ extern int drm_control(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_irq_install(struct drm_device *dev); extern int drm_irq_uninstall(struct drm_device *dev); extern int drm_vblank_init(struct drm_device *dev, int num_crtcs); extern int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *filp); extern int drm_vblank_wait(struct drm_device *dev, unsigned int *vbl_seq); extern u32 drm_vblank_count(struct drm_device *dev, int crtc); extern u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc, struct timeval *vblanktime); extern void drm_send_vblank_event(struct drm_device *dev, int crtc, struct drm_pending_vblank_event *e); extern bool drm_handle_vblank(struct drm_device *dev, int crtc); extern int drm_vblank_get(struct drm_device *dev, int crtc); extern void drm_vblank_put(struct drm_device *dev, int crtc); extern void drm_vblank_off(struct drm_device *dev, int crtc); extern void drm_vblank_cleanup(struct drm_device *dev); extern u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc, struct timeval *tvblank, unsigned flags); extern int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, int *max_error, struct timeval *vblank_time, unsigned flags, struct drm_crtc *refcrtc); extern void drm_calc_timestamping_constants(struct drm_crtc *crtc); extern bool drm_mode_parse_command_line_for_connector(const char *mode_option, struct drm_connector *connector, struct drm_cmdline_mode *mode); extern struct drm_display_mode * drm_mode_create_from_cmdline_mode(struct drm_device *dev, struct drm_cmdline_mode *cmd); /* Modesetting support */ extern void drm_vblank_pre_modeset(struct drm_device *dev, int crtc); extern void drm_vblank_post_modeset(struct drm_device *dev, int crtc); extern int drm_modeset_ctl(struct drm_device *dev, void *data, struct drm_file *file_priv); /* AGP/GART support (drm_agpsupport.h) */ extern struct drm_agp_head *drm_agp_init(struct drm_device *dev); extern int drm_agp_acquire(struct drm_device *dev); extern int drm_agp_acquire_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_agp_release(struct drm_device *dev); extern int drm_agp_release_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_agp_enable(struct drm_device *dev, struct drm_agp_mode mode); extern int drm_agp_enable_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_agp_info(struct drm_device *dev, struct drm_agp_info *info); extern int drm_agp_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_agp_alloc(struct drm_device *dev, struct drm_agp_buffer *request); extern int drm_agp_alloc_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_agp_free(struct drm_device *dev, struct drm_agp_buffer *request); extern int drm_agp_free_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_agp_unbind(struct drm_device *dev, struct drm_agp_binding *request); extern int drm_agp_unbind_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_agp_bind(struct drm_device *dev, struct drm_agp_binding *request); extern int drm_agp_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); /* Stub support (drm_stub.h) */ extern int drm_setmaster_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_dropmaster_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); struct drm_master *drm_master_create(struct drm_minor *minor); extern struct drm_master *drm_master_get(struct drm_master *master); extern void drm_master_put(struct drm_master **master); extern void drm_put_dev(struct drm_device *dev); extern int drm_put_minor(struct drm_minor **minor); extern void drm_unplug_dev(struct drm_device *dev); extern unsigned int drm_debug; extern unsigned int drm_notyet; extern unsigned int drm_vblank_offdelay; extern unsigned int drm_timestamp_precision; extern unsigned int drm_timestamp_monotonic; extern struct drm_local_map *drm_getsarea(struct drm_device *dev); #ifdef FREEBSD_NOTYET extern int drm_gem_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file_priv, uint32_t handle, uint32_t flags, int *prime_fd); extern int drm_gem_prime_fd_to_handle(struct drm_device *dev, struct drm_file *file_priv, int prime_fd, uint32_t *handle); extern int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, vm_page_t *pages, dma_addr_t *addrs, int max_pages); extern struct sg_table *drm_prime_pages_to_sg(vm_page_t *pages, int nr_pages); extern void drm_prime_gem_destroy(struct drm_gem_object *obj, struct sg_table *sg); void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv); void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv); int drm_prime_add_imported_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t handle); int drm_prime_lookup_imported_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t *handle); void drm_prime_remove_imported_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf); int drm_prime_add_dma_buf(struct drm_device *dev, struct drm_gem_object *obj); int drm_prime_lookup_obj(struct drm_device *dev, struct dma_buf *buf, struct drm_gem_object **obj); #endif /* FREEBSD_NOTYET */ /* Scatter Gather Support (drm_scatter.h) */ extern void drm_sg_cleanup(struct drm_sg_mem * entry); extern int drm_sg_alloc_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int drm_sg_alloc(struct drm_device *dev, struct drm_scatter_gather * request); extern int drm_sg_free(struct drm_device *dev, void *data, struct drm_file *file_priv); /* ATI PCIGART support (ati_pcigart.h) */ extern int drm_ati_pcigart_init(struct drm_device *dev, struct drm_ati_pcigart_info * gart_info); extern int drm_ati_pcigart_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info * gart_info); extern drm_dma_handle_t *drm_pci_alloc(struct drm_device *dev, size_t size, size_t align, dma_addr_t maxaddr); extern void __drm_pci_free(struct drm_device *dev, drm_dma_handle_t * dmah); extern void drm_pci_free(struct drm_device *dev, drm_dma_handle_t * dmah); /* Graphics Execution Manager library functions (drm_gem.c) */ int drm_gem_init(struct drm_device *dev); void drm_gem_destroy(struct drm_device *dev); void drm_gem_object_release(struct drm_gem_object *obj); void drm_gem_object_free(struct drm_gem_object *obj); struct drm_gem_object *drm_gem_object_alloc(struct drm_device *dev, size_t size); int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj, size_t size); int drm_gem_private_object_init(struct drm_device *dev, struct drm_gem_object *obj, size_t size); void drm_gem_object_handle_free(struct drm_gem_object *obj); int drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot); void drm_gem_pager_dtr(void *obj); #include static inline void drm_gem_object_reference(struct drm_gem_object *obj) { KASSERT(obj->refcount > 0, ("Dangling obj %p", obj)); refcount_acquire(&obj->refcount); } static inline void drm_gem_object_unreference(struct drm_gem_object *obj) { if (obj == NULL) return; if (refcount_release(&obj->refcount)) drm_gem_object_free(obj); } static inline void drm_gem_object_unreference_unlocked(struct drm_gem_object *obj) { if (obj != NULL) { struct drm_device *dev = obj->dev; DRM_LOCK(dev); drm_gem_object_unreference(obj); DRM_UNLOCK(dev); } } int drm_gem_handle_create(struct drm_file *file_priv, struct drm_gem_object *obj, u32 *handlep); int drm_gem_handle_delete(struct drm_file *filp, u32 handle); static inline void drm_gem_object_handle_reference(struct drm_gem_object *obj) { drm_gem_object_reference(obj); atomic_inc(&obj->handle_count); } static inline void drm_gem_object_handle_unreference(struct drm_gem_object *obj) { if (obj == NULL) return; if (atomic_read(&obj->handle_count) == 0) return; /* * Must bump handle count first as this may be the last * ref, in which case the object would disappear before we * checked for a name */ if (atomic_dec_and_test(&obj->handle_count)) drm_gem_object_handle_free(obj); drm_gem_object_unreference(obj); } static inline void drm_gem_object_handle_unreference_unlocked(struct drm_gem_object *obj) { if (obj == NULL) return; if (atomic_read(&obj->handle_count) == 0) return; /* * Must bump handle count first as this may be the last * ref, in which case the object would disappear before we * checked for a name */ if (atomic_dec_and_test(&obj->handle_count)) drm_gem_object_handle_free(obj); drm_gem_object_unreference_unlocked(obj); } void drm_gem_free_mmap_offset(struct drm_gem_object *obj); int drm_gem_create_mmap_offset(struct drm_gem_object *obj); struct drm_gem_object *drm_gem_object_lookup(struct drm_device *dev, struct drm_file *filp, u32 handle); int drm_gem_close_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_gem_flink_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int drm_gem_open_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); void drm_gem_open(struct drm_device *dev, struct drm_file *file_private); void drm_gem_release(struct drm_device *dev, struct drm_file *file_private); extern void drm_core_ioremap(struct drm_local_map *map, struct drm_device *dev); extern void drm_core_ioremap_wc(struct drm_local_map *map, struct drm_device *dev); extern void drm_core_ioremapfree(struct drm_local_map *map, struct drm_device *dev); static __inline__ struct drm_local_map *drm_core_findmap(struct drm_device *dev, unsigned int token) { struct drm_map_list *_entry; list_for_each_entry(_entry, &dev->maplist, head) if (_entry->user_token == token) return _entry->map; return NULL; } static __inline__ void drm_core_dropmap(struct drm_local_map *map) { } extern int drm_fill_in_dev(struct drm_device *dev, struct drm_driver *driver); extern void drm_cancel_fill_in_dev(struct drm_device *dev); int drm_get_minor(struct drm_device *dev, struct drm_minor **minor, int type); /*@}*/ /* PCI section */ int drm_pci_device_is_agp(struct drm_device *dev); int drm_pci_device_is_pcie(struct drm_device *dev); extern int drm_get_pci_dev(device_t kdev, struct drm_device *dev, struct drm_driver *driver); #define DRM_PCIE_SPEED_25 1 #define DRM_PCIE_SPEED_50 2 #define DRM_PCIE_SPEED_80 4 extern int drm_pcie_get_speed_cap_mask(struct drm_device *dev, u32 *speed_mask); #define drm_can_sleep() (DRM_HZ & 1) /* FreeBSD specific -- should be moved to drm_os_freebsd.h */ #define DRM_GEM_MAPPING_MASK (3ULL << 62) #define DRM_GEM_MAPPING_KEY (2ULL << 62) /* Non-canonical address form */ #define DRM_GEM_MAX_IDX 0x3fffff #define DRM_GEM_MAPPING_IDX(o) (((o) >> 40) & DRM_GEM_MAX_IDX) #define DRM_GEM_MAPPING_OFF(i) (((uint64_t)(i)) << 40) #define DRM_GEM_MAPPING_MAPOFF(o) \ ((o) & ~(DRM_GEM_MAPPING_OFF(DRM_GEM_MAX_IDX) | DRM_GEM_MAPPING_KEY)) SYSCTL_DECL(_hw_drm); #define DRM_DEV_MODE (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP) #define DRM_DEV_UID UID_ROOT #define DRM_DEV_GID GID_VIDEO #define DRM_WAKEUP(w) wakeup((void *)w) #define DRM_WAKEUP_INT(w) wakeup(w) #define DRM_INIT_WAITQUEUE(queue) do {(void)(queue);} while (0) #define DRM_CURPROC curthread #define DRM_STRUCTPROC struct thread #define DRM_SPINTYPE struct mtx #define DRM_SPININIT(l,name) mtx_init(l, name, NULL, MTX_DEF) #define DRM_SPINUNINIT(l) mtx_destroy(l) #define DRM_SPINLOCK(l) mtx_lock(l) #define DRM_SPINUNLOCK(u) mtx_unlock(u) #define DRM_SPINLOCK_IRQSAVE(l, irqflags) do { \ mtx_lock(l); \ (void)irqflags; \ } while (0) #define DRM_SPINUNLOCK_IRQRESTORE(u, irqflags) mtx_unlock(u) #define DRM_SPINLOCK_ASSERT(l) mtx_assert(l, MA_OWNED) #define DRM_LOCK_SLEEP(dev, chan, flags, msg, timeout) \ (sx_sleep((chan), &(dev)->dev_struct_lock, (flags), (msg), (timeout))) #if defined(INVARIANTS) #define DRM_LOCK_ASSERT(dev) sx_assert(&(dev)->dev_struct_lock, SA_XLOCKED) #define DRM_UNLOCK_ASSERT(dev) sx_assert(&(dev)->dev_struct_lock, SA_UNLOCKED) #else #define DRM_LOCK_ASSERT(d) #define DRM_UNLOCK_ASSERT(d) #endif #define DRM_SYSCTL_HANDLER_ARGS (SYSCTL_HANDLER_ARGS) enum { DRM_IS_NOT_AGP, DRM_IS_AGP, DRM_MIGHT_BE_AGP }; #define DRM_VERIFYAREA_READ( uaddr, size ) \ (!useracc(__DECONST(caddr_t, uaddr), size, VM_PROT_READ)) #define DRM_COPY_TO_USER(user, kern, size) \ copyout(kern, user, size) #define DRM_COPY_FROM_USER(kern, user, size) \ copyin(user, kern, size) #define DRM_COPY_FROM_USER_UNCHECKED(arg1, arg2, arg3) \ copyin(arg2, arg1, arg3) #define DRM_COPY_TO_USER_UNCHECKED(arg1, arg2, arg3) \ copyout(arg2, arg1, arg3) #define DRM_GET_USER_UNCHECKED(val, uaddr) \ ((val) = fuword32(uaddr), 0) #define DRM_GET_PRIV_SAREA(_dev, _ctx, _map) do { \ (_map) = (_dev)->context_sareas[_ctx]; \ } while(0) /* Returns -errno to shared code */ #define DRM_WAIT_ON( ret, queue, timeout, condition ) \ for ( ret = 0 ; !ret && !(condition) ; ) { \ DRM_UNLOCK(dev); \ mtx_lock(&dev->irq_lock); \ if (!(condition)) \ ret = -mtx_sleep(&(queue), &dev->irq_lock, \ PCATCH, "drmwtq", (timeout)); \ if (ret == -ERESTART) \ ret = -ERESTARTSYS; \ mtx_unlock(&dev->irq_lock); \ DRM_LOCK(dev); \ } #define dev_err(dev, fmt, ...) \ device_printf((dev), "error: " fmt, ## __VA_ARGS__) #define dev_warn(dev, fmt, ...) \ device_printf((dev), "warning: " fmt, ## __VA_ARGS__) #define dev_info(dev, fmt, ...) \ device_printf((dev), "info: " fmt, ## __VA_ARGS__) #define dev_dbg(dev, fmt, ...) do { \ if ((drm_debug& DRM_DEBUGBITS_KMS) != 0) { \ device_printf((dev), "debug: " fmt, ## __VA_ARGS__); \ } \ } while (0) struct drm_msi_blacklist_entry { int vendor; int device; }; struct drm_vblank_info { wait_queue_head_t queue; /* vblank wait queue */ atomic_t count; /* number of VBLANK interrupts */ /* (driver must alloc the right number of counters) */ atomic_t refcount; /* number of users of vblank interrupts */ u32 last; /* protected by dev->vbl_lock, used */ /* for wraparound handling */ int enabled; /* so we don't call enable more than */ /* once per disable */ int inmodeset; /* Display driver is setting mode */ }; #ifndef DMA_BIT_MASK #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : (1ULL<<(n)) - 1) #endif #define upper_32_bits(n) ((u32)(((n) >> 16) >> 16)) enum dmi_field { DMI_NONE, DMI_BIOS_VENDOR, DMI_BIOS_VERSION, DMI_BIOS_DATE, DMI_SYS_VENDOR, DMI_PRODUCT_NAME, DMI_PRODUCT_VERSION, DMI_PRODUCT_SERIAL, DMI_PRODUCT_UUID, DMI_BOARD_VENDOR, DMI_BOARD_NAME, DMI_BOARD_VERSION, DMI_BOARD_SERIAL, DMI_BOARD_ASSET_TAG, DMI_CHASSIS_VENDOR, DMI_CHASSIS_TYPE, DMI_CHASSIS_VERSION, DMI_CHASSIS_SERIAL, DMI_CHASSIS_ASSET_TAG, DMI_STRING_MAX, }; struct dmi_strmatch { unsigned char slot; char substr[79]; }; struct dmi_system_id { int (*callback)(const struct dmi_system_id *); const char *ident; struct dmi_strmatch matches[4]; }; #define DMI_MATCH(a, b) {(a), (b)} bool dmi_check_system(const struct dmi_system_id *); /* Device setup support (drm_drv.c) */ int drm_probe_helper(device_t kdev, drm_pci_id_list_t *idlist); int drm_attach_helper(device_t kdev, drm_pci_id_list_t *idlist, struct drm_driver *driver); int drm_generic_detach(device_t kdev); void drm_event_wakeup(struct drm_pending_event *e); int drm_add_busid_modesetting(struct drm_device *dev, struct sysctl_ctx_list *ctx, struct sysctl_oid *top); /* Buffer management support (drm_bufs.c) */ unsigned long drm_get_resource_start(struct drm_device *dev, unsigned int resource); unsigned long drm_get_resource_len(struct drm_device *dev, unsigned int resource); /* IRQ support (drm_irq.c) */ irqreturn_t drm_irq_handler(DRM_IRQ_ARGS); void drm_driver_irq_preinstall(struct drm_device *dev); void drm_driver_irq_postinstall(struct drm_device *dev); void drm_driver_irq_uninstall(struct drm_device *dev); /* sysctl support (drm_sysctl.h) */ extern int drm_sysctl_init(struct drm_device *dev); extern int drm_sysctl_cleanup(struct drm_device *dev); int drm_version(struct drm_device *dev, void *data, struct drm_file *file_priv); /* consistent PCI memory functions (drm_pci.c) */ int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master); int drm_pci_set_unique(struct drm_device *dev, struct drm_master *master, struct drm_unique *u); int drm_pci_agp_init(struct drm_device *dev); int drm_pci_enable_msi(struct drm_device *dev); void drm_pci_disable_msi(struct drm_device *dev); struct ttm_bo_device; int ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot); struct ttm_buffer_object; void ttm_bo_release_mmap(struct ttm_buffer_object *bo); #endif /* __KERNEL__ */ #endif Index: head/sys/mips/mips/stack_machdep.c =================================================================== --- head/sys/mips/mips/stack_machdep.c (revision 295880) +++ head/sys/mips/mips/stack_machdep.c (revision 295881) @@ -1,163 +1,162 @@ /*- * Copyright (c) 2005 Antoine Brodin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include -#include #include #include static u_register_t stack_register_fetch(u_register_t sp, u_register_t stack_pos) { u_register_t * stack = ((u_register_t *)(intptr_t)sp + (size_t)stack_pos/sizeof(u_register_t)); return *stack; } static void stack_capture(struct stack *st, u_register_t pc, u_register_t sp) { u_register_t ra = 0, i, stacksize; short ra_stack_pos = 0; InstFmt insn; stack_zero(st); for (;;) { stacksize = 0; if (pc <= (u_register_t)(intptr_t)btext) break; for (i = pc; i >= (u_register_t)(intptr_t)btext; i -= sizeof (insn)) { bcopy((void *)(intptr_t)i, &insn, sizeof insn); switch (insn.IType.op) { case OP_ADDI: case OP_ADDIU: case OP_DADDI: case OP_DADDIU: if (insn.IType.rs != SP || insn.IType.rt != SP) break; stacksize = -(short)insn.IType.imm; break; case OP_SW: case OP_SD: if (insn.IType.rs != SP || insn.IType.rt != RA) break; ra_stack_pos = (short)insn.IType.imm; break; default: break; } if (stacksize) break; } if (stack_put(st, pc) == -1) break; for (i = pc; !ra; i += sizeof (insn)) { bcopy((void *)(intptr_t)i, &insn, sizeof insn); switch (insn.IType.op) { case OP_SPECIAL: if((insn.RType.func == OP_JR)) { if (ra >= (u_register_t)(intptr_t)btext) break; if (insn.RType.rs != RA) break; ra = stack_register_fetch(sp, ra_stack_pos); if (!ra) goto done; ra -= 8; } break; default: break; } /* eret */ if (insn.word == 0x42000018) goto done; } if (pc == ra && stacksize == 0) break; sp += stacksize; pc = ra; ra = 0; } done: return; } void stack_save_td(struct stack *st, struct thread *td) { u_register_t pc, sp; if (TD_IS_SWAPPED(td)) panic("stack_save_td: swapped"); if (TD_IS_RUNNING(td)) panic("stack_save_td: running"); pc = td->td_pcb->pcb_regs.pc; sp = td->td_pcb->pcb_regs.sp; stack_capture(st, pc, sp); } int stack_save_td_running(struct stack *st, struct thread *td) { return (EOPNOTSUPP); } void stack_save(struct stack *st) { u_register_t pc, sp; if (curthread == NULL) panic("stack_save: curthread == NULL"); pc = curthread->td_pcb->pcb_regs.pc; sp = curthread->td_pcb->pcb_regs.sp; stack_capture(st, pc, sp); } Index: head/sys/mips/nlm/cms.c =================================================================== --- head/sys/mips/nlm/cms.c (revision 295880) +++ head/sys/mips/nlm/cms.c (revision 295881) @@ -1,497 +1,496 @@ /*- * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. * * NETLOGIC_BSD */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #include #include #include #include #include #define MSGRNG_NSTATIONS 1024 /* * Keep track of our message ring handler threads, each core has a * different message station. Ideally we will need to start a few * message handling threads every core, and wake them up depending on * load */ struct msgring_thread { struct thread *thread; /* msgring handler threads */ int needed; /* thread needs to wake up */ }; static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS]; static struct proc *msgring_proc; /* all threads are under a proc */ /* * The device drivers can register a handler for the messages sent * from a station (corresponding to the device). */ struct tx_stn_handler { msgring_handler action; void *arg; }; static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS]; static struct mtx msgmap_lock; uint32_t xlp_msg_thread_mask; static int xlp_msg_threads_per_core = XLP_MAX_THREADS; static void create_msgring_thread(int hwtid); static int msgring_process_fast_intr(void *arg); /* Debug counters */ static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS]; static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS]; static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS]; static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4]; static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS]; /* Whether polled driver implementation */ static int polled = 0; /* We do only i/o device credit setup here. CPU credit setup is now * moved to xlp_msgring_cpu_init() so that the credits get setup * only if the CPU exists. xlp_msgring_cpu_init() gets called from * platform_init_ap; and this makes it easy for us to setup CMS * credits for various types of XLP chips, with varying number of * cpu's and cores. */ static void xlp_cms_credit_setup(int credit) { uint64_t cmspcibase, cmsbase, pcibase; uint32_t devoffset; int dev, fn, maxqid; int src, qid, i; for (i = 0; i < XLP_MAX_NODES; i++) { cmspcibase = nlm_get_cms_pcibase(i); if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i))) continue; cmsbase = nlm_get_cms_regbase(i); maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0); for (dev = 0; dev < 8; dev++) { for (fn = 0; fn < 8; fn++) { devoffset = XLP_HDR_OFFSET(i, 0, dev, fn); if (nlm_dev_exists(devoffset) == 0) continue; pcibase = nlm_pcicfg_base(devoffset); src = nlm_qidstart(pcibase); if (src == 0) continue; #if 0 /* Debug */ printf("Setup CMS credits for queues "); printf("[%d to %d] from src %d\n", 0, maxqid, src); #endif for (qid = 0; qid < maxqid; qid++) nlm_cms_setup_credits(cmsbase, qid, src, credit); } } } } void xlp_msgring_cpu_init(int node, int cpu, int credit) { uint64_t cmspcibase = nlm_get_cms_pcibase(node); uint64_t cmsbase = nlm_get_cms_regbase(node); int qid, maxqid, src; maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0); /* cpu credit setup is done only from thread-0 of each core */ if((cpu % 4) == 0) { src = cpu << 2; /* each thread has 4 vc's */ for (qid = 0; qid < maxqid; qid++) nlm_cms_setup_credits(cmsbase, qid, src, credit); } } /* * Drain out max_messages for the buckets set in the bucket mask. * Use max_msgs = 0 to drain out all messages. */ int xlp_handle_msg_vc(u_int vcmask, int max_msgs) { struct nlm_fmn_msg msg; int srcid = 0, size = 0, code = 0; struct tx_stn_handler *he; uint32_t mflags, status; int n_msgs = 0, vc, m, hwtid; u_int msgmask; hwtid = nlm_cpuid(); for (;;) { /* check if VC empty */ mflags = nlm_save_flags_cop2(); status = nlm_read_c2_msgstatus1(); nlm_restore_flags(mflags); msgmask = ((status >> 24) & 0xf) ^ 0xf; msgmask &= vcmask; if (msgmask == 0) break; m = 0; for (vc = 0; vc < 4; vc++) { if ((msgmask & (1 << vc)) == 0) continue; mflags = nlm_save_flags_cop2(); status = nlm_fmn_msgrcv(vc, &srcid, &size, &code, &msg); nlm_restore_flags(mflags); if (status != 0) /* no msg or error */ continue; if (srcid < 0 && srcid >= 1024) { printf("[%s]: bad src id %d\n", __func__, srcid); continue; } he = &msgmap[srcid]; if(he->action != NULL) (he->action)(vc, size, code, srcid, &msg, he->arg); #if 0 else printf("[%s]: No Handler for msg from stn %d," " vc=%d, size=%d, msg0=%jx, droppinge\n", __func__, srcid, vc, size, (uintmax_t)msg.msg[0]); #endif fmn_msgcount[hwtid][vc] += 1; m++; /* msgs handled in this iter */ } if (m == 0) break; /* nothing done in this iter */ n_msgs += m; if (max_msgs > 0 && n_msgs >= max_msgs) break; } return (n_msgs); } static void xlp_discard_msg_vc(u_int vcmask) { struct nlm_fmn_msg msg; int srcid = 0, size = 0, code = 0, vc; uint32_t mflags, status; for (vc = 0; vc < 4; vc++) { for (;;) { mflags = nlm_save_flags_cop2(); status = nlm_fmn_msgrcv(vc, &srcid, &size, &code, &msg); nlm_restore_flags(mflags); /* break if there is no msg or error */ if (status != 0) break; } } } void xlp_cms_enable_intr(int node, int cpu, int type, int watermark) { uint64_t cmsbase; int i, qid; cmsbase = nlm_get_cms_regbase(node); for (i = 0; i < 4; i++) { qid = (i + (cpu * 4)) & 0x7f; nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark); nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0); } } static int msgring_process_fast_intr(void *arg) { struct msgring_thread *mthd; struct thread *td; int cpu; cpu = nlm_cpuid(); mthd = &msgring_threads[cpu]; msgring_nintr[cpu]++; td = mthd->thread; /* clear pending interrupts */ nlm_write_c0_eirr(1ULL << IRQ_MSGRING); /* wake up the target thread */ mthd->needed = 1; thread_lock(td); if (TD_AWAITING_INTR(td)) { msgring_wakeup_sleep[cpu]++; TD_CLR_IWAIT(td); sched_add(td, SRQ_INTR); } else msgring_wakeup_nosleep[cpu]++; thread_unlock(td); return (FILTER_HANDLED); } static void msgring_process(void * arg) { volatile struct msgring_thread *mthd; struct thread *td; uint32_t mflags, msgstatus1; int hwtid, nmsgs; hwtid = (intptr_t)arg; mthd = &msgring_threads[hwtid]; td = mthd->thread; KASSERT(curthread == td, ("%s:msg_ithread and proc linkage out of sync", __func__)); /* First bind this thread to the right CPU */ thread_lock(td); sched_bind(td, xlp_hwtid_to_cpuid[hwtid]); thread_unlock(td); if (hwtid != nlm_cpuid()) printf("Misscheduled hwtid %d != cpuid %d\n", hwtid, nlm_cpuid()); xlp_discard_msg_vc(0xf); xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT); if (polled == 0) { mflags = nlm_save_flags_cop2(); nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0); nlm_restore_flags(mflags); xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0); /* clear pending interrupts. * they will get re-raised if still valid */ nlm_write_c0_eirr(1ULL << IRQ_MSGRING); } /* start processing messages */ for (;;) { atomic_store_rel_int(&mthd->needed, 0); nmsgs = xlp_handle_msg_vc(0xf, 0); /* sleep */ if (polled == 0) { /* clear VC-pend bits */ mflags = nlm_save_flags_cop2(); msgstatus1 = nlm_read_c2_msgstatus1(); msgstatus1 |= (0xf << 16); nlm_write_c2_msgstatus1(msgstatus1); nlm_restore_flags(mflags); thread_lock(td); if (mthd->needed) { thread_unlock(td); continue; } sched_class(td, PRI_ITHD); TD_SET_IWAIT(td); mi_switch(SW_VOL, NULL); thread_unlock(td); } else pause("wmsg", 1); fmn_loops[hwtid]++; } } static void create_msgring_thread(int hwtid) { struct msgring_thread *mthd; struct thread *td; int error; mthd = &msgring_threads[hwtid]; error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid, &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc", "msgthr%d", hwtid); if (error) panic("kproc_kthread_add() failed with %d", error); mthd->thread = td; thread_lock(td); sched_class(td, PRI_ITHD); sched_add(td, SRQ_INTR); thread_unlock(td); } int register_msgring_handler(int startb, int endb, msgring_handler action, void *arg) { int i; if (bootverbose) printf("Register handler %d-%d %p(%p)\n", startb, endb, action, arg); KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS, ("Invalid value for bucket range %d,%d", startb, endb)); mtx_lock_spin(&msgmap_lock); for (i = startb; i <= endb; i++) { KASSERT(msgmap[i].action == NULL, ("Bucket %d already used [action %p]", i, msgmap[i].action)); msgmap[i].action = action; msgmap[i].arg = arg; } mtx_unlock_spin(&msgmap_lock); return (0); } /* * Initialize the messaging subsystem. * * Message Stations are shared among all threads in a cpu core, this * has to be called once from every core which is online. */ static void xlp_msgring_config(void *arg) { void *cookie; unsigned int thrmask, mask; int i; /* used polled handler for Ax silion */ if (nlm_is_xlp8xx_ax()) polled = 1; /* Don't poll on all threads, if polled */ if (polled) xlp_msg_threads_per_core -= 1; mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN); if (xlp_threads_per_core < xlp_msg_threads_per_core) xlp_msg_threads_per_core = xlp_threads_per_core; thrmask = ((1 << xlp_msg_threads_per_core) - 1); mask = 0; for (i = 0; i < XLP_MAX_CORES; i++) { mask <<= XLP_MAX_THREADS; mask |= thrmask; } xlp_msg_thread_mask = xlp_hw_thread_mask & mask; #if 0 printf("CMS Message handler thread mask %#jx\n", (uintmax_t)xlp_msg_thread_mask); #endif xlp_cms_credit_setup(CMS_DEFAULT_CREDIT); create_msgring_thread(0); cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL, NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie); } /* * Start message ring processing threads on other CPUs, after SMP start */ static void start_msgring_threads(void *arg) { int hwt; for (hwt = 1; hwt < XLP_MAX_CORES * XLP_MAX_THREADS; hwt++) { if ((xlp_msg_thread_mask & (1 << hwt)) == 0) continue; create_msgring_thread(hwt); } } SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST, xlp_msgring_config, NULL); SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, start_msgring_threads, NULL); /* * DEBUG support, XXX: static buffer, not locked */ static int sys_print_debug(SYSCTL_HANDLER_ARGS) { struct sbuf sb; int error, i; sbuf_new_for_sysctl(&sb, NULL, 64, req); sbuf_printf(&sb, "\nID vc0 vc1 vc2 vc3 loops\n"); for (i = 0; i < 32; i++) { if ((xlp_hw_thread_mask & (1 << i)) == 0) continue; sbuf_printf(&sb, "%2d: %8d %8d %8d %8d %8d\n", i, fmn_msgcount[i][0], fmn_msgcount[i][1], fmn_msgcount[i][2], fmn_msgcount[i][3], fmn_loops[i]); } error = sbuf_finish(&sb); sbuf_delete(&sb); return (error); } SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0, sys_print_debug, "A", "msgring debug info"); Index: head/sys/mips/nlm/dev/net/xlpge.c =================================================================== --- head/sys/mips/nlm/dev/net/xlpge.c (revision 295880) +++ head/sys/mips/nlm/dev/net/xlpge.c (revision 295881) @@ -1,1542 +1,1541 @@ /*- * Copyright (c) 2003-2012 Broadcom Corporation * All Rights Reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define __RMAN_RESOURCE_VISIBLE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include /* for DELAY */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "miidevs.h" #include #include "miibus_if.h" #include #include /*#define XLP_DRIVER_LOOPBACK*/ static struct nae_port_config nae_port_config[64]; int poe_cl_tbl[MAX_POE_CLASSES] = { 0x0, 0x249249, 0x492492, 0x6db6db, 0x924924, 0xb6db6d, 0xdb6db6, 0xffffff }; /* #define DUMP_PACKET */ static uint64_t nlm_paddr_ld(uint64_t paddr) { uint64_t xkaddr = 0x9800000000000000 | paddr; return (nlm_load_dword_daddr(xkaddr)); } struct nlm_xlp_portdata ifp_ports[64]; static uma_zone_t nl_tx_desc_zone; /* This implementation will register the following tree of device * registration: * pcibus * | * xlpnae (1 instance - virtual entity) * | * xlpge * (18 sgmii / 4 xaui / 2 interlaken instances) * | * miibus */ static int nlm_xlpnae_probe(device_t); static int nlm_xlpnae_attach(device_t); static int nlm_xlpnae_detach(device_t); static int nlm_xlpnae_suspend(device_t); static int nlm_xlpnae_resume(device_t); static int nlm_xlpnae_shutdown(device_t); static device_method_t nlm_xlpnae_methods[] = { /* Methods from the device interface */ DEVMETHOD(device_probe, nlm_xlpnae_probe), DEVMETHOD(device_attach, nlm_xlpnae_attach), DEVMETHOD(device_detach, nlm_xlpnae_detach), DEVMETHOD(device_suspend, nlm_xlpnae_suspend), DEVMETHOD(device_resume, nlm_xlpnae_resume), DEVMETHOD(device_shutdown, nlm_xlpnae_shutdown), DEVMETHOD(bus_driver_added, bus_generic_driver_added), DEVMETHOD_END }; static driver_t nlm_xlpnae_driver = { "xlpnae", nlm_xlpnae_methods, sizeof(struct nlm_xlpnae_softc) }; static devclass_t nlm_xlpnae_devclass; static int nlm_xlpge_probe(device_t); static int nlm_xlpge_attach(device_t); static int nlm_xlpge_detach(device_t); static int nlm_xlpge_suspend(device_t); static int nlm_xlpge_resume(device_t); static int nlm_xlpge_shutdown(device_t); /* mii override functions */ static int nlm_xlpge_mii_read(struct device *, int, int); static int nlm_xlpge_mii_write(struct device *, int, int, int); static void nlm_xlpge_mii_statchg(device_t); static device_method_t nlm_xlpge_methods[] = { /* Methods from the device interface */ DEVMETHOD(device_probe, nlm_xlpge_probe), DEVMETHOD(device_attach, nlm_xlpge_attach), DEVMETHOD(device_detach, nlm_xlpge_detach), DEVMETHOD(device_suspend, nlm_xlpge_suspend), DEVMETHOD(device_resume, nlm_xlpge_resume), DEVMETHOD(device_shutdown, nlm_xlpge_shutdown), /* Methods from the nexus bus needed for explicitly * probing children when driver is loaded as a kernel module */ DEVMETHOD(miibus_readreg, nlm_xlpge_mii_read), DEVMETHOD(miibus_writereg, nlm_xlpge_mii_write), DEVMETHOD(miibus_statchg, nlm_xlpge_mii_statchg), /* Terminate method list */ DEVMETHOD_END }; static driver_t nlm_xlpge_driver = { "xlpge", nlm_xlpge_methods, sizeof(struct nlm_xlpge_softc) }; static devclass_t nlm_xlpge_devclass; DRIVER_MODULE(xlpnae, pci, nlm_xlpnae_driver, nlm_xlpnae_devclass, 0, 0); DRIVER_MODULE(xlpge, xlpnae, nlm_xlpge_driver, nlm_xlpge_devclass, 0, 0); DRIVER_MODULE(miibus, xlpge, miibus_driver, miibus_devclass, 0, 0); MODULE_DEPEND(pci, xlpnae, 1, 1, 1); MODULE_DEPEND(xlpnae, xlpge, 1, 1, 1); MODULE_DEPEND(xlpge, ether, 1, 1, 1); MODULE_DEPEND(xlpge, miibus, 1, 1, 1); #define SGMII_RCV_CONTEXT_WIDTH 8 /* prototypes */ static void nlm_xlpge_msgring_handler(int vc, int size, int code, int srcid, struct nlm_fmn_msg *msg, void *data); static void nlm_xlpge_submit_rx_free_desc(struct nlm_xlpge_softc *sc, int num); static void nlm_xlpge_init(void *addr); static void nlm_xlpge_port_disable(struct nlm_xlpge_softc *sc); static void nlm_xlpge_port_enable(struct nlm_xlpge_softc *sc); /* globals */ int dbg_on = 1; int cntx2port[524]; static __inline void atomic_incr_long(unsigned long *addr) { atomic_add_long(addr, 1); } /* * xlpnae driver implementation */ static int nlm_xlpnae_probe(device_t dev) { if (pci_get_vendor(dev) != PCI_VENDOR_NETLOGIC || pci_get_device(dev) != PCI_DEVICE_ID_NLM_NAE) return (ENXIO); return (BUS_PROBE_DEFAULT); } static void nlm_xlpnae_print_frin_desc_carving(struct nlm_xlpnae_softc *sc) { int intf; uint32_t value; int start, size; /* XXXJC: use max_ports instead of 20 ? */ for (intf = 0; intf < 20; intf++) { nlm_write_nae_reg(sc->base, NAE_FREE_IN_FIFO_CFG, (0x80000000 | intf)); value = nlm_read_nae_reg(sc->base, NAE_FREE_IN_FIFO_CFG); size = 2 * ((value >> 20) & 0x3ff); start = 2 * ((value >> 8) & 0x1ff); } } static void nlm_config_egress(struct nlm_xlpnae_softc *sc, int nblock, int context_base, int hwport, int max_channels) { int offset, num_channels; uint32_t data; num_channels = sc->portcfg[hwport].num_channels; data = (2048 << 12) | (hwport << 4) | 1; nlm_write_nae_reg(sc->base, NAE_TX_IF_BURSTMAX_CMD, data); data = ((context_base + num_channels - 1) << 22) | (context_base << 12) | (hwport << 4) | 1; nlm_write_nae_reg(sc->base, NAE_TX_DDR_ACTVLIST_CMD, data); config_egress_fifo_carvings(sc->base, hwport, context_base, num_channels, max_channels, sc->portcfg); config_egress_fifo_credits(sc->base, hwport, context_base, num_channels, max_channels, sc->portcfg); data = nlm_read_nae_reg(sc->base, NAE_DMA_TX_CREDIT_TH); data |= (1 << 25) | (1 << 24); nlm_write_nae_reg(sc->base, NAE_DMA_TX_CREDIT_TH, data); for (offset = 0; offset < num_channels; offset++) { nlm_write_nae_reg(sc->base, NAE_TX_SCHED_MAP_CMD1, NAE_DRR_QUANTA); data = (hwport << 15) | ((context_base + offset) << 5); if (sc->cmplx_type[nblock] == ILC) data |= (offset << 20); nlm_write_nae_reg(sc->base, NAE_TX_SCHED_MAP_CMD0, data | 1); nlm_write_nae_reg(sc->base, NAE_TX_SCHED_MAP_CMD0, data); } } static int xlpnae_get_maxchannels(struct nlm_xlpnae_softc *sc) { int maxchans = 0; int i; for (i = 0; i < sc->max_ports; i++) { if (sc->portcfg[i].type == UNKNOWN) continue; maxchans += sc->portcfg[i].num_channels; } return (maxchans); } static void nlm_setup_interface(struct nlm_xlpnae_softc *sc, int nblock, int port, uint32_t cur_flow_base, uint32_t flow_mask, int max_channels, int context) { uint64_t nae_base = sc->base; int mtu = 1536; /* XXXJC: don't hard code */ uint32_t ucore_mask; if (sc->cmplx_type[nblock] == XAUIC) nlm_config_xaui(nae_base, nblock, mtu, mtu, sc->portcfg[port].vlan_pri_en); nlm_config_freein_fifo_uniq_cfg(nae_base, port, sc->portcfg[port].free_desc_sizes); nlm_config_ucore_iface_mask_cfg(nae_base, port, sc->portcfg[port].ucore_mask); nlm_program_flow_cfg(nae_base, port, cur_flow_base, flow_mask); if (sc->cmplx_type[nblock] == SGMIIC) nlm_configure_sgmii_interface(nae_base, nblock, port, mtu, 0); nlm_config_egress(sc, nblock, context, port, max_channels); nlm_nae_init_netior(nae_base, sc->nblocks); nlm_nae_open_if(nae_base, nblock, sc->cmplx_type[nblock], port, sc->portcfg[port].free_desc_sizes); /* XXXJC: check mask calculation */ ucore_mask = (1 << sc->nucores) - 1; nlm_nae_init_ucore(nae_base, port, ucore_mask); } static void nlm_setup_interfaces(struct nlm_xlpnae_softc *sc) { uint64_t nae_base; uint32_t cur_slot, cur_slot_base; uint32_t cur_flow_base, port, flow_mask; int max_channels; int i, context; cur_slot = 0; cur_slot_base = 0; cur_flow_base = 0; nae_base = sc->base; flow_mask = nlm_get_flow_mask(sc->total_num_ports); /* calculate max_channels */ max_channels = xlpnae_get_maxchannels(sc); port = 0; context = 0; for (i = 0; i < sc->max_ports; i++) { if (sc->portcfg[i].type == UNKNOWN) continue; nlm_setup_interface(sc, sc->portcfg[i].block, i, cur_flow_base, flow_mask, max_channels, context); cur_flow_base += sc->per_port_num_flows; context += sc->portcfg[i].num_channels; } } static void nlm_xlpnae_init(int node, struct nlm_xlpnae_softc *sc) { uint64_t nae_base; uint32_t ucoremask = 0; uint32_t val; int i; nae_base = sc->base; nlm_nae_flush_free_fifo(nae_base, sc->nblocks); nlm_deflate_frin_fifo_carving(nae_base, sc->max_ports); nlm_reset_nae(node); for (i = 0; i < sc->nucores; i++) /* XXXJC: code repeated below */ ucoremask |= (0x1 << i); printf("Loading 0x%x ucores with microcode\n", ucoremask); nlm_ucore_load_all(nae_base, ucoremask, 1); val = nlm_set_device_frequency(node, DFS_DEVICE_NAE, sc->freq); printf("Setup NAE frequency to %dMHz\n", val); nlm_mdio_reset_all(nae_base); printf("Initialze SGMII PCS for blocks 0x%x\n", sc->sgmiimask); nlm_sgmii_pcs_init(nae_base, sc->sgmiimask); printf("Initialze XAUI PCS for blocks 0x%x\n", sc->xauimask); nlm_xaui_pcs_init(nae_base, sc->xauimask); /* clear NETIOR soft reset */ nlm_write_nae_reg(nae_base, NAE_LANE_CFG_SOFTRESET, 0x0); /* Disable RX enable bit in RX_CONFIG */ val = nlm_read_nae_reg(nae_base, NAE_RX_CONFIG); val &= 0xfffffffe; nlm_write_nae_reg(nae_base, NAE_RX_CONFIG, val); if (nlm_is_xlp8xx_ax() == 0) { val = nlm_read_nae_reg(nae_base, NAE_TX_CONFIG); val &= ~(1 << 3); nlm_write_nae_reg(nae_base, NAE_TX_CONFIG, val); } nlm_setup_poe_class_config(nae_base, MAX_POE_CLASSES, sc->ncontexts, poe_cl_tbl); nlm_setup_vfbid_mapping(nae_base); nlm_setup_flow_crc_poly(nae_base, sc->flow_crc_poly); nlm_setup_rx_cal_cfg(nae_base, sc->max_ports, sc->portcfg); /* note: xlp8xx Ax does not have Tx Calendering */ if (!nlm_is_xlp8xx_ax()) nlm_setup_tx_cal_cfg(nae_base, sc->max_ports, sc->portcfg); nlm_setup_interfaces(sc); nlm_config_poe(sc->poe_base, sc->poedv_base); if (sc->hw_parser_en) nlm_enable_hardware_parser(nae_base); if (sc->prepad_en) nlm_prepad_enable(nae_base, sc->prepad_size); if (sc->ieee_1588_en) nlm_setup_1588_timer(sc->base, sc->portcfg); } static void nlm_xlpnae_update_pde(void *dummy __unused) { struct nlm_xlpnae_softc *sc; uint32_t dv[NUM_WORDS_PER_DV]; device_t dev; int vec; dev = devclass_get_device(devclass_find("xlpnae"), 0); sc = device_get_softc(dev); nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 0); for (vec = 0; vec < NUM_DIST_VEC; vec++) { if (nlm_get_poe_distvec(vec, dv) != 0) continue; nlm_write_poe_distvec(sc->poedv_base, vec, dv); } nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 1); } SYSINIT(nlm_xlpnae_update_pde, SI_SUB_SMP, SI_ORDER_ANY, nlm_xlpnae_update_pde, NULL); /* configuration common for sgmii, xaui, ilaken goes here */ static void nlm_setup_portcfg(struct nlm_xlpnae_softc *sc, struct xlp_nae_ivars *naep, int block, int port) { int i; uint32_t ucore_mask = 0; struct xlp_block_ivars *bp; struct xlp_port_ivars *p; bp = &(naep->block_ivars[block]); p = &(bp->port_ivars[port & 0x3]); sc->portcfg[port].node = p->node; sc->portcfg[port].block = p->block; sc->portcfg[port].port = p->port; sc->portcfg[port].type = p->type; sc->portcfg[port].mdio_bus = p->mdio_bus; sc->portcfg[port].phy_addr = p->phy_addr; sc->portcfg[port].loopback_mode = p->loopback_mode; sc->portcfg[port].num_channels = p->num_channels; if (p->free_desc_sizes != MCLBYTES) { printf("[%d, %d] Error: free_desc_sizes %d != %d\n", block, port, p->free_desc_sizes, MCLBYTES); return; } sc->portcfg[port].free_desc_sizes = p->free_desc_sizes; for (i = 0; i < sc->nucores; i++) /* XXXJC: configure this */ ucore_mask |= (0x1 << i); sc->portcfg[port].ucore_mask = ucore_mask; sc->portcfg[port].vlan_pri_en = p->vlan_pri_en; sc->portcfg[port].num_free_descs = p->num_free_descs; sc->portcfg[port].iface_fifo_size = p->iface_fifo_size; sc->portcfg[port].rxbuf_size = p->rxbuf_size; sc->portcfg[port].rx_slots_reqd = p->rx_slots_reqd; sc->portcfg[port].tx_slots_reqd = p->tx_slots_reqd; sc->portcfg[port].pseq_fifo_size = p->pseq_fifo_size; sc->portcfg[port].stg2_fifo_size = p->stg2_fifo_size; sc->portcfg[port].eh_fifo_size = p->eh_fifo_size; sc->portcfg[port].frout_fifo_size = p->frout_fifo_size; sc->portcfg[port].ms_fifo_size = p->ms_fifo_size; sc->portcfg[port].pkt_fifo_size = p->pkt_fifo_size; sc->portcfg[port].pktlen_fifo_size = p->pktlen_fifo_size; sc->portcfg[port].max_stg2_offset = p->max_stg2_offset; sc->portcfg[port].max_eh_offset = p->max_eh_offset; sc->portcfg[port].max_frout_offset = p->max_frout_offset; sc->portcfg[port].max_ms_offset = p->max_ms_offset; sc->portcfg[port].max_pmem_offset = p->max_pmem_offset; sc->portcfg[port].stg1_2_credit = p->stg1_2_credit; sc->portcfg[port].stg2_eh_credit = p->stg2_eh_credit; sc->portcfg[port].stg2_frout_credit = p->stg2_frout_credit; sc->portcfg[port].stg2_ms_credit = p->stg2_ms_credit; sc->portcfg[port].ieee1588_inc_intg = p->ieee1588_inc_intg; sc->portcfg[port].ieee1588_inc_den = p->ieee1588_inc_den; sc->portcfg[port].ieee1588_inc_num = p->ieee1588_inc_num; sc->portcfg[port].ieee1588_userval = p->ieee1588_userval; sc->portcfg[port].ieee1588_ptpoff = p->ieee1588_ptpoff; sc->portcfg[port].ieee1588_tmr1 = p->ieee1588_tmr1; sc->portcfg[port].ieee1588_tmr2 = p->ieee1588_tmr2; sc->portcfg[port].ieee1588_tmr3 = p->ieee1588_tmr3; sc->total_free_desc += sc->portcfg[port].free_desc_sizes; sc->total_num_ports++; } static int nlm_xlpnae_attach(device_t dev) { struct xlp_nae_ivars *nae_ivars; struct nlm_xlpnae_softc *sc; device_t tmpd; uint32_t dv[NUM_WORDS_PER_DV]; int port, i, j, nchan, nblock, node, qstart, qnum; int offset, context, txq_base, rxvcbase; uint64_t poe_pcibase, nae_pcibase; node = pci_get_slot(dev) / 8; nae_ivars = &xlp_board_info.nodes[node].nae_ivars; sc = device_get_softc(dev); sc->xlpnae_dev = dev; sc->node = nae_ivars->node; sc->base = nlm_get_nae_regbase(sc->node); sc->poe_base = nlm_get_poe_regbase(sc->node); sc->poedv_base = nlm_get_poedv_regbase(sc->node); sc->portcfg = nae_port_config; sc->blockmask = nae_ivars->blockmask; sc->ilmask = nae_ivars->ilmask; sc->xauimask = nae_ivars->xauimask; sc->sgmiimask = nae_ivars->sgmiimask; sc->nblocks = nae_ivars->nblocks; sc->freq = nae_ivars->freq; /* flow table generation is done by CRC16 polynomial */ sc->flow_crc_poly = nae_ivars->flow_crc_poly; sc->hw_parser_en = nae_ivars->hw_parser_en; sc->prepad_en = nae_ivars->prepad_en; sc->prepad_size = nae_ivars->prepad_size; sc->ieee_1588_en = nae_ivars->ieee_1588_en; nae_pcibase = nlm_get_nae_pcibase(sc->node); sc->ncontexts = nlm_read_reg(nae_pcibase, XLP_PCI_DEVINFO_REG5); sc->nucores = nlm_num_uengines(nae_pcibase); for (nblock = 0; nblock < sc->nblocks; nblock++) { sc->cmplx_type[nblock] = nae_ivars->block_ivars[nblock].type; sc->portmask[nblock] = nae_ivars->block_ivars[nblock].portmask; } for (i = 0; i < sc->ncontexts; i++) cntx2port[i] = 18; /* 18 is an invalid port */ if (sc->nblocks == 5) sc->max_ports = 18; /* 8xx has a block 4 with 2 ports */ else sc->max_ports = sc->nblocks * PORTS_PER_CMPLX; for (i = 0; i < sc->max_ports; i++) sc->portcfg[i].type = UNKNOWN; /* Port Not Present */ /* * Now setup all internal fifo carvings based on * total number of ports in the system */ sc->total_free_desc = 0; sc->total_num_ports = 0; port = 0; context = 0; txq_base = nlm_qidstart(nae_pcibase); rxvcbase = txq_base + sc->ncontexts; for (i = 0; i < sc->nblocks; i++) { uint32_t portmask; if ((nae_ivars->blockmask & (1 << i)) == 0) { port += 4; continue; } portmask = nae_ivars->block_ivars[i].portmask; for (j = 0; j < PORTS_PER_CMPLX; j++, port++) { if ((portmask & (1 << j)) == 0) continue; nlm_setup_portcfg(sc, nae_ivars, i, port); nchan = sc->portcfg[port].num_channels; for (offset = 0; offset < nchan; offset++) cntx2port[context + offset] = port; sc->portcfg[port].txq = txq_base + context; sc->portcfg[port].rxfreeq = rxvcbase + port; context += nchan; } } poe_pcibase = nlm_get_poe_pcibase(sc->node); sc->per_port_num_flows = nlm_poe_max_flows(poe_pcibase) / sc->total_num_ports; /* zone for P2P descriptors */ nl_tx_desc_zone = uma_zcreate("NL Tx Desc", sizeof(struct xlpge_tx_desc), NULL, NULL, NULL, NULL, NAE_CACHELINE_SIZE, 0); /* NAE FMN messages have CMS src station id's in the * range of qstart to qnum. */ qstart = nlm_qidstart(nae_pcibase); qnum = nlm_qnum(nae_pcibase); if (register_msgring_handler(qstart, qstart + qnum - 1, nlm_xlpge_msgring_handler, sc)) { panic("Couldn't register NAE msgring handler\n"); } /* POE FMN messages have CMS src station id's in the * range of qstart to qnum. */ qstart = nlm_qidstart(poe_pcibase); qnum = nlm_qnum(poe_pcibase); if (register_msgring_handler(qstart, qstart + qnum - 1, nlm_xlpge_msgring_handler, sc)) { panic("Couldn't register POE msgring handler\n"); } nlm_xlpnae_init(node, sc); for (i = 0; i < sc->max_ports; i++) { char desc[32]; int block, port; if (sc->portcfg[i].type == UNKNOWN) continue; block = sc->portcfg[i].block; port = sc->portcfg[i].port; tmpd = device_add_child(dev, "xlpge", i); device_set_ivars(tmpd, &(nae_ivars->block_ivars[block].port_ivars[port])); sprintf(desc, "XLP NAE Port %d,%d", block, port); device_set_desc_copy(tmpd, desc); } nlm_setup_iface_fifo_cfg(sc->base, sc->max_ports, sc->portcfg); nlm_setup_rx_base_config(sc->base, sc->max_ports, sc->portcfg); nlm_setup_rx_buf_config(sc->base, sc->max_ports, sc->portcfg); nlm_setup_freein_fifo_cfg(sc->base, sc->portcfg); nlm_program_nae_parser_seq_fifo(sc->base, sc->max_ports, sc->portcfg); nlm_xlpnae_print_frin_desc_carving(sc); bus_generic_probe(dev); bus_generic_attach(dev); /* * Enable only boot cpu at this point, full distribution comes * only after SMP is started */ nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 0); nlm_calc_poe_distvec(0x1, 0, 0, 0, 0x1 << XLPGE_RX_VC, dv); nlm_write_poe_distvec(sc->poedv_base, 0, dv); nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 1); return (0); } static int nlm_xlpnae_detach(device_t dev) { /* TODO - free zone here */ return (0); } static int nlm_xlpnae_suspend(device_t dev) { return (0); } static int nlm_xlpnae_resume(device_t dev) { return (0); } static int nlm_xlpnae_shutdown(device_t dev) { return (0); } /* * xlpge driver implementation */ static void nlm_xlpge_mac_set_rx_mode(struct nlm_xlpge_softc *sc) { if (sc->if_flags & IFF_PROMISC) { if (sc->type == SGMIIC) nlm_nae_setup_rx_mode_sgmii(sc->base_addr, sc->block, sc->port, sc->type, 1 /* broadcast */, 1/* multicast */, 0 /* pause */, 1 /* promisc */); else nlm_nae_setup_rx_mode_xaui(sc->base_addr, sc->block, sc->port, sc->type, 1 /* broadcast */, 1/* multicast */, 0 /* pause */, 1 /* promisc */); } else { if (sc->type == SGMIIC) nlm_nae_setup_rx_mode_sgmii(sc->base_addr, sc->block, sc->port, sc->type, 1 /* broadcast */, 1/* multicast */, 0 /* pause */, 0 /* promisc */); else nlm_nae_setup_rx_mode_xaui(sc->base_addr, sc->block, sc->port, sc->type, 1 /* broadcast */, 1/* multicast */, 0 /* pause */, 0 /* promisc */); } } static int nlm_xlpge_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct mii_data *mii; struct nlm_xlpge_softc *sc; struct ifreq *ifr; int error; sc = ifp->if_softc; error = 0; ifr = (struct ifreq *)data; switch (command) { case SIOCSIFFLAGS: XLPGE_LOCK(sc); sc->if_flags = ifp->if_flags; if (ifp->if_flags & IFF_UP) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) nlm_xlpge_init(sc); else nlm_xlpge_port_enable(sc); nlm_xlpge_mac_set_rx_mode(sc); sc->link = NLM_LINK_UP; } else { if (ifp->if_drv_flags & IFF_DRV_RUNNING) nlm_xlpge_port_disable(sc); sc->link = NLM_LINK_DOWN; } XLPGE_UNLOCK(sc); error = 0; break; case SIOCGIFMEDIA: case SIOCSIFMEDIA: if (sc->mii_bus != NULL) { mii = device_get_softc(sc->mii_bus); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command); } break; default: error = ether_ioctl(ifp, command, data); break; } return (error); } static int xlpge_tx(struct ifnet *ifp, struct mbuf *mbuf_chain) { struct nlm_fmn_msg msg; struct xlpge_tx_desc *p2p; struct nlm_xlpge_softc *sc; struct mbuf *m; vm_paddr_t paddr; int fbid, dst, pos, err; int ret = 0, tx_msgstatus, retries; err = 0; if (mbuf_chain == NULL) return (0); sc = ifp->if_softc; p2p = NULL; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING) || ifp->if_drv_flags & IFF_DRV_OACTIVE) { err = ENXIO; goto fail; } /* free a few in coming messages on the fb vc */ xlp_handle_msg_vc(1 << XLPGE_FB_VC, 2); /* vfb id table is setup to map cpu to vc 3 of the cpu */ fbid = nlm_cpuid(); dst = sc->txq; pos = 0; p2p = uma_zalloc(nl_tx_desc_zone, M_NOWAIT); if (p2p == NULL) { printf("alloc fail\n"); err = ENOBUFS; goto fail; } for (m = mbuf_chain; m != NULL; m = m->m_next) { vm_offset_t buf = (vm_offset_t) m->m_data; int len = m->m_len; int frag_sz; uint64_t desc; /*printf("m_data = %p len %d\n", m->m_data, len); */ while (len) { if (pos == XLP_NTXFRAGS - 3) { device_printf(sc->xlpge_dev, "packet defrag %d\n", m_length(mbuf_chain, NULL)); err = ENOBUFS; /* TODO fix error */ goto fail; } paddr = vtophys(buf); frag_sz = PAGE_SIZE - (buf & PAGE_MASK); if (len < frag_sz) frag_sz = len; desc = nae_tx_desc(P2D_NEOP, 0, 127, frag_sz, paddr); p2p->frag[pos] = htobe64(desc); pos++; len -= frag_sz; buf += frag_sz; } } KASSERT(pos != 0, ("Zero-length mbuf chain?\n")); /* Make the last one P2D EOP */ p2p->frag[pos-1] |= htobe64((uint64_t)P2D_EOP << 62); /* stash useful pointers in the desc */ p2p->frag[XLP_NTXFRAGS-3] = 0xf00bad; p2p->frag[XLP_NTXFRAGS-2] = (uintptr_t)p2p; p2p->frag[XLP_NTXFRAGS-1] = (uintptr_t)mbuf_chain; paddr = vtophys(p2p); msg.msg[0] = nae_tx_desc(P2P, 0, fbid, pos, paddr); for (retries = 16; retries > 0; retries--) { ret = nlm_fmn_msgsend(dst, 1, FMN_SWCODE_NAE, &msg); if (ret == 0) return (0); } fail: if (ret != 0) { tx_msgstatus = nlm_read_c2_txmsgstatus(); if ((tx_msgstatus >> 24) & 0x1) device_printf(sc->xlpge_dev, "Transmit queue full - "); if ((tx_msgstatus >> 3) & 0x1) device_printf(sc->xlpge_dev, "ECC error - "); if ((tx_msgstatus >> 2) & 0x1) device_printf(sc->xlpge_dev, "Pending Sync - "); if ((tx_msgstatus >> 1) & 0x1) device_printf(sc->xlpge_dev, "Insufficient input queue credits - "); if (tx_msgstatus & 0x1) device_printf(sc->xlpge_dev, "Insufficient output queue credits - "); } device_printf(sc->xlpge_dev, "Send failed! err = %d\n", err); if (p2p) uma_zfree(nl_tx_desc_zone, p2p); m_freem(mbuf_chain); if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); return (err); } static int nlm_xlpge_gmac_config_speed(struct nlm_xlpge_softc *sc) { struct mii_data *mii; if (sc->type == XAUIC || sc->type == ILC) return (0); if (sc->mii_bus) { mii = device_get_softc(sc->mii_bus); mii_pollstat(mii); } return (0); } static void nlm_xlpge_port_disable(struct nlm_xlpge_softc *sc) { struct ifnet *ifp; ifp = sc->xlpge_if; ifp->if_drv_flags &= ~IFF_DRV_RUNNING; callout_stop(&sc->xlpge_callout); nlm_mac_disable(sc->base_addr, sc->block, sc->type, sc->port); } static void nlm_mii_pollstat(void *arg) { struct nlm_xlpge_softc *sc = (struct nlm_xlpge_softc *)arg; struct mii_data *mii = NULL; if (sc->mii_bus) { mii = device_get_softc(sc->mii_bus); KASSERT(mii != NULL, ("mii ptr is NULL")); mii_pollstat(mii); callout_reset(&sc->xlpge_callout, hz, nlm_mii_pollstat, sc); } } static void nlm_xlpge_port_enable(struct nlm_xlpge_softc *sc) { if ((sc->type != SGMIIC) && (sc->type != XAUIC)) return; nlm_mac_enable(sc->base_addr, sc->block, sc->type, sc->port); nlm_mii_pollstat((void *)sc); } static void nlm_xlpge_init(void *addr) { struct nlm_xlpge_softc *sc; struct ifnet *ifp; struct mii_data *mii = NULL; sc = (struct nlm_xlpge_softc *)addr; ifp = sc->xlpge_if; if (ifp->if_drv_flags & IFF_DRV_RUNNING) return; if (sc->mii_bus) { mii = device_get_softc(sc->mii_bus); mii_mediachg(mii); } nlm_xlpge_gmac_config_speed(sc); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; nlm_xlpge_port_enable(sc); /* start the callout */ callout_reset(&sc->xlpge_callout, hz, nlm_mii_pollstat, sc); } /* * Read the MAC address from FDT or board eeprom. */ static void xlpge_read_mac_addr(struct nlm_xlpge_softc *sc) { xlpge_get_macaddr(sc->dev_addr); /* last octet is port specific */ sc->dev_addr[5] += (sc->block * 4) + sc->port; if (sc->type == SGMIIC) nlm_nae_setup_mac_addr_sgmii(sc->base_addr, sc->block, sc->port, sc->type, sc->dev_addr); else if (sc->type == XAUIC) nlm_nae_setup_mac_addr_xaui(sc->base_addr, sc->block, sc->port, sc->type, sc->dev_addr); } static int xlpge_mediachange(struct ifnet *ifp) { return (0); } static void xlpge_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr) { struct nlm_xlpge_softc *sc; struct mii_data *md; md = NULL; sc = ifp->if_softc; if (sc->mii_bus) md = device_get_softc(sc->mii_bus); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (sc->link == NLM_LINK_DOWN) return; if (md != NULL) ifmr->ifm_active = md->mii_media.ifm_cur->ifm_media; ifmr->ifm_status |= IFM_ACTIVE; } static int nlm_xlpge_ifinit(struct nlm_xlpge_softc *sc) { struct ifnet *ifp; device_t dev; int port = sc->block * 4 + sc->port; dev = sc->xlpge_dev; ifp = sc->xlpge_if = if_alloc(IFT_ETHER); /*(sc->network_sc)->ifp_ports[port].xlpge_if = ifp;*/ ifp_ports[port].xlpge_if = ifp; if (ifp == NULL) { device_printf(dev, "cannot if_alloc()\n"); return (ENOSPC); } ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; sc->if_flags = ifp->if_flags; /*ifp->if_capabilities = IFCAP_TXCSUM | IFCAP_VLAN_HWTAGGING;*/ ifp->if_capabilities = 0; ifp->if_capenable = ifp->if_capabilities; ifp->if_ioctl = nlm_xlpge_ioctl; ifp->if_init = nlm_xlpge_init ; ifp->if_hwassist = 0; ifp->if_snd.ifq_drv_maxlen = NLM_XLPGE_TXQ_SIZE; /* TODO: make this a sysint */ IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); IFQ_SET_READY(&ifp->if_snd); ifmedia_init(&sc->xlpge_mii.mii_media, 0, xlpge_mediachange, xlpge_mediastatus); ifmedia_add(&sc->xlpge_mii.mii_media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&sc->xlpge_mii.mii_media, IFM_ETHER | IFM_AUTO); sc->xlpge_mii.mii_media.ifm_media = sc->xlpge_mii.mii_media.ifm_cur->ifm_media; xlpge_read_mac_addr(sc); ether_ifattach(ifp, sc->dev_addr); /* override if_transmit : per ifnet(9), do it after if_attach */ ifp->if_transmit = xlpge_tx; return (0); } static int nlm_xlpge_probe(device_t dev) { return (BUS_PROBE_DEFAULT); } static void * get_buf(void) { struct mbuf *m_new; uint64_t *md; #ifdef INVARIANTS vm_paddr_t temp1, temp2; #endif if ((m_new = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR)) == NULL) return (NULL); m_new->m_len = m_new->m_pkthdr.len = MCLBYTES; KASSERT(((uintptr_t)m_new->m_data & (NAE_CACHELINE_SIZE - 1)) == 0, ("m_new->m_data is not cacheline aligned")); md = (uint64_t *)m_new->m_data; md[0] = (intptr_t)m_new; /* Back Ptr */ md[1] = 0xf00bad; m_adj(m_new, NAE_CACHELINE_SIZE); #ifdef INVARIANTS temp1 = vtophys((vm_offset_t) m_new->m_data); temp2 = vtophys((vm_offset_t) m_new->m_data + 1536); KASSERT((temp1 + 1536) == temp2, ("Alloced buffer is not contiguous")); #endif return ((void *)m_new->m_data); } static void nlm_xlpge_mii_init(device_t dev, struct nlm_xlpge_softc *sc) { int error; error = mii_attach(dev, &sc->mii_bus, sc->xlpge_if, xlpge_mediachange, xlpge_mediastatus, BMSR_DEFCAPMASK, sc->phy_addr, MII_OFFSET_ANY, 0); if (error) { device_printf(dev, "attaching PHYs failed\n"); sc->mii_bus = NULL; } if (sc->mii_bus != NULL) { /* enable MDIO interrupts in the PHY */ /* XXXJC: TODO */ } } static int xlpge_stats_sysctl(SYSCTL_HANDLER_ARGS) { struct nlm_xlpge_softc *sc; uint32_t val; int reg, field; sc = arg1; field = arg2; reg = SGMII_STATS_MLR(sc->block, sc->port) + field; val = nlm_read_nae_reg(sc->base_addr, reg); return (sysctl_handle_int(oidp, &val, 0, req)); } static void nlm_xlpge_setup_stats_sysctl(device_t dev, struct nlm_xlpge_softc *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *child; struct sysctl_oid *tree; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); #define XLPGE_STAT(name, offset, desc) \ SYSCTL_ADD_PROC(ctx, child, OID_AUTO, name, \ CTLTYPE_UINT | CTLFLAG_RD, sc, offset, \ xlpge_stats_sysctl, "IU", desc) XLPGE_STAT("tr127", nlm_sgmii_stats_tr127, "TxRx 64 - 127 Bytes"); XLPGE_STAT("tr255", nlm_sgmii_stats_tr255, "TxRx 128 - 255 Bytes"); XLPGE_STAT("tr511", nlm_sgmii_stats_tr511, "TxRx 256 - 511 Bytes"); XLPGE_STAT("tr1k", nlm_sgmii_stats_tr1k, "TxRx 512 - 1023 Bytes"); XLPGE_STAT("trmax", nlm_sgmii_stats_trmax, "TxRx 1024 - 1518 Bytes"); XLPGE_STAT("trmgv", nlm_sgmii_stats_trmgv, "TxRx 1519 - 1522 Bytes"); XLPGE_STAT("rbyt", nlm_sgmii_stats_rbyt, "Rx Bytes"); XLPGE_STAT("rpkt", nlm_sgmii_stats_rpkt, "Rx Packets"); XLPGE_STAT("rfcs", nlm_sgmii_stats_rfcs, "Rx FCS Error"); XLPGE_STAT("rmca", nlm_sgmii_stats_rmca, "Rx Multicast Packets"); XLPGE_STAT("rbca", nlm_sgmii_stats_rbca, "Rx Broadcast Packets"); XLPGE_STAT("rxcf", nlm_sgmii_stats_rxcf, "Rx Control Frames"); XLPGE_STAT("rxpf", nlm_sgmii_stats_rxpf, "Rx Pause Frames"); XLPGE_STAT("rxuo", nlm_sgmii_stats_rxuo, "Rx Unknown Opcode"); XLPGE_STAT("raln", nlm_sgmii_stats_raln, "Rx Alignment Errors"); XLPGE_STAT("rflr", nlm_sgmii_stats_rflr, "Rx Framelength Errors"); XLPGE_STAT("rcde", nlm_sgmii_stats_rcde, "Rx Code Errors"); XLPGE_STAT("rcse", nlm_sgmii_stats_rcse, "Rx Carrier Sense Errors"); XLPGE_STAT("rund", nlm_sgmii_stats_rund, "Rx Undersize Packet Errors"); XLPGE_STAT("rovr", nlm_sgmii_stats_rovr, "Rx Oversize Packet Errors"); XLPGE_STAT("rfrg", nlm_sgmii_stats_rfrg, "Rx Fragments"); XLPGE_STAT("rjbr", nlm_sgmii_stats_rjbr, "Rx Jabber"); XLPGE_STAT("tbyt", nlm_sgmii_stats_tbyt, "Tx Bytes"); XLPGE_STAT("tpkt", nlm_sgmii_stats_tpkt, "Tx Packets"); XLPGE_STAT("tmca", nlm_sgmii_stats_tmca, "Tx Multicast Packets"); XLPGE_STAT("tbca", nlm_sgmii_stats_tbca, "Tx Broadcast Packets"); XLPGE_STAT("txpf", nlm_sgmii_stats_txpf, "Tx Pause Frame"); XLPGE_STAT("tdfr", nlm_sgmii_stats_tdfr, "Tx Deferral Packets"); XLPGE_STAT("tedf", nlm_sgmii_stats_tedf, "Tx Excessive Deferral Pkts"); XLPGE_STAT("tscl", nlm_sgmii_stats_tscl, "Tx Single Collisions"); XLPGE_STAT("tmcl", nlm_sgmii_stats_tmcl, "Tx Multiple Collisions"); XLPGE_STAT("tlcl", nlm_sgmii_stats_tlcl, "Tx Late Collision Pkts"); XLPGE_STAT("txcl", nlm_sgmii_stats_txcl, "Tx Excessive Collisions"); XLPGE_STAT("tncl", nlm_sgmii_stats_tncl, "Tx Total Collisions"); XLPGE_STAT("tjbr", nlm_sgmii_stats_tjbr, "Tx Jabber Frames"); XLPGE_STAT("tfcs", nlm_sgmii_stats_tfcs, "Tx FCS Errors"); XLPGE_STAT("txcf", nlm_sgmii_stats_txcf, "Tx Control Frames"); XLPGE_STAT("tovr", nlm_sgmii_stats_tovr, "Tx Oversize Frames"); XLPGE_STAT("tund", nlm_sgmii_stats_tund, "Tx Undersize Frames"); XLPGE_STAT("tfrg", nlm_sgmii_stats_tfrg, "Tx Fragments"); #undef XLPGE_STAT } static int nlm_xlpge_attach(device_t dev) { struct xlp_port_ivars *pv; struct nlm_xlpge_softc *sc; int port; pv = device_get_ivars(dev); sc = device_get_softc(dev); sc->xlpge_dev = dev; sc->mii_bus = NULL; sc->block = pv->block; sc->node = pv->node; sc->port = pv->port; sc->type = pv->type; sc->xlpge_if = NULL; sc->phy_addr = pv->phy_addr; sc->mdio_bus = pv->mdio_bus; sc->portcfg = nae_port_config; sc->hw_parser_en = pv->hw_parser_en; /* default settings */ sc->speed = NLM_SGMII_SPEED_10; sc->duplexity = NLM_SGMII_DUPLEX_FULL; sc->link = NLM_LINK_DOWN; sc->flowctrl = NLM_FLOWCTRL_DISABLED; sc->network_sc = device_get_softc(device_get_parent(dev)); sc->base_addr = sc->network_sc->base; sc->prepad_en = sc->network_sc->prepad_en; sc->prepad_size = sc->network_sc->prepad_size; callout_init(&sc->xlpge_callout, 1); XLPGE_LOCK_INIT(sc, device_get_nameunit(dev)); port = (sc->block*4)+sc->port; sc->nfree_desc = nae_port_config[port].num_free_descs; sc->txq = nae_port_config[port].txq; sc->rxfreeq = nae_port_config[port].rxfreeq; nlm_xlpge_submit_rx_free_desc(sc, sc->nfree_desc); if (sc->hw_parser_en) nlm_enable_hardware_parser_per_port(sc->base_addr, sc->block, sc->port); nlm_xlpge_ifinit(sc); ifp_ports[port].xlpge_sc = sc; nlm_xlpge_mii_init(dev, sc); nlm_xlpge_setup_stats_sysctl(dev, sc); return (0); } static int nlm_xlpge_detach(device_t dev) { return (0); } static int nlm_xlpge_suspend(device_t dev) { return (0); } static int nlm_xlpge_resume(device_t dev) { return (0); } static int nlm_xlpge_shutdown(device_t dev) { return (0); } /* * miibus function with custom implementation */ static int nlm_xlpge_mii_read(struct device *dev, int phyaddr, int regidx) { struct nlm_xlpge_softc *sc; int val; sc = device_get_softc(dev); if (sc->type == SGMIIC) val = nlm_gmac_mdio_read(sc->base_addr, sc->mdio_bus, BLOCK_7, LANE_CFG, phyaddr, regidx); else val = 0xffff; return (val); } static int nlm_xlpge_mii_write(struct device *dev, int phyaddr, int regidx, int val) { struct nlm_xlpge_softc *sc; sc = device_get_softc(dev); if (sc->type == SGMIIC) nlm_gmac_mdio_write(sc->base_addr, sc->mdio_bus, BLOCK_7, LANE_CFG, phyaddr, regidx, val); return (0); } static void nlm_xlpge_mii_statchg(device_t dev) { struct nlm_xlpge_softc *sc; struct mii_data *mii; char *speed, *duplexity; sc = device_get_softc(dev); if (sc->mii_bus == NULL) return; mii = device_get_softc(sc->mii_bus); if (mii->mii_media_status & IFM_ACTIVE) { if (IFM_SUBTYPE(mii->mii_media_active) == IFM_10_T) { sc->speed = NLM_SGMII_SPEED_10; speed = "10Mbps"; } else if (IFM_SUBTYPE(mii->mii_media_active) == IFM_100_TX) { sc->speed = NLM_SGMII_SPEED_100; speed = "100Mbps"; } else { /* default to 1G */ sc->speed = NLM_SGMII_SPEED_1000; speed = "1Gbps"; } if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) { sc->duplexity = NLM_SGMII_DUPLEX_FULL; duplexity = "full"; } else { sc->duplexity = NLM_SGMII_DUPLEX_HALF; duplexity = "half"; } printf("Port [%d, %d] setup with speed=%s duplex=%s\n", sc->block, sc->port, speed, duplexity); nlm_nae_setup_mac(sc->base_addr, sc->block, sc->port, 0, 1, 1, sc->speed, sc->duplexity); } } /* * xlpge support function implementations */ static void nlm_xlpge_release_mbuf(uint64_t paddr) { uint64_t mag, desc, mbuf; paddr += (XLP_NTXFRAGS - 3) * sizeof(uint64_t); mag = nlm_paddr_ld(paddr); desc = nlm_paddr_ld(paddr + sizeof(uint64_t)); mbuf = nlm_paddr_ld(paddr + 2 * sizeof(uint64_t)); if (mag != 0xf00bad) { /* somebody else packet Error - FIXME in intialization */ printf("cpu %d: ERR Tx packet paddr %jx, mag %jx, desc %jx mbuf %jx\n", nlm_cpuid(), (uintmax_t)paddr, (uintmax_t)mag, (intmax_t)desc, (uintmax_t)mbuf); return; } m_freem((struct mbuf *)(uintptr_t)mbuf); uma_zfree(nl_tx_desc_zone, (void *)(uintptr_t)desc); } static void nlm_xlpge_rx(struct nlm_xlpge_softc *sc, int port, vm_paddr_t paddr, int len) { struct ifnet *ifp; struct mbuf *m; vm_offset_t temp; unsigned long mag; int prepad_size; ifp = sc->xlpge_if; temp = nlm_paddr_ld(paddr - NAE_CACHELINE_SIZE); mag = nlm_paddr_ld(paddr - NAE_CACHELINE_SIZE + sizeof(uint64_t)); m = (struct mbuf *)(intptr_t)temp; if (mag != 0xf00bad) { /* somebody else packet Error - FIXME in intialization */ printf("cpu %d: ERR Rx packet paddr %jx, temp %p, mag %lx\n", nlm_cpuid(), (uintmax_t)paddr, (void *)temp, mag); return; } m->m_pkthdr.rcvif = ifp; #ifdef DUMP_PACKET { int i = 0, j = 64; unsigned char *buf = (char *)m->m_data; printf("(cpu_%d: nlge_rx, !RX_COPY) Rx Packet: length=%d\n", nlm_cpuid(), len); if (len < j) j = len; if (sc->prepad_en) j += ((sc->prepad_size + 1) * 16); for (i = 0; i < j; i++) { if (i && (i % 16) == 0) printf("\n"); printf("%02x ", buf[i]); } printf("\n"); } #endif if (sc->prepad_en) { prepad_size = ((sc->prepad_size + 1) * 16); m->m_data += prepad_size; m->m_pkthdr.len = m->m_len = (len - prepad_size); } else m->m_pkthdr.len = m->m_len = len; if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); #ifdef XLP_DRIVER_LOOPBACK if (port == 16 || port == 17) (*ifp->if_input)(ifp, m); else xlpge_tx(ifp, m); #else (*ifp->if_input)(ifp, m); #endif } void nlm_xlpge_submit_rx_free_desc(struct nlm_xlpge_softc *sc, int num) { int i, size, ret, n; struct nlm_fmn_msg msg; void *ptr; for(i = 0; i < num; i++) { memset(&msg, 0, sizeof(msg)); ptr = get_buf(); if (!ptr) { device_printf(sc->xlpge_dev, "Cannot allocate mbuf\n"); break; } msg.msg[0] = vtophys(ptr); if (msg.msg[0] == 0) { printf("Bad ptr for %p\n", ptr); break; } size = 1; n = 0; while (1) { /* on success returns 1, else 0 */ ret = nlm_fmn_msgsend(sc->rxfreeq, size, 0, &msg); if (ret == 0) break; if (n++ > 10000) { printf("Too many credit fails for send free desc\n"); break; } } } } void nlm_xlpge_msgring_handler(int vc, int size, int code, int src_id, struct nlm_fmn_msg *msg, void *data) { uint64_t phys_addr; struct nlm_xlpnae_softc *sc; struct nlm_xlpge_softc *xlpge_sc; struct ifnet *ifp; uint32_t context; uint32_t port = 0; uint32_t length; sc = (struct nlm_xlpnae_softc *)data; KASSERT(sc != NULL, ("Null sc in msgring handler")); if (size == 1) { /* process transmit complete */ phys_addr = msg->msg[0] & 0xffffffffffULL; /* context is SGMII_RCV_CONTEXT_NUM + three bit vlan type * or vlan priority */ context = (msg->msg[0] >> 40) & 0x3fff; port = cntx2port[context]; if (port >= XLP_MAX_PORTS) { printf("%s:%d Bad port %d (context=%d)\n", __func__, __LINE__, port, context); return; } ifp = ifp_ports[port].xlpge_if; xlpge_sc = ifp_ports[port].xlpge_sc; nlm_xlpge_release_mbuf(phys_addr); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); } else if (size > 1) { /* Recieve packet */ phys_addr = msg->msg[1] & 0xffffffffc0ULL; length = (msg->msg[1] >> 40) & 0x3fff; length -= MAC_CRC_LEN; /* context is SGMII_RCV_CONTEXT_NUM + three bit vlan type * or vlan priority */ context = (msg->msg[1] >> 54) & 0x3ff; port = cntx2port[context]; if (port >= XLP_MAX_PORTS) { printf("%s:%d Bad port %d (context=%d)\n", __func__, __LINE__, port, context); return; } ifp = ifp_ports[port].xlpge_if; xlpge_sc = ifp_ports[port].xlpge_sc; nlm_xlpge_rx(xlpge_sc, port, phys_addr, length); /* return back a free descriptor to NA */ nlm_xlpge_submit_rx_free_desc(xlpge_sc, 1); } } Index: head/sys/mips/rmi/dev/nlge/if_nlge.c =================================================================== --- head/sys/mips/rmi/dev/nlge/if_nlge.c (revision 295880) +++ head/sys/mips/rmi/dev/nlge/if_nlge.c (revision 295881) @@ -1,2564 +1,2563 @@ /*- * Copyright (c) 2003-2009 RMI Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of RMI Corporation, nor the names of its contributors, * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RMI_BSD */ /* * The XLR device supports upto four 10/100/1000 Ethernet MACs and upto * two 10G Ethernet MACs (of XGMII). Alternatively, each 10G port can used * as a SPI-4 interface, with 8 ports per such interface. The MACs are * encapsulated in another hardware block referred to as network accelerator, * such that there are three instances of these in a XLR. One of them controls * the four 1G RGMII ports while one each of the others controls an XGMII port. * Enabling MACs requires configuring the corresponding network accelerator * and the individual port. * The XLS device supports upto 8 10/100/1000 Ethernet MACs or max 2 10G * Ethernet MACs. The 1G MACs are of SGMII and 10G MACs are of XAUI * interface. These ports are part of two network accelerators. * The nlge driver configures and initializes non-SPI4 Ethernet ports in the * XLR/XLS devices and enables data transfer on them. */ #include __FBSDID("$FreeBSD$"); #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_device_polling.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define __RMAN_RESOURCE_VISIBLE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include /* for DELAY */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "miidevs.h" #include #include "miibus_if.h" #include MODULE_DEPEND(nlna, nlge, 1, 1, 1); MODULE_DEPEND(nlge, ether, 1, 1, 1); MODULE_DEPEND(nlge, miibus, 1, 1, 1); /* Network accelarator entry points */ static int nlna_probe(device_t); static int nlna_attach(device_t); static int nlna_detach(device_t); static int nlna_suspend(device_t); static int nlna_resume(device_t); static int nlna_shutdown(device_t); /* GMAC port entry points */ static int nlge_probe(device_t); static int nlge_attach(device_t); static int nlge_detach(device_t); static int nlge_suspend(device_t); static int nlge_resume(device_t); static void nlge_init(void *); static int nlge_ioctl(struct ifnet *, u_long, caddr_t); static int nlge_tx(struct ifnet *ifp, struct mbuf *m); static void nlge_rx(struct nlge_softc *sc, vm_paddr_t paddr, int len); static int nlge_mii_write(struct device *, int, int, int); static int nlge_mii_read(struct device *, int, int); static void nlge_mac_mii_statchg(device_t); static int nlge_mediachange(struct ifnet *ifp); static void nlge_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr); /* Other internal/helper functions */ static void *get_buf(void); static void nlna_add_to_port_set(struct nlge_port_set *pset, struct nlge_softc *sc); static void nlna_config_pde(struct nlna_softc *); static void nlna_config_parser(struct nlna_softc *); static void nlna_config_classifier(struct nlna_softc *); static void nlna_config_fifo_spill_area(struct nlna_softc *sc); static void nlna_config_translate_table(struct nlna_softc *sc); static void nlna_config_common(struct nlna_softc *); static void nlna_disable_ports(struct nlna_softc *sc); static void nlna_enable_intr(struct nlna_softc *sc); static void nlna_disable_intr(struct nlna_softc *sc); static void nlna_enable_ports(struct nlna_softc *sc); static void nlna_get_all_softc(device_t iodi_dev, struct nlna_softc **sc_vec, uint32_t vec_sz); static void nlna_hw_init(struct nlna_softc *sc); static int nlna_is_last_active_na(struct nlna_softc *sc); static void nlna_media_specific_config(struct nlna_softc *sc); static void nlna_reset_ports(struct nlna_softc *sc, struct xlr_gmac_block_t *blk); static struct nlna_softc *nlna_sc_init(device_t dev, struct xlr_gmac_block_t *blk); static void nlna_setup_intr(struct nlna_softc *sc); static void nlna_smp_update_pde(void *dummy __unused); static void nlna_submit_rx_free_desc(struct nlna_softc *sc, uint32_t n_desc); static int nlge_gmac_config_speed(struct nlge_softc *, int quick); static void nlge_hw_init(struct nlge_softc *sc); static int nlge_if_init(struct nlge_softc *sc); static void nlge_intr(void *arg); static int nlge_irq_init(struct nlge_softc *sc); static void nlge_irq_fini(struct nlge_softc *sc); static void nlge_media_specific_init(struct nlge_softc *sc); static void nlge_mii_init(device_t dev, struct nlge_softc *sc); static int nlge_mii_read_internal(xlr_reg_t *mii_base, int phyaddr, int regidx); static void nlge_mii_write_internal(xlr_reg_t *mii_base, int phyaddr, int regidx, int regval); void nlge_msgring_handler(int bucket, int size, int code, int stid, struct msgrng_msg *msg, void *data); static void nlge_port_disable(struct nlge_softc *sc); static void nlge_port_enable(struct nlge_softc *sc); static void nlge_read_mac_addr(struct nlge_softc *sc); static void nlge_sc_init(struct nlge_softc *sc, device_t dev, struct xlr_gmac_port *port_info); static void nlge_set_mac_addr(struct nlge_softc *sc); static void nlge_set_port_attribs(struct nlge_softc *, struct xlr_gmac_port *); static void nlge_mac_set_rx_mode(struct nlge_softc *sc); static void nlge_sgmii_init(struct nlge_softc *sc); static int nlge_start_locked(struct ifnet *ifp, struct nlge_softc *sc, struct mbuf *m); static int prepare_fmn_message(struct nlge_softc *sc, struct msgrng_msg *msg, uint32_t *n_entries, struct mbuf *m_head, uint64_t fr_stid, struct nlge_tx_desc **tx_desc); static void release_tx_desc(vm_paddr_t phy_addr); static int send_fmn_msg_tx(struct nlge_softc *, struct msgrng_msg *, uint32_t n_entries); //#define DEBUG #ifdef DEBUG static int mac_debug = 1; #undef PDEBUG #define PDEBUG(fmt, args...) \ do {\ if (mac_debug) {\ printf("[%s@%d|%s]: cpu_%d: " fmt, \ __FILE__, __LINE__, __FUNCTION__, PCPU_GET(cpuid), ##args);\ }\ } while(0); /* Debug/dump functions */ static void dump_reg(xlr_reg_t *addr, uint32_t offset, char *name); static void dump_gmac_registers(struct nlge_softc *); static void dump_na_registers(xlr_reg_t *base, int port_id); static void dump_mac_stats(struct nlge_softc *sc); static void dump_mii_regs(struct nlge_softc *sc) __attribute__((used)); static void dump_mii_data(struct mii_data *mii) __attribute__((used)); static void dump_board_info(struct xlr_board_info *); static void dump_pcs_regs(struct nlge_softc *sc, int phy); #else #undef PDEBUG #define PDEBUG(fmt, args...) #define dump_reg(a, o, n) /* nop */ #define dump_gmac_registers(a) /* nop */ #define dump_na_registers(a, p) /* nop */ #define dump_board_info(b) /* nop */ #define dump_mac_stats(sc) /* nop */ #define dump_mii_regs(sc) /* nop */ #define dump_mii_data(mii) /* nop */ #define dump_pcs_regs(sc, phy) /* nop */ #endif /* Wrappers etc. to export the driver entry points. */ static device_method_t nlna_methods[] = { /* Device interface */ DEVMETHOD(device_probe, nlna_probe), DEVMETHOD(device_attach, nlna_attach), DEVMETHOD(device_detach, nlna_detach), DEVMETHOD(device_shutdown, nlna_shutdown), DEVMETHOD(device_suspend, nlna_suspend), DEVMETHOD(device_resume, nlna_resume), /* bus interface : TBD : what are these for ? */ DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD_END }; static driver_t nlna_driver = { "nlna", nlna_methods, sizeof(struct nlna_softc) }; static devclass_t nlna_devclass; static device_method_t nlge_methods[] = { /* Device interface */ DEVMETHOD(device_probe, nlge_probe), DEVMETHOD(device_attach, nlge_attach), DEVMETHOD(device_detach, nlge_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, nlge_suspend), DEVMETHOD(device_resume, nlge_resume), /* MII interface */ DEVMETHOD(miibus_readreg, nlge_mii_read), DEVMETHOD(miibus_writereg, nlge_mii_write), DEVMETHOD(miibus_statchg, nlge_mac_mii_statchg), {0, 0} }; static driver_t nlge_driver = { "nlge", nlge_methods, sizeof(struct nlge_softc) }; static devclass_t nlge_devclass; DRIVER_MODULE(nlna, iodi, nlna_driver, nlna_devclass, 0, 0); DRIVER_MODULE(nlge, nlna, nlge_driver, nlge_devclass, 0, 0); DRIVER_MODULE(miibus, nlge, miibus_driver, miibus_devclass, 0, 0); static uma_zone_t nl_tx_desc_zone; /* Tunables. */ static int flow_classification = 0; TUNABLE_INT("hw.nlge.flow_classification", &flow_classification); #define NLGE_HW_CHKSUM 1 static __inline void atomic_incr_long(unsigned long *addr) { /* XXX: fix for 64 bit */ unsigned int *iaddr = (unsigned int *)addr; xlr_ldaddwu(1, iaddr); } static int nlna_probe(device_t dev) { return (BUS_PROBE_DEFAULT); } /* * Add all attached GMAC/XGMAC ports to the device tree. Port * configuration is spread in two regions - common configuration * for all ports in the NA and per-port configuration in MAC-specific * region. This function does the following: * - adds the ports to the device tree * - reset the ports * - do all the common initialization * - invoke bus_generic_attach for per-port configuration * - supply initial free rx descriptors to ports * - initialize s/w data structures * - finally, enable interrupts (only in the last NA). * * For reference, sample address space for common and per-port * registers is given below. * * The address map for RNA0 is: (typical value) * * XLR_IO_BASE +--------------------------------------+ 0xbef0_0000 * | | * | | * | | * | | * | | * | | * GMAC0 ---> +--------------------------------------+ 0xbef0_c000 * | | * | | * (common) -> |......................................| 0xbef0_c400 * | | * | (RGMII/SGMII: common registers) | * | | * GMAC1 ---> |--------------------------------------| 0xbef0_d000 * | | * | | * (common) -> |......................................| 0xbef0_d400 * | | * | (RGMII/SGMII: common registers) | * | | * |......................................| * and so on .... * * Ref: Figure 14-3 and Table 14-1 of XLR PRM */ static int nlna_attach(device_t dev) { struct xlr_gmac_block_t *block_info; device_t gmac_dev; struct nlna_softc *sc; int error; int i; int id; id = device_get_unit(dev); block_info = device_get_ivars(dev); if (!block_info->enabled) { return 0; } #ifdef DEBUG dump_board_info(&xlr_board_info); #endif /* Initialize nlna state in softc structure */ sc = nlna_sc_init(dev, block_info); /* Add device's for the ports controlled by this NA. */ if (block_info->type == XLR_GMAC) { KASSERT(id < 2, ("No GMACs supported with this network" "accelerator: %d", id)); for (i = 0; i < sc->num_ports; i++) { gmac_dev = device_add_child(dev, "nlge", -1); device_set_ivars(gmac_dev, &block_info->gmac_port[i]); } } else if (block_info->type == XLR_XGMAC) { KASSERT(id > 0 && id <= 2, ("No XGMACs supported with this" "network accelerator: %d", id)); gmac_dev = device_add_child(dev, "nlge", -1); device_set_ivars(gmac_dev, &block_info->gmac_port[0]); } else if (block_info->type == XLR_SPI4) { /* SPI4 is not supported here */ device_printf(dev, "Unsupported: NA with SPI4 type"); return (ENOTSUP); } nlna_reset_ports(sc, block_info); /* Initialize Network Accelarator registers. */ nlna_hw_init(sc); error = bus_generic_attach(dev); if (error) { device_printf(dev, "failed to attach port(s)\n"); goto fail; } /* Send out the initial pool of free-descriptors for the rx path */ nlna_submit_rx_free_desc(sc, MAX_FRIN_SPILL); /* S/w data structure initializations shared by all NA's. */ if (nl_tx_desc_zone == NULL) { /* Create a zone for allocating tx descriptors */ nl_tx_desc_zone = uma_zcreate("NL Tx Desc", sizeof(struct nlge_tx_desc), NULL, NULL, NULL, NULL, XLR_CACHELINE_SIZE, 0); } /* Enable NA interrupts */ nlna_setup_intr(sc); return (0); fail: return (error); } static int nlna_detach(device_t dev) { struct nlna_softc *sc; sc = device_get_softc(dev); if (device_is_alive(dev)) { nlna_disable_intr(sc); /* This will make sure that per-port detach is complete * and all traffic on the ports has been stopped. */ bus_generic_detach(dev); uma_zdestroy(nl_tx_desc_zone); } return (0); } static int nlna_suspend(device_t dev) { return (0); } static int nlna_resume(device_t dev) { return (0); } static int nlna_shutdown(device_t dev) { return (0); } /* GMAC port entry points */ static int nlge_probe(device_t dev) { struct nlge_softc *sc; struct xlr_gmac_port *port_info; int index; char *desc[] = { "RGMII", "SGMII", "RGMII/SGMII", "XGMAC", "XAUI", "Unknown"}; port_info = device_get_ivars(dev); index = (port_info->type < XLR_RGMII || port_info->type > XLR_XAUI) ? 5 : port_info->type; device_set_desc_copy(dev, desc[index]); sc = device_get_softc(dev); nlge_sc_init(sc, dev, port_info); nlge_port_disable(sc); return (0); } static int nlge_attach(device_t dev) { struct nlge_softc *sc; struct nlna_softc *nsc; int error; sc = device_get_softc(dev); nlge_if_init(sc); nlge_mii_init(dev, sc); error = nlge_irq_init(sc); if (error) return error; nlge_hw_init(sc); nsc = (struct nlna_softc *)device_get_softc(device_get_parent(dev)); nsc->child_sc[sc->instance] = sc; return (0); } static int nlge_detach(device_t dev) { struct nlge_softc *sc; struct ifnet *ifp; sc = device_get_softc(dev); ifp = sc->nlge_if; if (device_is_attached(dev)) { nlge_port_disable(sc); nlge_irq_fini(sc); ether_ifdetach(ifp); bus_generic_detach(dev); } if (ifp) if_free(ifp); return (0); } static int nlge_suspend(device_t dev) { return (0); } static int nlge_resume(device_t dev) { return (0); } static void nlge_init(void *addr) { struct nlge_softc *sc; struct ifnet *ifp; sc = (struct nlge_softc *)addr; ifp = sc->nlge_if; if (ifp->if_drv_flags & IFF_DRV_RUNNING) return; nlge_gmac_config_speed(sc, 1); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; nlge_port_enable(sc); if (sc->port_type == XLR_SGMII) { dump_pcs_regs(sc, 27); } dump_gmac_registers(sc); dump_mac_stats(sc); } static int nlge_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct mii_data *mii; struct nlge_softc *sc; struct ifreq *ifr; int error; sc = ifp->if_softc; error = 0; ifr = (struct ifreq *)data; switch(command) { case SIOCSIFFLAGS: NLGE_LOCK(sc); if (ifp->if_flags & IFF_UP) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { nlge_init(sc); } if (ifp->if_flags & IFF_PROMISC && !(sc->if_flags & IFF_PROMISC)) { sc->if_flags |= IFF_PROMISC; nlge_mac_set_rx_mode(sc); } else if (!(ifp->if_flags & IFF_PROMISC) && sc->if_flags & IFF_PROMISC) { sc->if_flags &= IFF_PROMISC; nlge_mac_set_rx_mode(sc); } } else { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { nlge_port_disable(sc); } } sc->if_flags = ifp->if_flags; NLGE_UNLOCK(sc); error = 0; break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: if (sc->mii_bus != NULL) { mii = (struct mii_data *)device_get_softc(sc->mii_bus); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command); } break; default: error = ether_ioctl(ifp, command, data); break; } return (error); } /* This function is called from an interrupt handler */ void nlge_msgring_handler(int bucket, int size, int code, int stid, struct msgrng_msg *msg, void *data) { struct nlna_softc *na_sc; struct nlge_softc *sc; struct ifnet *ifp; struct mbuf *m; vm_paddr_t phys_addr; uint32_t length; int ctrl; int tx_error; int port; int is_p2p; is_p2p = 0; tx_error = 0; length = (msg->msg0 >> 40) & 0x3fff; na_sc = (struct nlna_softc *)data; if (length == 0) { ctrl = CTRL_REG_FREE; phys_addr = msg->msg0 & 0xffffffffffULL; port = (msg->msg0 >> 54) & 0x0f; is_p2p = (msg->msg0 >> 62) & 0x1; tx_error = (msg->msg0 >> 58) & 0xf; } else { ctrl = CTRL_SNGL; phys_addr = msg->msg0 & 0xffffffffe0ULL; length = length - BYTE_OFFSET - MAC_CRC_LEN; port = msg->msg0 & 0x0f; } sc = na_sc->child_sc[port]; if (sc == NULL) { printf("Message (of %d len) with softc=NULL on %d port (type=%s)\n", length, port, (ctrl == CTRL_SNGL ? "Pkt rx" : "Freeback for tx packet")); return; } if (ctrl == CTRL_REG_FREE || ctrl == CTRL_JUMBO_FREE) { ifp = sc->nlge_if; if (!tx_error) { if (is_p2p) { release_tx_desc(phys_addr); } else { #ifdef __mips_n64 m = (struct mbuf *)(uintptr_t)xlr_paddr_ld(phys_addr); m->m_nextpkt = NULL; #else m = (struct mbuf *)(uintptr_t)phys_addr; #endif m_freem(m); } NLGE_LOCK(sc); if (ifp->if_drv_flags & IFF_DRV_OACTIVE){ ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; } NLGE_UNLOCK(sc); } else { printf("ERROR: Tx fb error (%d) on port %d\n", tx_error, port); } tx_error ? if_inc_counter(ifp, IFCOUNTER_OERRORS, 1) : if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); } else if (ctrl == CTRL_SNGL || ctrl == CTRL_START) { /* Rx Packet */ nlge_rx(sc, phys_addr, length); nlna_submit_rx_free_desc(na_sc, 1); /* return free descr to NA */ } else { printf("[%s]: unrecognized ctrl=%d!\n", __func__, ctrl); } } static int nlge_tx(struct ifnet *ifp, struct mbuf *m) { return (nlge_start_locked(ifp, ifp->if_softc, m)); } static int nlge_start_locked(struct ifnet *ifp, struct nlge_softc *sc, struct mbuf *m) { struct msgrng_msg msg; struct nlge_tx_desc *tx_desc; uint64_t fr_stid; uint32_t cpu; uint32_t n_entries; uint32_t tid; int error, ret; if (m == NULL) return (0); tx_desc = NULL; error = 0; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING) || ifp->if_drv_flags & IFF_DRV_OACTIVE) { error = ENXIO; goto fail; // note: mbuf will get free'd } cpu = xlr_core_id(); tid = xlr_thr_id(); /* H/w threads [0, 2] --> bucket 6 and [1, 3] --> bucket 7 */ fr_stid = cpu * 8 + 6 + (tid % 2); /* * First, remove some freeback messages before transmitting * any new packets. However, cap the number of messages * drained to permit this thread to continue with its * transmission. * * Mask for buckets {6, 7} is 0xc0 */ xlr_msgring_handler(0xc0, 4); ret = prepare_fmn_message(sc, &msg, &n_entries, m, fr_stid, &tx_desc); if (ret) { error = (ret == 2) ? ENOBUFS : ENOTSUP; goto fail; } ret = send_fmn_msg_tx(sc, &msg, n_entries); if (ret != 0) { error = EBUSY; goto fail; } return (0); fail: if (tx_desc != NULL) { uma_zfree(nl_tx_desc_zone, tx_desc); } if (m != NULL) { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { NLGE_LOCK(sc); ifp->if_drv_flags |= IFF_DRV_OACTIVE; NLGE_UNLOCK(sc); } m_freem(m); if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); } return (error); } static void nlge_rx(struct nlge_softc *sc, vm_paddr_t paddr, int len) { struct ifnet *ifp; struct mbuf *m; uint64_t tm, mag; uint32_t sr; sr = xlr_enable_kx(); tm = xlr_paddr_ld(paddr - XLR_CACHELINE_SIZE); mag = xlr_paddr_ld(paddr - XLR_CACHELINE_SIZE + sizeof(uint64_t)); xlr_restore_kx(sr); m = (struct mbuf *)(intptr_t)tm; if (mag != 0xf00bad) { /* somebody else's packet. Error - FIXME in intialization */ printf("cpu %d: *ERROR* Not my packet paddr %jx\n", xlr_core_id(), (uintmax_t)paddr); return; } ifp = sc->nlge_if; #ifdef NLGE_HW_CHKSUM m->m_pkthdr.csum_flags = CSUM_IP_CHECKED; if (m->m_data[10] & 0x2) { m->m_pkthdr.csum_flags |= CSUM_IP_VALID; if (m->m_data[10] & 0x1) { m->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); m->m_pkthdr.csum_data = htons(0xffff); } } m->m_data += NLGE_PREPAD_LEN; len -= NLGE_PREPAD_LEN; #else m->m_pkthdr.csum_flags = 0; #endif /* align the data */ m->m_data += BYTE_OFFSET ; m->m_pkthdr.len = m->m_len = len; m->m_pkthdr.rcvif = ifp; if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); (*ifp->if_input)(ifp, m); } static int nlge_mii_write(struct device *dev, int phyaddr, int regidx, int regval) { struct nlge_softc *sc; sc = device_get_softc(dev); if (sc->port_type != XLR_XGMII) nlge_mii_write_internal(sc->mii_base, phyaddr, regidx, regval); return (0); } static int nlge_mii_read(struct device *dev, int phyaddr, int regidx) { struct nlge_softc *sc; int val; sc = device_get_softc(dev); val = (sc->port_type == XLR_XGMII) ? (0xffff) : nlge_mii_read_internal(sc->mii_base, phyaddr, regidx); return (val); } static void nlge_mac_mii_statchg(device_t dev) { } static int nlge_mediachange(struct ifnet *ifp) { return 0; } static void nlge_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr) { struct nlge_softc *sc; struct mii_data *md; md = NULL; sc = ifp->if_softc; if (sc->mii_bus) md = device_get_softc(sc->mii_bus); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (sc->link == xlr_mac_link_down) return; if (md != NULL) ifmr->ifm_active = md->mii_media.ifm_cur->ifm_media; ifmr->ifm_status |= IFM_ACTIVE; } static struct nlna_softc * nlna_sc_init(device_t dev, struct xlr_gmac_block_t *blk) { struct nlna_softc *sc; sc = device_get_softc(dev); memset(sc, 0, sizeof(*sc)); sc->nlna_dev = dev; sc->base = xlr_io_mmio(blk->baseaddr); sc->rfrbucket = blk->station_rfr; sc->station_id = blk->station_id; sc->na_type = blk->type; sc->mac_type = blk->mode; sc->num_ports = blk->num_ports; sc->mdio_set.port_vec = sc->mdio_sc; sc->mdio_set.vec_sz = XLR_MAX_MACS; return (sc); } /* * Do: * - Initialize common GMAC registers (index range 0x100-0x3ff). */ static void nlna_hw_init(struct nlna_softc *sc) { /* * Register message ring handler for the NA block, messages from * the GMAC will have source station id to the first bucket of the * NA FMN station, so register just that station id. */ if (register_msgring_handler(sc->station_id, sc->station_id + 1, nlge_msgring_handler, sc)) { panic("Couldn't register msgring handler\n"); } nlna_config_fifo_spill_area(sc); nlna_config_pde(sc); nlna_config_common(sc); nlna_config_parser(sc); nlna_config_classifier(sc); } /* * Enable interrupts on all the ports controlled by this NA. For now, we * only care about the MII interrupt and this has to be enabled only * on the port id0. * * This function is not in-sync with the regular way of doing things - it * executes only in the context of the last active network accelerator (and * thereby has some ugly accesses in the device tree). Though inelegant, it * is necessary to do it this way as the per-port interrupts can be * setup/enabled only after all the network accelerators have been * initialized. */ static void nlna_setup_intr(struct nlna_softc *sc) { struct nlna_softc *na_sc[XLR_MAX_NLNA]; struct nlge_port_set *pset; struct xlr_gmac_port *port_info; device_t iodi_dev; int i, j; if (!nlna_is_last_active_na(sc)) return ; /* Collect all nlna softc pointers */ memset(na_sc, 0, sizeof(*na_sc) * XLR_MAX_NLNA); iodi_dev = device_get_parent(sc->nlna_dev); nlna_get_all_softc(iodi_dev, na_sc, XLR_MAX_NLNA); /* Setup the MDIO interrupt lists. */ /* * MDIO interrupts are coarse - a single interrupt line provides * information about one of many possible ports. To figure out the * exact port on which action is to be taken, all of the ports * linked to an MDIO interrupt should be read. To enable this, * ports need to add themselves to port sets. */ for (i = 0; i < XLR_MAX_NLNA; i++) { if (na_sc[i] == NULL) continue; for (j = 0; j < na_sc[i]->num_ports; j++) { /* processing j-th port on i-th NA */ port_info = device_get_ivars( na_sc[i]->child_sc[j]->nlge_dev); pset = &na_sc[port_info->mdint_id]->mdio_set; nlna_add_to_port_set(pset, na_sc[i]->child_sc[j]); } } /* Enable interrupts */ for (i = 0; i < XLR_MAX_NLNA; i++) { if (na_sc[i] != NULL && na_sc[i]->na_type != XLR_XGMAC) { nlna_enable_intr(na_sc[i]); } } } static void nlna_add_to_port_set(struct nlge_port_set *pset, struct nlge_softc *sc) { int i; /* step past the non-NULL elements */ for (i = 0; i < pset->vec_sz && pset->port_vec[i] != NULL; i++) ; if (i < pset->vec_sz) pset->port_vec[i] = sc; else printf("warning: internal error: out-of-bounds for MDIO array"); } static void nlna_enable_intr(struct nlna_softc *sc) { int i; for (i = 0; i < sc->num_ports; i++) { if (sc->child_sc[i]->instance == 0) NLGE_WRITE(sc->child_sc[i]->base, R_INTMASK, (1 << O_INTMASK__MDInt)); } } static void nlna_disable_intr(struct nlna_softc *sc) { int i; for (i = 0; i < sc->num_ports; i++) { if (sc->child_sc[i]->instance == 0) NLGE_WRITE(sc->child_sc[i]->base, R_INTMASK, 0); } } static int nlna_is_last_active_na(struct nlna_softc *sc) { int id; id = device_get_unit(sc->nlna_dev); return (id == 2 || xlr_board_info.gmac_block[id + 1].enabled == 0); } static void nlna_submit_rx_free_desc(struct nlna_softc *sc, uint32_t n_desc) { struct msgrng_msg msg; void *ptr; uint32_t msgrng_flags; int i, n, stid, ret, code; if (n_desc > 1) { PDEBUG("Sending %d free-in descriptors to station=%d\n", n_desc, sc->rfrbucket); } stid = sc->rfrbucket; code = (sc->na_type == XLR_XGMAC) ? MSGRNG_CODE_XGMAC : MSGRNG_CODE_MAC; memset(&msg, 0, sizeof(msg)); for (i = 0; i < n_desc; i++) { ptr = get_buf(); if (!ptr) { ret = -ENOMEM; device_printf(sc->nlna_dev, "Cannot allocate mbuf\n"); break; } /* Send the free Rx desc to the MAC */ msg.msg0 = vtophys(ptr) & 0xffffffffe0ULL; n = 0; do { msgrng_flags = msgrng_access_enable(); ret = message_send(1, code, stid, &msg); msgrng_restore(msgrng_flags); KASSERT(n++ < 100000, ("Too many credit fails in rx path\n")); } while (ret != 0); } } static __inline__ void * nlna_config_spill(xlr_reg_t *base, int reg_start_0, int reg_start_1, int reg_size, int size) { void *spill; uint64_t phys_addr; uint32_t spill_size; spill_size = size; spill = contigmalloc((spill_size + XLR_CACHELINE_SIZE), M_DEVBUF, M_NOWAIT | M_ZERO, 0, 0xffffffff, XLR_CACHELINE_SIZE, 0); if (spill == NULL || ((vm_offset_t) spill & (XLR_CACHELINE_SIZE - 1))) { panic("Unable to allocate memory for spill area!\n"); } phys_addr = vtophys(spill); PDEBUG("Allocated spill %d bytes at %llx\n", size, phys_addr); NLGE_WRITE(base, reg_start_0, (phys_addr >> 5) & 0xffffffff); NLGE_WRITE(base, reg_start_1, (phys_addr >> 37) & 0x07); NLGE_WRITE(base, reg_size, spill_size); return (spill); } /* * Configure the 6 FIFO's that are used by the network accelarator to * communicate with the rest of the XLx device. 4 of the FIFO's are for * packets from NA --> cpu (called Class FIFO's) and 2 are for feeding * the NA with free descriptors. */ static void nlna_config_fifo_spill_area(struct nlna_softc *sc) { sc->frin_spill = nlna_config_spill(sc->base, R_REG_FRIN_SPILL_MEM_START_0, R_REG_FRIN_SPILL_MEM_START_1, R_REG_FRIN_SPILL_MEM_SIZE, MAX_FRIN_SPILL * sizeof(struct fr_desc)); sc->frout_spill = nlna_config_spill(sc->base, R_FROUT_SPILL_MEM_START_0, R_FROUT_SPILL_MEM_START_1, R_FROUT_SPILL_MEM_SIZE, MAX_FROUT_SPILL * sizeof(struct fr_desc)); sc->class_0_spill = nlna_config_spill(sc->base, R_CLASS0_SPILL_MEM_START_0, R_CLASS0_SPILL_MEM_START_1, R_CLASS0_SPILL_MEM_SIZE, MAX_CLASS_0_SPILL * sizeof(union rx_tx_desc)); sc->class_1_spill = nlna_config_spill(sc->base, R_CLASS1_SPILL_MEM_START_0, R_CLASS1_SPILL_MEM_START_1, R_CLASS1_SPILL_MEM_SIZE, MAX_CLASS_1_SPILL * sizeof(union rx_tx_desc)); sc->class_2_spill = nlna_config_spill(sc->base, R_CLASS2_SPILL_MEM_START_0, R_CLASS2_SPILL_MEM_START_1, R_CLASS2_SPILL_MEM_SIZE, MAX_CLASS_2_SPILL * sizeof(union rx_tx_desc)); sc->class_3_spill = nlna_config_spill(sc->base, R_CLASS3_SPILL_MEM_START_0, R_CLASS3_SPILL_MEM_START_1, R_CLASS3_SPILL_MEM_SIZE, MAX_CLASS_3_SPILL * sizeof(union rx_tx_desc)); } /* Set the CPU buckets that receive packets from the NA class FIFOs. */ static void nlna_config_pde(struct nlna_softc *sc) { uint64_t bucket_map; uint32_t cpumask; int i, cpu, bucket; cpumask = 0x1; #ifdef SMP /* * nlna may be called before SMP start in a BOOTP/NFSROOT * setup. we will distribute packets to other cpus only when * the SMP is started. */ if (smp_started) cpumask = xlr_hw_thread_mask; #endif bucket_map = 0; for (i = 0; i < 32; i++) { if (cpumask & (1 << i)) { cpu = i; /* use bucket 0 and 1 on every core for NA msgs */ bucket = cpu/4 * 8; bucket_map |= (3ULL << bucket); } } NLGE_WRITE(sc->base, R_PDE_CLASS_0, (bucket_map & 0xffffffff)); NLGE_WRITE(sc->base, R_PDE_CLASS_0 + 1, ((bucket_map >> 32) & 0xffffffff)); NLGE_WRITE(sc->base, R_PDE_CLASS_1, (bucket_map & 0xffffffff)); NLGE_WRITE(sc->base, R_PDE_CLASS_1 + 1, ((bucket_map >> 32) & 0xffffffff)); NLGE_WRITE(sc->base, R_PDE_CLASS_2, (bucket_map & 0xffffffff)); NLGE_WRITE(sc->base, R_PDE_CLASS_2 + 1, ((bucket_map >> 32) & 0xffffffff)); NLGE_WRITE(sc->base, R_PDE_CLASS_3, (bucket_map & 0xffffffff)); NLGE_WRITE(sc->base, R_PDE_CLASS_3 + 1, ((bucket_map >> 32) & 0xffffffff)); } /* * Update the network accelerator packet distribution engine for SMP. * On bootup, we have just the boot hw thread handling all packets, on SMP * start, we can start distributing packets across all the cores which are up. */ static void nlna_smp_update_pde(void *dummy __unused) { device_t iodi_dev; struct nlna_softc *na_sc[XLR_MAX_NLNA]; int i; printf("Updating packet distribution for SMP\n"); iodi_dev = devclass_get_device(devclass_find("iodi"), 0); nlna_get_all_softc(iodi_dev, na_sc, XLR_MAX_NLNA); for (i = 0; i < XLR_MAX_NLNA; i++) { if (na_sc[i] == NULL) continue; nlna_disable_ports(na_sc[i]); nlna_config_pde(na_sc[i]); nlna_config_translate_table(na_sc[i]); nlna_enable_ports(na_sc[i]); } } SYSINIT(nlna_smp_update_pde, SI_SUB_SMP, SI_ORDER_ANY, nlna_smp_update_pde, NULL); static void nlna_config_translate_table(struct nlna_softc *sc) { uint32_t cpu_mask; uint32_t val; int bkts[32]; /* one bucket is assumed for each cpu */ int b1, b2, c1, c2, i, j, k; int use_bkt; if (!flow_classification) return; use_bkt = 1; if (smp_started) cpu_mask = xlr_hw_thread_mask; else return; printf("Using %s-based distribution\n", (use_bkt) ? "bucket" : "class"); j = 0; for(i = 0; i < 32; i++) { if ((1 << i) & cpu_mask){ /* for each cpu, mark the 4+threadid bucket */ bkts[j] = ((i / 4) * 8) + (i % 4); j++; } } /*configure the 128 * 9 Translation table to send to available buckets*/ k = 0; c1 = 3; c2 = 0; for(i = 0; i < 64; i++) { /* Get the next 2 pairs of (class, bucket): (c1, b1), (c2, b2). c1, c2 limited to {0, 1, 2, 3} i.e, the 4 classes defined by h/w b1, b2 limited to { bkts[i], where 0 <= i < j} i.e, the set of buckets computed in the above loop. */ c1 = (c1 + 1) & 3; c2 = (c1 + 1) & 3; b1 = bkts[k]; k = (k + 1) % j; b2 = bkts[k]; k = (k + 1) % j; PDEBUG("Translation table[%d] b1=%d b2=%d c1=%d c2=%d\n", i, b1, b2, c1, c2); val = ((c1 << 23) | (b1 << 17) | (use_bkt << 16) | (c2 << 7) | (b2 << 1) | (use_bkt << 0)); NLGE_WRITE(sc->base, R_TRANSLATETABLE + i, val); c1 = c2; } } static void nlna_config_parser(struct nlna_softc *sc) { uint32_t val; /* * Mark it as ETHERNET type. */ NLGE_WRITE(sc->base, R_L2TYPE_0, 0x01); #ifndef NLGE_HW_CHKSUM if (!flow_classification) return; #endif /* Use 7bit CRChash for flow classification with 127 as CRC polynomial*/ NLGE_WRITE(sc->base, R_PARSERCONFIGREG, ((0x7f << 8) | (1 << 1))); /* configure the parser : L2 Type is configured in the bootloader */ /* extract IP: src, dest protocol */ NLGE_WRITE(sc->base, R_L3CTABLE, (9 << 20) | (1 << 19) | (1 << 18) | (0x01 << 16) | (0x0800 << 0)); NLGE_WRITE(sc->base, R_L3CTABLE + 1, (9 << 25) | (1 << 21) | (12 << 14) | (4 << 10) | (16 << 4) | 4); #ifdef NLGE_HW_CHKSUM device_printf(sc->nlna_dev, "Enabled h/w support to compute TCP/IP" " checksum\n"); #endif /* Configure to extract SRC port and Dest port for TCP and UDP pkts */ NLGE_WRITE(sc->base, R_L4CTABLE, 6); NLGE_WRITE(sc->base, R_L4CTABLE + 2, 17); val = ((0 << 21) | (2 << 17) | (2 << 11) | (2 << 7)); NLGE_WRITE(sc->base, R_L4CTABLE + 1, val); NLGE_WRITE(sc->base, R_L4CTABLE + 3, val); } static void nlna_config_classifier(struct nlna_softc *sc) { int i; if (sc->mac_type == XLR_XGMII) { /* TBD: XGMII init sequence */ /* xgmac translation table doesn't have sane values on reset */ for (i = 0; i < 64; i++) NLGE_WRITE(sc->base, R_TRANSLATETABLE + i, 0x0); /* * use upper 7 bits of the parser extract to index the * translate table */ NLGE_WRITE(sc->base, R_PARSERCONFIGREG, 0x0); } } /* * Complete a bunch of h/w register initializations that are common for all the * ports controlled by a NA. */ static void nlna_config_common(struct nlna_softc *sc) { struct xlr_gmac_block_t *block_info; struct stn_cc *gmac_cc_config; int i; block_info = device_get_ivars(sc->nlna_dev); gmac_cc_config = block_info->credit_config; for (i = 0; i < MAX_NUM_MSGRNG_STN_CC; i++) { NLGE_WRITE(sc->base, R_CC_CPU0_0 + i, gmac_cc_config->counters[i >> 3][i & 0x07]); } NLGE_WRITE(sc->base, R_MSG_TX_THRESHOLD, 3); NLGE_WRITE(sc->base, R_DMACR0, 0xffffffff); NLGE_WRITE(sc->base, R_DMACR1, 0xffffffff); NLGE_WRITE(sc->base, R_DMACR2, 0xffffffff); NLGE_WRITE(sc->base, R_DMACR3, 0xffffffff); NLGE_WRITE(sc->base, R_FREEQCARVE, 0); nlna_media_specific_config(sc); } static void nlna_media_specific_config(struct nlna_softc *sc) { struct bucket_size *bucket_sizes; bucket_sizes = xlr_board_info.bucket_sizes; switch (sc->mac_type) { case XLR_RGMII: case XLR_SGMII: case XLR_XAUI: NLGE_WRITE(sc->base, R_GMAC_JFR0_BUCKET_SIZE, bucket_sizes->bucket[MSGRNG_STNID_GMACJFR_0]); NLGE_WRITE(sc->base, R_GMAC_RFR0_BUCKET_SIZE, bucket_sizes->bucket[MSGRNG_STNID_GMACRFR_0]); NLGE_WRITE(sc->base, R_GMAC_JFR1_BUCKET_SIZE, bucket_sizes->bucket[MSGRNG_STNID_GMACJFR_1]); NLGE_WRITE(sc->base, R_GMAC_RFR1_BUCKET_SIZE, bucket_sizes->bucket[MSGRNG_STNID_GMACRFR_1]); if (sc->mac_type == XLR_XAUI) { NLGE_WRITE(sc->base, R_TXDATAFIFO0, (224 << 16)); } break; case XLR_XGMII: NLGE_WRITE(sc->base, R_XGS_RFR_BUCKET_SIZE, bucket_sizes->bucket[sc->rfrbucket]); default: break; } } static void nlna_reset_ports(struct nlna_softc *sc, struct xlr_gmac_block_t *blk) { xlr_reg_t *addr; int i; uint32_t rx_ctrl; /* Refer Section 13.9.3 in the PRM for the reset sequence */ for (i = 0; i < sc->num_ports; i++) { addr = xlr_io_mmio(blk->gmac_port[i].base_addr); /* 1. Reset RxEnable in MAC_CONFIG */ switch (sc->mac_type) { case XLR_RGMII: case XLR_SGMII: NLGE_UPDATE(addr, R_MAC_CONFIG_1, 0, (1 << O_MAC_CONFIG_1__rxen)); break; case XLR_XAUI: case XLR_XGMII: NLGE_UPDATE(addr, R_RX_CONTROL, 0, (1 << O_RX_CONTROL__RxEnable)); break; default: printf("Error: Unsupported port_type=%d\n", sc->mac_type); } /* 1.1 Wait for RxControl.RxHalt to be set */ do { rx_ctrl = NLGE_READ(addr, R_RX_CONTROL); } while (!(rx_ctrl & 0x2)); /* 2. Set the soft reset bit in RxControl */ NLGE_UPDATE(addr, R_RX_CONTROL, (1 << O_RX_CONTROL__SoftReset), (1 << O_RX_CONTROL__SoftReset)); /* 2.1 Wait for RxControl.SoftResetDone to be set */ do { rx_ctrl = NLGE_READ(addr, R_RX_CONTROL); } while (!(rx_ctrl & 0x8)); /* 3. Clear the soft reset bit in RxControl */ NLGE_UPDATE(addr, R_RX_CONTROL, 0, (1 << O_RX_CONTROL__SoftReset)); /* Turn off tx/rx on the port. */ NLGE_UPDATE(addr, R_RX_CONTROL, 0, (1 << O_RX_CONTROL__RxEnable)); NLGE_UPDATE(addr, R_TX_CONTROL, 0, (1 << O_TX_CONTROL__TxEnable)); } } static void nlna_disable_ports(struct nlna_softc *sc) { int i; for (i = 0; i < sc->num_ports; i++) { if (sc->child_sc[i] != NULL) nlge_port_disable(sc->child_sc[i]); } } static void nlna_enable_ports(struct nlna_softc *sc) { device_t nlge_dev, *devlist; struct nlge_softc *port_sc; int i, numdevs; device_get_children(sc->nlna_dev, &devlist, &numdevs); for (i = 0; i < numdevs; i++) { nlge_dev = devlist[i]; if (nlge_dev == NULL) continue; port_sc = device_get_softc(nlge_dev); if (port_sc->nlge_if->if_drv_flags & IFF_DRV_RUNNING) nlge_port_enable(port_sc); } free(devlist, M_TEMP); } static void nlna_get_all_softc(device_t iodi_dev, struct nlna_softc **sc_vec, uint32_t vec_sz) { device_t na_dev; int i; for (i = 0; i < vec_sz; i++) { sc_vec[i] = NULL; na_dev = device_find_child(iodi_dev, "nlna", i); if (na_dev != NULL) sc_vec[i] = device_get_softc(na_dev); } } static void nlge_port_disable(struct nlge_softc *sc) { struct ifnet *ifp; xlr_reg_t *base; uint32_t rd; int id, port_type; id = sc->id; port_type = sc->port_type; base = sc->base; ifp = sc->nlge_if; NLGE_UPDATE(base, R_RX_CONTROL, 0x0, 1 << O_RX_CONTROL__RxEnable); do { rd = NLGE_READ(base, R_RX_CONTROL); } while (!(rd & (1 << O_RX_CONTROL__RxHalt))); NLGE_UPDATE(base, R_TX_CONTROL, 0, 1 << O_TX_CONTROL__TxEnable); do { rd = NLGE_READ(base, R_TX_CONTROL); } while (!(rd & (1 << O_TX_CONTROL__TxIdle))); switch (port_type) { case XLR_RGMII: case XLR_SGMII: NLGE_UPDATE(base, R_MAC_CONFIG_1, 0, ((1 << O_MAC_CONFIG_1__rxen) | (1 << O_MAC_CONFIG_1__txen))); break; case XLR_XGMII: case XLR_XAUI: NLGE_UPDATE(base, R_XGMAC_CONFIG_1, 0, ((1 << O_XGMAC_CONFIG_1__hsttfen) | (1 << O_XGMAC_CONFIG_1__hstrfen))); break; default: panic("Unknown MAC type on port %d\n", id); } if (ifp) { ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); } } static void nlge_port_enable(struct nlge_softc *sc) { struct xlr_gmac_port *self; xlr_reg_t *base; base = sc->base; self = device_get_ivars(sc->nlge_dev); if (xlr_board_info.is_xls && sc->port_type == XLR_RGMII) NLGE_UPDATE(base, R_RX_CONTROL, (1 << O_RX_CONTROL__RGMII), (1 << O_RX_CONTROL__RGMII)); NLGE_UPDATE(base, R_RX_CONTROL, (1 << O_RX_CONTROL__RxEnable), (1 << O_RX_CONTROL__RxEnable)); NLGE_UPDATE(base, R_TX_CONTROL, (1 << O_TX_CONTROL__TxEnable | RGE_TX_THRESHOLD_BYTES), (1 << O_TX_CONTROL__TxEnable | 0x3fff)); switch (sc->port_type) { case XLR_RGMII: case XLR_SGMII: NLGE_UPDATE(base, R_MAC_CONFIG_1, ((1 << O_MAC_CONFIG_1__rxen) | (1 << O_MAC_CONFIG_1__txen)), ((1 << O_MAC_CONFIG_1__rxen) | (1 << O_MAC_CONFIG_1__txen))); break; case XLR_XGMII: case XLR_XAUI: NLGE_UPDATE(base, R_XGMAC_CONFIG_1, ((1 << O_XGMAC_CONFIG_1__hsttfen) | (1 << O_XGMAC_CONFIG_1__hstrfen)), ((1 << O_XGMAC_CONFIG_1__hsttfen) | (1 << O_XGMAC_CONFIG_1__hstrfen))); break; default: panic("Unknown MAC type on port %d\n", sc->id); } } static void nlge_mac_set_rx_mode(struct nlge_softc *sc) { uint32_t regval; regval = NLGE_READ(sc->base, R_MAC_FILTER_CONFIG); if (sc->if_flags & IFF_PROMISC) { regval |= (1 << O_MAC_FILTER_CONFIG__BROADCAST_EN) | (1 << O_MAC_FILTER_CONFIG__PAUSE_FRAME_EN) | (1 << O_MAC_FILTER_CONFIG__ALL_MCAST_EN) | (1 << O_MAC_FILTER_CONFIG__ALL_UCAST_EN); } else { regval &= ~((1 << O_MAC_FILTER_CONFIG__PAUSE_FRAME_EN) | (1 << O_MAC_FILTER_CONFIG__ALL_UCAST_EN)); } NLGE_WRITE(sc->base, R_MAC_FILTER_CONFIG, regval); } static void nlge_sgmii_init(struct nlge_softc *sc) { xlr_reg_t *mmio_gpio; int phy; if (sc->port_type != XLR_SGMII) return; nlge_mii_write_internal(sc->serdes_addr, 26, 0, 0x6DB0); nlge_mii_write_internal(sc->serdes_addr, 26, 1, 0xFFFF); nlge_mii_write_internal(sc->serdes_addr, 26, 2, 0xB6D0); nlge_mii_write_internal(sc->serdes_addr, 26, 3, 0x00FF); nlge_mii_write_internal(sc->serdes_addr, 26, 4, 0x0000); nlge_mii_write_internal(sc->serdes_addr, 26, 5, 0x0000); nlge_mii_write_internal(sc->serdes_addr, 26, 6, 0x0005); nlge_mii_write_internal(sc->serdes_addr, 26, 7, 0x0001); nlge_mii_write_internal(sc->serdes_addr, 26, 8, 0x0000); nlge_mii_write_internal(sc->serdes_addr, 26, 9, 0x0000); nlge_mii_write_internal(sc->serdes_addr, 26,10, 0x0000); /* program GPIO values for serdes init parameters */ DELAY(100); mmio_gpio = xlr_io_mmio(XLR_IO_GPIO_OFFSET); xlr_write_reg(mmio_gpio, 0x20, 0x7e6802); xlr_write_reg(mmio_gpio, 0x10, 0x7104); DELAY(100); /* * This kludge is needed to setup serdes (?) clock correctly on some * XLS boards */ if ((xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_XI || xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_XII) && xlr_boot1_info.board_minor_version == 4) { /* use 125 Mhz instead of 156.25Mhz ref clock */ DELAY(100); xlr_write_reg(mmio_gpio, 0x10, 0x7103); xlr_write_reg(mmio_gpio, 0x21, 0x7103); DELAY(100); } /* enable autoneg - more magic */ phy = sc->phy_addr % 4 + 27; nlge_mii_write_internal(sc->pcs_addr, phy, 0, 0x1000); DELAY(100000); nlge_mii_write_internal(sc->pcs_addr, phy, 0, 0x0200); DELAY(100000); } static void nlge_intr(void *arg) { struct nlge_port_set *pset; struct nlge_softc *sc; struct nlge_softc *port_sc; xlr_reg_t *base; uint32_t intreg; uint32_t intr_status; int i; sc = arg; if (sc == NULL) { printf("warning: No port registered for interrupt\n"); return; } base = sc->base; intreg = NLGE_READ(base, R_INTREG); if (intreg & (1 << O_INTREG__MDInt)) { pset = sc->mdio_pset; if (pset == NULL) { printf("warning: No ports for MDIO interrupt\n"); return; } for (i = 0; i < pset->vec_sz; i++) { port_sc = pset->port_vec[i]; if (port_sc == NULL) continue; /* Ack phy interrupt - clear on read*/ intr_status = nlge_mii_read_internal(port_sc->mii_base, port_sc->phy_addr, 26); PDEBUG("Phy_%d: int_status=0x%08x\n", port_sc->phy_addr, intr_status); if (!(intr_status & 0x8000)) { /* no interrupt for this port */ continue; } if (intr_status & 0x2410) { /* update link status for port */ nlge_gmac_config_speed(port_sc, 1); } else { printf("%s: Unsupported phy interrupt" " (0x%08x)\n", device_get_nameunit(port_sc->nlge_dev), intr_status); } } } /* Clear the NA interrupt */ xlr_write_reg(base, R_INTREG, 0xffffffff); return; } static int nlge_irq_init(struct nlge_softc *sc) { struct resource irq_res; struct nlna_softc *na_sc; struct xlr_gmac_block_t *block_info; device_t na_dev; int ret; int irq_num; na_dev = device_get_parent(sc->nlge_dev); block_info = device_get_ivars(na_dev); irq_num = block_info->baseirq + sc->instance; irq_res.__r_i = (struct resource_i *)(intptr_t) (irq_num); ret = bus_setup_intr(sc->nlge_dev, &irq_res, INTR_TYPE_NET | INTR_MPSAFE, NULL, nlge_intr, sc, NULL); if (ret) { nlge_detach(sc->nlge_dev); device_printf(sc->nlge_dev, "couldn't set up irq: error=%d\n", ret); return (ENXIO); } PDEBUG("Setup intr for dev=%s, irq=%d\n", device_get_nameunit(sc->nlge_dev), irq_num); if (sc->instance == 0) { na_sc = device_get_softc(na_dev); sc->mdio_pset = &na_sc->mdio_set; } return (0); } static void nlge_irq_fini(struct nlge_softc *sc) { } static void nlge_hw_init(struct nlge_softc *sc) { struct xlr_gmac_port *port_info; xlr_reg_t *base; base = sc->base; port_info = device_get_ivars(sc->nlge_dev); sc->tx_bucket_id = port_info->tx_bucket_id; /* each packet buffer is 1536 bytes */ NLGE_WRITE(base, R_DESC_PACK_CTRL, (1 << O_DESC_PACK_CTRL__MaxEntry) | #ifdef NLGE_HW_CHKSUM (1 << O_DESC_PACK_CTRL__PrePadEnable) | #endif (MAX_FRAME_SIZE << O_DESC_PACK_CTRL__RegularSize)); NLGE_WRITE(base, R_STATCTRL, ((1 << O_STATCTRL__Sten) | (1 << O_STATCTRL__ClrCnt))); NLGE_WRITE(base, R_L2ALLOCCTRL, 0xffffffff); NLGE_WRITE(base, R_INTMASK, 0); nlge_set_mac_addr(sc); nlge_media_specific_init(sc); } static void nlge_sc_init(struct nlge_softc *sc, device_t dev, struct xlr_gmac_port *port_info) { memset(sc, 0, sizeof(*sc)); sc->nlge_dev = dev; sc->id = device_get_unit(dev); nlge_set_port_attribs(sc, port_info); } static void nlge_media_specific_init(struct nlge_softc *sc) { struct mii_data *media; struct bucket_size *bucket_sizes; bucket_sizes = xlr_board_info.bucket_sizes; switch (sc->port_type) { case XLR_RGMII: case XLR_SGMII: case XLR_XAUI: NLGE_UPDATE(sc->base, R_DESC_PACK_CTRL, (BYTE_OFFSET << O_DESC_PACK_CTRL__ByteOffset), (W_DESC_PACK_CTRL__ByteOffset << O_DESC_PACK_CTRL__ByteOffset)); NLGE_WRITE(sc->base, R_GMAC_TX0_BUCKET_SIZE + sc->instance, bucket_sizes->bucket[sc->tx_bucket_id]); if (sc->port_type != XLR_XAUI) { nlge_gmac_config_speed(sc, 1); if (sc->mii_bus) { media = (struct mii_data *)device_get_softc( sc->mii_bus); } } break; case XLR_XGMII: NLGE_WRITE(sc->base, R_BYTEOFFSET0, 0x2); NLGE_WRITE(sc->base, R_XGMACPADCALIBRATION, 0x30); NLGE_WRITE(sc->base, R_XGS_TX0_BUCKET_SIZE, bucket_sizes->bucket[sc->tx_bucket_id]); break; default: break; } } /* * Read the MAC address from the XLR boot registers. All port addresses * are identical except for the lowest octet. */ static void nlge_read_mac_addr(struct nlge_softc *sc) { int i, j; for (i = 0, j = 40; i < ETHER_ADDR_LEN && j >= 0; i++, j-= 8) sc->dev_addr[i] = (xlr_boot1_info.mac_addr >> j) & 0xff; sc->dev_addr[i - 1] += sc->id; /* last octet is port-specific */ } /* * Write the MAC address to the XLR MAC port. Also, set the address * masks and MAC filter configuration. */ static void nlge_set_mac_addr(struct nlge_softc *sc) { NLGE_WRITE(sc->base, R_MAC_ADDR0, ((sc->dev_addr[5] << 24) | (sc->dev_addr[4] << 16) | (sc->dev_addr[3] << 8) | (sc->dev_addr[2]))); NLGE_WRITE(sc->base, R_MAC_ADDR0 + 1, ((sc->dev_addr[1] << 24) | (sc-> dev_addr[0] << 16))); NLGE_WRITE(sc->base, R_MAC_ADDR_MASK2, 0xffffffff); NLGE_WRITE(sc->base, R_MAC_ADDR_MASK2 + 1, 0xffffffff); NLGE_WRITE(sc->base, R_MAC_ADDR_MASK3, 0xffffffff); NLGE_WRITE(sc->base, R_MAC_ADDR_MASK3 + 1, 0xffffffff); NLGE_WRITE(sc->base, R_MAC_FILTER_CONFIG, (1 << O_MAC_FILTER_CONFIG__BROADCAST_EN) | (1 << O_MAC_FILTER_CONFIG__ALL_MCAST_EN) | (1 << O_MAC_FILTER_CONFIG__MAC_ADDR0_VALID)); if (sc->port_type == XLR_RGMII || sc->port_type == XLR_SGMII) { NLGE_UPDATE(sc->base, R_IPG_IFG, MAC_B2B_IPG, 0x7f); } } static int nlge_if_init(struct nlge_softc *sc) { struct ifnet *ifp; device_t dev; int error; error = 0; dev = sc->nlge_dev; NLGE_LOCK_INIT(sc, device_get_nameunit(dev)); ifp = sc->nlge_if = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "can not if_alloc()\n"); error = ENOSPC; goto fail; } ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_capabilities = 0; ifp->if_capenable = ifp->if_capabilities; ifp->if_ioctl = nlge_ioctl; ifp->if_init = nlge_init; ifp->if_hwassist = 0; ifp->if_snd.ifq_drv_maxlen = RGE_TX_Q_SIZE; IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); IFQ_SET_READY(&ifp->if_snd); ifmedia_init(&sc->nlge_mii.mii_media, 0, nlge_mediachange, nlge_mediastatus); ifmedia_add(&sc->nlge_mii.mii_media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&sc->nlge_mii.mii_media, IFM_ETHER | IFM_AUTO); sc->nlge_mii.mii_media.ifm_media = sc->nlge_mii.mii_media.ifm_cur->ifm_media; nlge_read_mac_addr(sc); ether_ifattach(ifp, sc->dev_addr); /* override if_transmit : per ifnet(9), do it after if_attach */ ifp->if_transmit = nlge_tx; fail: return (error); } static void nlge_mii_init(device_t dev, struct nlge_softc *sc) { int error; if (sc->port_type != XLR_XAUI && sc->port_type != XLR_XGMII) { NLGE_WRITE(sc->mii_base, R_MII_MGMT_CONFIG, 0x07); } error = mii_attach(dev, &sc->mii_bus, sc->nlge_if, nlge_mediachange, nlge_mediastatus, BMSR_DEFCAPMASK, sc->phy_addr, MII_OFFSET_ANY, 0); if (error) { device_printf(dev, "attaching PHYs failed\n"); sc->mii_bus = NULL; } if (sc->mii_bus != NULL) { /* * Enable all MDIO interrupts in the phy. RX_ER bit seems to get * set about every 1 sec in GigE mode, ignore it for now... */ nlge_mii_write_internal(sc->mii_base, sc->phy_addr, 25, 0xfffffffe); } } /* * Read a PHY register. * * Input parameters: * mii_base - Base address of MII * phyaddr - PHY's address * regidx = index of register to read * * Return value: * value read, or 0 if an error occurred. */ static int nlge_mii_read_internal(xlr_reg_t *mii_base, int phyaddr, int regidx) { int i, val; /* setup the phy reg to be used */ NLGE_WRITE(mii_base, R_MII_MGMT_ADDRESS, (phyaddr << 8) | (regidx << 0)); /* Issue the read command */ NLGE_WRITE(mii_base, R_MII_MGMT_COMMAND, (1 << O_MII_MGMT_COMMAND__rstat)); /* poll for the read cycle to complete */ for (i = 0; i < PHY_STATUS_RETRIES; i++) { if (NLGE_READ(mii_base, R_MII_MGMT_INDICATORS) == 0) break; } /* clear the read cycle */ NLGE_WRITE(mii_base, R_MII_MGMT_COMMAND, 0); if (i == PHY_STATUS_RETRIES) { return (0xffffffff); } val = NLGE_READ(mii_base, R_MII_MGMT_STATUS); return (val); } /* * Write a value to a PHY register. * * Input parameters: * mii_base - Base address of MII * phyaddr - PHY to use * regidx - register within the PHY * regval - data to write to register * * Return value: * nothing */ static void nlge_mii_write_internal(xlr_reg_t *mii_base, int phyaddr, int regidx, int regval) { int i; NLGE_WRITE(mii_base, R_MII_MGMT_ADDRESS, (phyaddr << 8) | (regidx << 0)); /* Write the data which starts the write cycle */ NLGE_WRITE(mii_base, R_MII_MGMT_WRITE_DATA, regval); /* poll for the write cycle to complete */ for (i = 0; i < PHY_STATUS_RETRIES; i++) { if (NLGE_READ(mii_base, R_MII_MGMT_INDICATORS) == 0) break; } } /* * Function to optimize the use of p2d descriptors for the given PDU. * As it is on the fast-path (called during packet transmission), it * described in more detail than the initialization functions. * * Input: mbuf chain (MC), pointer to fmn message * Input constraints: None * Output: FMN message to transmit the data in MC * Return values: 0 - success * 1 - MC cannot be handled (see Limitations below) * 2 - MC cannot be handled presently (maybe worth re-trying) * Other output: Number of entries filled in the FMN message * * Output structure/constraints: * 1. Max 3 p2d's + 1 zero-len (ZL) p2d with virtual address of MC. * 2. 3 p2d's + 1 p2p with max 14 p2d's (ZL p2d not required in this case). * 3. Each p2d points to physically contiguous chunk of data (subject to * entire MC requiring max 17 p2d's). * Limitations: * 1. MC's that require more than 17 p2d's are not handled. * Benefits: MC's that require <= 3 p2d's avoid the overhead of allocating * the p2p structure. Small packets (which typically give low * performance) are expected to have a small MC that takes * advantage of this. */ static int prepare_fmn_message(struct nlge_softc *sc, struct msgrng_msg *fmn_msg, uint32_t *n_entries, struct mbuf *mbuf_chain, uint64_t fb_stn_id, struct nlge_tx_desc **tx_desc) { struct mbuf *m; struct nlge_tx_desc *p2p; uint64_t *cur_p2d; uint64_t fbpaddr; vm_offset_t buf; vm_paddr_t paddr; int msg_sz, p2p_sz, len, frag_sz; /* Num entries per FMN msg is 4 for XLR/XLS */ const int FMN_SZ = sizeof(*fmn_msg) / sizeof(uint64_t); msg_sz = p2p_sz = 0; p2p = NULL; cur_p2d = &fmn_msg->msg0; for (m = mbuf_chain; m != NULL; m = m->m_next) { buf = (vm_offset_t) m->m_data; len = m->m_len; while (len) { if (msg_sz == (FMN_SZ - 1)) { p2p = uma_zalloc(nl_tx_desc_zone, M_NOWAIT); if (p2p == NULL) { return (2); } /* * Save the virtual address in the descriptor, * it makes freeing easy. */ p2p->frag[XLR_MAX_TX_FRAGS] = (uint64_t)(vm_offset_t)p2p; cur_p2d = &p2p->frag[0]; } else if (msg_sz == (FMN_SZ - 2 + XLR_MAX_TX_FRAGS)) { uma_zfree(nl_tx_desc_zone, p2p); return (1); } paddr = vtophys(buf); frag_sz = PAGE_SIZE - (buf & PAGE_MASK); if (len < frag_sz) frag_sz = len; *cur_p2d++ = (127ULL << 54) | ((uint64_t)frag_sz << 40) | paddr; msg_sz++; if (p2p != NULL) p2p_sz++; len -= frag_sz; buf += frag_sz; } } if (msg_sz == 0) { printf("Zero-length mbuf chain ??\n"); *n_entries = msg_sz ; return (0); } /* set eop in most-recent p2d */ cur_p2d[-1] |= (1ULL << 63); #ifdef __mips_n64 /* * On n64, we cannot store our mbuf pointer(64 bit) in the freeback * message (40bit available), so we put the mbuf in m_nextpkt and * use the physical addr of that in freeback message. */ mbuf_chain->m_nextpkt = mbuf_chain; fbpaddr = vtophys(&mbuf_chain->m_nextpkt); #else /* Careful, don't sign extend when going to 64bit */ fbpaddr = (uint64_t)(uintptr_t)mbuf_chain; #endif *cur_p2d = (1ULL << 63) | ((uint64_t)fb_stn_id << 54) | fbpaddr; *tx_desc = p2p; if (p2p != NULL) { paddr = vtophys(p2p); p2p_sz++; fmn_msg->msg3 = (1ULL << 62) | ((uint64_t)fb_stn_id << 54) | ((uint64_t)(p2p_sz * 8) << 40) | paddr; *n_entries = FMN_SZ; } else { *n_entries = msg_sz + 1; } return (0); } static int send_fmn_msg_tx(struct nlge_softc *sc, struct msgrng_msg *msg, uint32_t n_entries) { uint32_t msgrng_flags; int ret; int i = 0; do { msgrng_flags = msgrng_access_enable(); ret = message_send(n_entries, MSGRNG_CODE_MAC, sc->tx_bucket_id, msg); msgrng_restore(msgrng_flags); if (ret == 0) return (0); i++; } while (i < 100000); device_printf(sc->nlge_dev, "Too many credit fails in tx path\n"); return (1); } static void release_tx_desc(vm_paddr_t paddr) { struct nlge_tx_desc *tx_desc; uint32_t sr; uint64_t vaddr; paddr += (XLR_MAX_TX_FRAGS * sizeof(uint64_t)); sr = xlr_enable_kx(); vaddr = xlr_paddr_ld(paddr); xlr_restore_kx(sr); tx_desc = (struct nlge_tx_desc*)(intptr_t)vaddr; uma_zfree(nl_tx_desc_zone, tx_desc); } static void * get_buf(void) { struct mbuf *m_new; uint64_t *md; #ifdef INVARIANTS vm_paddr_t temp1, temp2; #endif if ((m_new = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR)) == NULL) return (NULL); m_new->m_len = m_new->m_pkthdr.len = MCLBYTES; m_adj(m_new, XLR_CACHELINE_SIZE - ((uintptr_t)m_new->m_data & 0x1f)); md = (uint64_t *)m_new->m_data; md[0] = (intptr_t)m_new; /* Back Ptr */ md[1] = 0xf00bad; m_adj(m_new, XLR_CACHELINE_SIZE); #ifdef INVARIANTS temp1 = vtophys((vm_offset_t) m_new->m_data); temp2 = vtophys((vm_offset_t) m_new->m_data + 1536); if ((temp1 + 1536) != temp2) panic("ALLOCED BUFFER IS NOT CONTIGUOUS\n"); #endif return ((void *)m_new->m_data); } static int nlge_gmac_config_speed(struct nlge_softc *sc, int quick) { struct mii_data *md; xlr_reg_t *mmio; int bmsr, n_tries, max_tries; int core_ctl[] = { 0x2, 0x1, 0x0, 0x1 }; int sgmii_speed[] = { SGMII_SPEED_10, SGMII_SPEED_100, SGMII_SPEED_1000, SGMII_SPEED_100 }; /* default to 100Mbps */ char *speed_str[] = { "10", "100", "1000", "unknown, defaulting to 100" }; int link_state = LINK_STATE_DOWN; if (sc->port_type == XLR_XAUI || sc->port_type == XLR_XGMII) return 0; md = NULL; mmio = sc->base; if (sc->mii_base != NULL) { max_tries = (quick == 1) ? 100 : 4000; bmsr = 0; for (n_tries = 0; n_tries < max_tries; n_tries++) { bmsr = nlge_mii_read_internal(sc->mii_base, sc->phy_addr, MII_BMSR); if ((bmsr & BMSR_ACOMP) && (bmsr & BMSR_LINK)) break; /* Auto-negotiation is complete and link is up */ DELAY(1000); } bmsr &= BMSR_LINK; sc->link = (bmsr == 0) ? xlr_mac_link_down : xlr_mac_link_up; sc->speed = nlge_mii_read_internal(sc->mii_base, sc->phy_addr, 28); sc->speed = (sc->speed >> 3) & 0x03; if (sc->link == xlr_mac_link_up) { link_state = LINK_STATE_UP; nlge_sgmii_init(sc); } if (sc->mii_bus) md = (struct mii_data *)device_get_softc(sc->mii_bus); } if (sc->port_type != XLR_RGMII) NLGE_WRITE(mmio, R_INTERFACE_CONTROL, sgmii_speed[sc->speed]); if (sc->speed == xlr_mac_speed_10 || sc->speed == xlr_mac_speed_100 || sc->speed == xlr_mac_speed_rsvd) { NLGE_WRITE(mmio, R_MAC_CONFIG_2, 0x7117); } else if (sc->speed == xlr_mac_speed_1000) { NLGE_WRITE(mmio, R_MAC_CONFIG_2, 0x7217); if (md != NULL) { ifmedia_set(&md->mii_media, IFM_MAKEWORD(IFM_ETHER, IFM_1000_T, IFM_FDX, md->mii_instance)); } } NLGE_WRITE(mmio, R_CORECONTROL, core_ctl[sc->speed]); if_link_state_change(sc->nlge_if, link_state); printf("%s: [%sMbps]\n", device_get_nameunit(sc->nlge_dev), speed_str[sc->speed]); return (0); } /* * This function is called for each port that was added to the device tree * and it initializes the following port attributes: * - type * - base (base address to access port-specific registers) * - mii_base * - phy_addr */ static void nlge_set_port_attribs(struct nlge_softc *sc, struct xlr_gmac_port *port_info) { sc->instance = port_info->instance % 4; /* TBD: will not work for SPI-4 */ sc->port_type = port_info->type; sc->base = xlr_io_mmio(port_info->base_addr); sc->mii_base = xlr_io_mmio(port_info->mii_addr); if (port_info->pcs_addr != 0) sc->pcs_addr = xlr_io_mmio(port_info->pcs_addr); if (port_info->serdes_addr != 0) sc->serdes_addr = xlr_io_mmio(port_info->serdes_addr); sc->phy_addr = port_info->phy_addr; PDEBUG("Port%d: base=%p, mii_base=%p, phy_addr=%d\n", sc->id, sc->base, sc->mii_base, sc->phy_addr); } /* ------------------------------------------------------------------------ */ /* Debug dump functions */ #ifdef DEBUG static void dump_reg(xlr_reg_t *base, uint32_t offset, char *name) { int val; val = NLGE_READ(base, offset); printf("%-30s: 0x%8x 0x%8x\n", name, offset, val); } #define STRINGIFY(x) #x static void dump_na_registers(xlr_reg_t *base_addr, int port_id) { PDEBUG("Register dump for NA (of port=%d)\n", port_id); dump_reg(base_addr, R_PARSERCONFIGREG, STRINGIFY(R_PARSERCONFIGREG)); PDEBUG("Tx bucket sizes\n"); dump_reg(base_addr, R_GMAC_JFR0_BUCKET_SIZE, STRINGIFY(R_GMAC_JFR0_BUCKET_SIZE)); dump_reg(base_addr, R_GMAC_RFR0_BUCKET_SIZE, STRINGIFY(R_GMAC_RFR0_BUCKET_SIZE)); dump_reg(base_addr, R_GMAC_TX0_BUCKET_SIZE, STRINGIFY(R_GMAC_TX0_BUCKET_SIZE)); dump_reg(base_addr, R_GMAC_TX1_BUCKET_SIZE, STRINGIFY(R_GMAC_TX1_BUCKET_SIZE)); dump_reg(base_addr, R_GMAC_TX2_BUCKET_SIZE, STRINGIFY(R_GMAC_TX2_BUCKET_SIZE)); dump_reg(base_addr, R_GMAC_TX3_BUCKET_SIZE, STRINGIFY(R_GMAC_TX3_BUCKET_SIZE)); dump_reg(base_addr, R_GMAC_JFR1_BUCKET_SIZE, STRINGIFY(R_GMAC_JFR1_BUCKET_SIZE)); dump_reg(base_addr, R_GMAC_RFR1_BUCKET_SIZE, STRINGIFY(R_GMAC_RFR1_BUCKET_SIZE)); dump_reg(base_addr, R_TXDATAFIFO0, STRINGIFY(R_TXDATAFIFO0)); dump_reg(base_addr, R_TXDATAFIFO1, STRINGIFY(R_TXDATAFIFO1)); } static void dump_gmac_registers(struct nlge_softc *sc) { xlr_reg_t *base_addr = sc->base; int port_id = sc->instance; PDEBUG("Register dump for port=%d\n", port_id); if (sc->port_type == XLR_RGMII || sc->port_type == XLR_SGMII) { dump_reg(base_addr, R_MAC_CONFIG_1, STRINGIFY(R_MAC_CONFIG_1)); dump_reg(base_addr, R_MAC_CONFIG_2, STRINGIFY(R_MAC_CONFIG_2)); dump_reg(base_addr, R_IPG_IFG, STRINGIFY(R_IPG_IFG)); dump_reg(base_addr, R_HALF_DUPLEX, STRINGIFY(R_HALF_DUPLEX)); dump_reg(base_addr, R_MAXIMUM_FRAME_LENGTH, STRINGIFY(R_MAXIMUM_FRAME_LENGTH)); dump_reg(base_addr, R_TEST, STRINGIFY(R_TEST)); dump_reg(base_addr, R_MII_MGMT_CONFIG, STRINGIFY(R_MII_MGMT_CONFIG)); dump_reg(base_addr, R_MII_MGMT_COMMAND, STRINGIFY(R_MII_MGMT_COMMAND)); dump_reg(base_addr, R_MII_MGMT_ADDRESS, STRINGIFY(R_MII_MGMT_ADDRESS)); dump_reg(base_addr, R_MII_MGMT_WRITE_DATA, STRINGIFY(R_MII_MGMT_WRITE_DATA)); dump_reg(base_addr, R_MII_MGMT_STATUS, STRINGIFY(R_MII_MGMT_STATUS)); dump_reg(base_addr, R_MII_MGMT_INDICATORS, STRINGIFY(R_MII_MGMT_INDICATORS)); dump_reg(base_addr, R_INTERFACE_CONTROL, STRINGIFY(R_INTERFACE_CONTROL)); dump_reg(base_addr, R_INTERFACE_STATUS, STRINGIFY(R_INTERFACE_STATUS)); } else if (sc->port_type == XLR_XAUI || sc->port_type == XLR_XGMII) { dump_reg(base_addr, R_XGMAC_CONFIG_0, STRINGIFY(R_XGMAC_CONFIG_0)); dump_reg(base_addr, R_XGMAC_CONFIG_1, STRINGIFY(R_XGMAC_CONFIG_1)); dump_reg(base_addr, R_XGMAC_CONFIG_2, STRINGIFY(R_XGMAC_CONFIG_2)); dump_reg(base_addr, R_XGMAC_CONFIG_3, STRINGIFY(R_XGMAC_CONFIG_3)); dump_reg(base_addr, R_XGMAC_STATION_ADDRESS_LS, STRINGIFY(R_XGMAC_STATION_ADDRESS_LS)); dump_reg(base_addr, R_XGMAC_STATION_ADDRESS_MS, STRINGIFY(R_XGMAC_STATION_ADDRESS_MS)); dump_reg(base_addr, R_XGMAC_MAX_FRAME_LEN, STRINGIFY(R_XGMAC_MAX_FRAME_LEN)); dump_reg(base_addr, R_XGMAC_REV_LEVEL, STRINGIFY(R_XGMAC_REV_LEVEL)); dump_reg(base_addr, R_XGMAC_MIIM_COMMAND, STRINGIFY(R_XGMAC_MIIM_COMMAND)); dump_reg(base_addr, R_XGMAC_MIIM_FILED, STRINGIFY(R_XGMAC_MIIM_FILED)); dump_reg(base_addr, R_XGMAC_MIIM_CONFIG, STRINGIFY(R_XGMAC_MIIM_CONFIG)); dump_reg(base_addr, R_XGMAC_MIIM_LINK_FAIL_VECTOR, STRINGIFY(R_XGMAC_MIIM_LINK_FAIL_VECTOR)); dump_reg(base_addr, R_XGMAC_MIIM_INDICATOR, STRINGIFY(R_XGMAC_MIIM_INDICATOR)); } dump_reg(base_addr, R_MAC_ADDR0, STRINGIFY(R_MAC_ADDR0)); dump_reg(base_addr, R_MAC_ADDR0 + 1, STRINGIFY(R_MAC_ADDR0+1)); dump_reg(base_addr, R_MAC_ADDR1, STRINGIFY(R_MAC_ADDR1)); dump_reg(base_addr, R_MAC_ADDR2, STRINGIFY(R_MAC_ADDR2)); dump_reg(base_addr, R_MAC_ADDR3, STRINGIFY(R_MAC_ADDR3)); dump_reg(base_addr, R_MAC_ADDR_MASK2, STRINGIFY(R_MAC_ADDR_MASK2)); dump_reg(base_addr, R_MAC_ADDR_MASK3, STRINGIFY(R_MAC_ADDR_MASK3)); dump_reg(base_addr, R_MAC_FILTER_CONFIG, STRINGIFY(R_MAC_FILTER_CONFIG)); dump_reg(base_addr, R_TX_CONTROL, STRINGIFY(R_TX_CONTROL)); dump_reg(base_addr, R_RX_CONTROL, STRINGIFY(R_RX_CONTROL)); dump_reg(base_addr, R_DESC_PACK_CTRL, STRINGIFY(R_DESC_PACK_CTRL)); dump_reg(base_addr, R_STATCTRL, STRINGIFY(R_STATCTRL)); dump_reg(base_addr, R_L2ALLOCCTRL, STRINGIFY(R_L2ALLOCCTRL)); dump_reg(base_addr, R_INTMASK, STRINGIFY(R_INTMASK)); dump_reg(base_addr, R_INTREG, STRINGIFY(R_INTREG)); dump_reg(base_addr, R_TXRETRY, STRINGIFY(R_TXRETRY)); dump_reg(base_addr, R_CORECONTROL, STRINGIFY(R_CORECONTROL)); dump_reg(base_addr, R_BYTEOFFSET0, STRINGIFY(R_BYTEOFFSET0)); dump_reg(base_addr, R_BYTEOFFSET1, STRINGIFY(R_BYTEOFFSET1)); dump_reg(base_addr, R_L2TYPE_0, STRINGIFY(R_L2TYPE_0)); dump_na_registers(base_addr, port_id); } static void dump_fmn_cpu_credits_for_gmac(struct xlr_board_info *board, int gmac_id) { struct stn_cc *cc; int gmac_bucket_ids[] = { 97, 98, 99, 100, 101, 103 }; int j, k, r, c; int n_gmac_buckets; n_gmac_buckets = sizeof (gmac_bucket_ids) / sizeof (gmac_bucket_ids[0]); for (j = 0; j < 8; j++) { // for each cpu cc = board->credit_configs[j]; printf("Credits for Station CPU_%d ---> GMAC buckets (tx path)\n", j); for (k = 0; k < n_gmac_buckets; k++) { r = gmac_bucket_ids[k] / 8; c = gmac_bucket_ids[k] % 8; printf (" --> gmac%d_bucket_%-3d: credits=%d\n", gmac_id, gmac_bucket_ids[k], cc->counters[r][c]); } } } static void dump_fmn_gmac_credits(struct xlr_board_info *board, int gmac_id) { struct stn_cc *cc; int j, k; cc = board->gmac_block[gmac_id].credit_config; printf("Credits for Station: GMAC_%d ---> CPU buckets (rx path)\n", gmac_id); for (j = 0; j < 8; j++) { // for each cpu printf(" ---> cpu_%d\n", j); for (k = 0; k < 8; k++) { // for each bucket in cpu printf(" ---> bucket_%d: credits=%d\n", j * 8 + k, cc->counters[j][k]); } } } static void dump_board_info(struct xlr_board_info *board) { struct xlr_gmac_block_t *gm; int i, k; printf("cpu=%x ", xlr_revision()); printf("board_version: major=%llx, minor=%llx\n", xlr_boot1_info.board_major_version, xlr_boot1_info.board_minor_version); printf("is_xls=%d, nr_cpus=%d, usb=%s, cfi=%s, ata=%s\npci_irq=%d," "gmac_ports=%d\n", board->is_xls, board->nr_cpus, board->usb ? "Yes" : "No", board->cfi ? "Yes": "No", board->ata ? "Yes" : "No", board->pci_irq, board->gmacports); printf("FMN: Core-station bucket sizes\n"); for (i = 0; i < 128; i++) { if (i && ((i % 16) == 0)) printf("\n"); printf ("b[%d] = %d ", i, board->bucket_sizes->bucket[i]); } printf("\n"); for (i = 0; i < 3; i++) { gm = &board->gmac_block[i]; printf("RNA_%d: type=%d, enabled=%s, mode=%d, station_id=%d," "station_txbase=%d, station_rfr=%d ", i, gm->type, gm->enabled ? "Yes" : "No", gm->mode, gm->station_id, gm->station_txbase, gm->station_rfr); printf("n_ports=%d, baseaddr=%p, baseirq=%d, baseinst=%d\n", gm->num_ports, (xlr_reg_t *)gm->baseaddr, gm->baseirq, gm->baseinst); } for (k = 0; k < 3; k++) { // for each NA dump_fmn_cpu_credits_for_gmac(board, k); dump_fmn_gmac_credits(board, k); } } static void dump_mac_stats(struct nlge_softc *sc) { xlr_reg_t *addr; uint32_t pkts_tx, pkts_rx; addr = sc->base; pkts_rx = NLGE_READ(sc->base, R_RPKT); pkts_tx = NLGE_READ(sc->base, R_TPKT); printf("[nlge_%d mac stats]: pkts_tx=%u, pkts_rx=%u\n", sc->id, pkts_tx, pkts_rx); if (pkts_rx > 0) { uint32_t r; /* dump all rx counters. we need this because pkts_rx includes bad packets. */ for (r = R_RFCS; r <= R_ROVR; r++) printf("[nlge_%d mac stats]: [0x%x]=%u\n", sc->id, r, NLGE_READ(sc->base, r)); } if (pkts_tx > 0) { uint32_t r; /* dump all tx counters. might be useful for debugging. */ for (r = R_TMCA; r <= R_TFRG; r++) { if ((r == (R_TNCL + 1)) || (r == (R_TNCL + 2))) continue; printf("[nlge_%d mac stats]: [0x%x]=%u\n", sc->id, r, NLGE_READ(sc->base, r)); } } } static void dump_mii_regs(struct nlge_softc *sc) { uint32_t mii_regs[] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e}; int i, n_regs; if (sc->mii_base == NULL || sc->mii_bus == NULL) return; n_regs = sizeof (mii_regs) / sizeof (mii_regs[0]); for (i = 0; i < n_regs; i++) { printf("[mii_0x%x] = %x\n", mii_regs[i], nlge_mii_read_internal(sc->mii_base, sc->phy_addr, mii_regs[i])); } } static void dump_ifmedia(struct ifmedia *ifm) { printf("ifm_mask=%08x, ifm_media=%08x, cur=%p\n", ifm->ifm_mask, ifm->ifm_media, ifm->ifm_cur); if (ifm->ifm_cur != NULL) { printf("Cur attribs: ifmedia_entry.ifm_media=%08x," " ifmedia_entry.ifm_data=%08x\n", ifm->ifm_cur->ifm_media, ifm->ifm_cur->ifm_data); } } static void dump_mii_data(struct mii_data *mii) { dump_ifmedia(&mii->mii_media); printf("ifp=%p, mii_instance=%d, mii_media_status=%08x," " mii_media_active=%08x\n", mii->mii_ifp, mii->mii_instance, mii->mii_media_status, mii->mii_media_active); } static void dump_pcs_regs(struct nlge_softc *sc, int phy) { int i, val; printf("PCS regs from %p for phy=%d\n", sc->pcs_addr, phy); for (i = 0; i < 18; i++) { if (i == 2 || i == 3 || (i >= 9 && i <= 14)) continue; val = nlge_mii_read_internal(sc->pcs_addr, phy, i); printf("PHY:%d pcs[0x%x] is 0x%x\n", phy, i, val); } } #endif Index: head/sys/mips/rmi/fmn.c =================================================================== --- head/sys/mips/rmi/fmn.c (revision 295880) +++ head/sys/mips/rmi/fmn.c (revision 295881) @@ -1,492 +1,491 @@ /*- * Copyright (c) 2003-2009 RMI Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of RMI Corporation, nor the names of its contributors, * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RMI_BSD */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #define MSGRNG_CC_INIT_CPU_DEST(dest, counter) \ do { \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][0], 0 ); \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][1], 1 ); \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][2], 2 ); \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][3], 3 ); \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][4], 4 ); \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][5], 5 ); \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][6], 6 ); \ msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][7], 7 ); \ } while(0) /* * Keep track of our message ring handler threads, each core has a * different message station. Ideally we will need to start a few * message handling threads every core, and wake them up depending on * load */ struct msgring_thread { struct { struct thread *thread; /* msgring handler threads */ int needed; /* thread needs to wake up */ } threads[XLR_NTHREADS]; int running; /* number of threads running */ int nthreads; /* number of threads started */ struct mtx lock; /* for changing running/active */ }; static struct msgring_thread msgring_threads[XLR_MAX_CORES]; static struct proc *msgring_proc; /* all threads are under a proc */ /* * The maximum number of software message handler threads to be started * per core. Default is 3 per core */ static int msgring_maxthreads = 3; TUNABLE_INT("hw.fmn.maxthreads", &msgring_maxthreads); /* * The device drivers can register a handler for the messages sent * from a station (corresponding to the device). */ struct tx_stn_handler { msgring_handler action; void *arg; }; static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS]; static struct mtx msgmap_lock; /* * Initialize the messaging subsystem. * * Message Stations are shared among all threads in a cpu core, this * has to be called once from every core which is online. */ void xlr_msgring_cpu_init(void) { struct stn_cc *cc_config; struct bucket_size *bucket_sizes; uint32_t flags; int id; KASSERT(xlr_thr_id() == 0, ("xlr_msgring_cpu_init from non-zero thread")); id = xlr_core_id(); bucket_sizes = xlr_board_info.bucket_sizes; cc_config = xlr_board_info.credit_configs[id]; flags = msgrng_access_enable(); /* * FMN messages are received in 8 buckets per core, set up * the bucket sizes for each bucket */ msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]); msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]); msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]); msgrng_write_bucksize(3, bucket_sizes->bucket[id * 8 + 3]); msgrng_write_bucksize(4, bucket_sizes->bucket[id * 8 + 4]); msgrng_write_bucksize(5, bucket_sizes->bucket[id * 8 + 5]); msgrng_write_bucksize(6, bucket_sizes->bucket[id * 8 + 6]); msgrng_write_bucksize(7, bucket_sizes->bucket[id * 8 + 7]); /* * For sending FMN messages, we need credits on the destination * bucket. Program the credits this core has on the 128 possible * destination buckets. * We cannot use a loop here, because the first argument has * to be a constant integer value. */ MSGRNG_CC_INIT_CPU_DEST(0, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(1, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(2, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(3, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(4, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(5, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(6, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(7, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(8, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(9, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(10, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(11, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(12, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(13, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters); MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters); msgrng_restore(flags); } /* * Boot time init, called only once */ void xlr_msgring_config(void) { mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN); /* check value */ if (msgring_maxthreads < 0 || msgring_maxthreads > XLR_NTHREADS) msgring_maxthreads = XLR_NTHREADS; } /* * Drain out max_messages for the buckets set in the bucket mask. * Use max_messages = 0 to drain out all messages. */ uint32_t xlr_msgring_handler(uint8_t bucket_mask, uint32_t max_messages) { int bucket = 0; int size = 0, code = 0, rx_stid = 0; struct msgrng_msg msg; struct tx_stn_handler *he; unsigned int status = 0; unsigned long mflags; uint32_t n_msgs; uint32_t msgbuckets; n_msgs = 0; mflags = msgrng_access_enable(); for (;;) { msgbuckets = (~msgrng_read_status() >> 24) & bucket_mask; /* all buckets empty, break */ if (msgbuckets == 0) break; for (bucket = 0; bucket < 8; bucket++) { if ((msgbuckets & (1 << bucket)) == 0) /* empty */ continue; status = message_receive(bucket, &size, &code, &rx_stid, &msg); if (status != 0) continue; n_msgs++; he = &msgmap[rx_stid]; if (he->action == NULL) { printf("[%s]: No Handler for message from " "stn_id=%d, bucket=%d, size=%d, msg0=%jx\n", __func__, rx_stid, bucket, size, (uintmax_t)msg.msg0); } else { msgrng_restore(mflags); (*he->action)(bucket, size, code, rx_stid, &msg, he->arg); mflags = msgrng_access_enable(); } if (max_messages > 0 && n_msgs >= max_messages) goto done; } } done: msgrng_restore(mflags); return (n_msgs); } /* * XLR COP2 supports watermark interrupts based on the number of * messages pending in all the buckets in the core. We increase * the watermark until all the possible handler threads in the core * are woken up. */ static void msgrng_setconfig(int running, int nthr) { uint32_t config, mflags; int watermark = 1; /* non zero needed */ int wm_intr_value; KASSERT(nthr >= 0 && nthr <= msgring_maxthreads, ("Bad value of nthr %d", nthr)); KASSERT(running <= nthr, ("Bad value of running %d", running)); if (running == nthr) { wm_intr_value = 0; } else { switch (running) { case 0: break; /* keep default */ case 1: watermark = 32; break; case 2: watermark = 48; break; case 3: watermark = 56; break; } wm_intr_value = 0x2; /* set watermark enable interrupt */ } mflags = msgrng_access_enable(); config = (watermark << 24) | (IRQ_MSGRING << 16) | (1 << 8) | wm_intr_value; /* clear pending interrupts, they will get re-raised if still valid */ write_c0_eirr64(1ULL << IRQ_MSGRING); msgrng_write_config(config); msgrng_restore(mflags); } /* Debug counters */ static int msgring_nintr[XLR_MAX_CORES]; static int msgring_badintr[XLR_MAX_CORES]; static int msgring_wakeup_sleep[XLR_MAX_CORES * XLR_NTHREADS]; static int msgring_wakeup_nosleep[XLR_MAX_CORES * XLR_NTHREADS]; static int msgring_nmsgs[XLR_MAX_CORES * XLR_NTHREADS]; static int msgring_process_fast_intr(void *arg) { struct msgring_thread *mthd; struct thread *td; uint32_t mflags; int core, nt; core = xlr_core_id(); mthd = &msgring_threads[core]; msgring_nintr[core]++; mtx_lock_spin(&mthd->lock); nt = mthd->running; if(nt >= mthd->nthreads) { msgring_badintr[core]++; mtx_unlock_spin(&mthd->lock); return (FILTER_HANDLED); } td = mthd->threads[nt].thread; mflags = msgrng_access_enable(); /* default value with interrupts disabled */ msgrng_write_config((1 << 24) | (IRQ_MSGRING << 16) | (1 << 8)); /* clear pending interrupts */ write_c0_eirr64(1ULL << IRQ_MSGRING); msgrng_restore(mflags); mtx_unlock_spin(&mthd->lock); /* wake up the target thread */ mthd->threads[nt].needed = 1; thread_lock(td); if (TD_AWAITING_INTR(td)) { msgring_wakeup_sleep[core*4+nt]++; TD_CLR_IWAIT(td); sched_add(td, SRQ_INTR); } else msgring_wakeup_nosleep[core*4+nt]++; thread_unlock(td); return (FILTER_HANDLED); } static void msgring_process(void *arg) { struct msgring_thread *mthd; struct thread *td; int hwtid, tid, core; int nmsgs; hwtid = (intptr_t)arg; core = hwtid / 4; tid = hwtid % 4; mthd = &msgring_threads[core]; td = mthd->threads[tid].thread; KASSERT(curthread == td, ("Incorrect thread core %d, thread %d", core, hwtid)); /* First bind this thread to the right CPU */ thread_lock(td); sched_bind(td, xlr_hwtid_to_cpuid[hwtid]); thread_unlock(td); mtx_lock_spin(&mthd->lock); ++mthd->nthreads; /* Active thread count */ mtx_unlock_spin(&mthd->lock); /* start processing messages */ for(;;) { mtx_lock_spin(&mthd->lock); ++mthd->running; msgrng_setconfig(mthd->running, mthd->nthreads); mtx_unlock_spin(&mthd->lock); atomic_store_rel_int(&mthd->threads[tid].needed, 0); nmsgs = xlr_msgring_handler(0xff, 0); msgring_nmsgs[hwtid] += nmsgs; mtx_lock_spin(&mthd->lock); --mthd->running; msgrng_setconfig(mthd->running, mthd->nthreads); mtx_unlock_spin(&mthd->lock); /* sleep */ thread_lock(td); if (mthd->threads[tid].needed) { thread_unlock(td); continue; } sched_class(td, PRI_ITHD); TD_SET_IWAIT(td); mi_switch(SW_VOL, NULL); thread_unlock(td); } } static void create_msgring_thread(int hwtid) { struct msgring_thread *mthd; struct thread *td; int tid, core; int error; core = hwtid / 4; tid = hwtid % 4; mthd = &msgring_threads[core]; if (tid == 0) { mtx_init(&mthd->lock, "msgrngcore", NULL, MTX_SPIN); mthd->running = mthd->nthreads = 0; } error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid, &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc", "msgthr%d", hwtid); if (error) panic("kproc_kthread_add() failed with %d", error); mthd->threads[tid].thread = td; thread_lock(td); sched_class(td, PRI_ITHD); sched_add(td, SRQ_INTR); thread_unlock(td); CTR2(KTR_INTR, "%s: created %s", __func__, td->td_name); } int register_msgring_handler(int startb, int endb, msgring_handler action, void *arg) { void *cookie; int i; static int msgring_int_enabled = 0; KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS, ("Invalid value for for bucket range %d,%d", startb, endb)); mtx_lock_spin(&msgmap_lock); for (i = startb; i <= endb; i++) { KASSERT(msgmap[i].action == NULL, ("Bucket %d already used [action %p]", i, msgmap[i].action)); msgmap[i].action = action; msgmap[i].arg = arg; } mtx_unlock_spin(&msgmap_lock); if (xlr_test_and_set(&msgring_int_enabled)) { create_msgring_thread(0); if (msgring_maxthreads > xlr_threads_per_core) msgring_maxthreads = xlr_threads_per_core; cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL, NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie); } return (0); } /* * Start message ring processing threads on other CPUs, after SMP start */ static void start_msgring_threads(void *arg) { int hwt, tid; for (hwt = 1; hwt < XLR_MAX_CORES * XLR_NTHREADS; hwt++) { if ((xlr_hw_thread_mask & (1 << hwt)) == 0) continue; tid = hwt % XLR_NTHREADS; if (tid >= msgring_maxthreads) continue; create_msgring_thread(hwt); } } SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, start_msgring_threads, NULL); /* * DEBUG support, XXX: static buffer, not locked */ static int sys_print_debug(SYSCTL_HANDLER_ARGS) { struct sbuf sb; int error, i; sbuf_new_for_sysctl(&sb, NULL, 64, req); sbuf_printf(&sb, "\nID INTR ER WU-SLP WU-ERR MSGS\n"); for (i = 0; i < 32; i++) { if ((xlr_hw_thread_mask & (1 << i)) == 0) continue; sbuf_printf(&sb, "%2d: %8d %4d %8d %8d %8d\n", i, msgring_nintr[i/4], msgring_badintr[i/4], msgring_wakeup_sleep[i], msgring_wakeup_nosleep[i], msgring_nmsgs[i]); } error = sbuf_finish(&sb); sbuf_delete(&sb); return (error); } SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0, sys_print_debug, "A", "msgring debug info"); Index: head/sys/mips/rmi/iodi.c =================================================================== --- head/sys/mips/rmi/iodi.c (revision 295880) +++ head/sys/mips/rmi/iodi.c (revision 295881) @@ -1,276 +1,275 @@ /*- * Copyright (c) 2003-2009 RMI Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of RMI Corporation, nor the names of its contributors, * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RMI_BSD */ #include __FBSDID("$FreeBSD$"); #define __RMAN_RESOURCE_VISIBLE #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include /* for DELAY */ #include #include #include #include #include #include #include #include #include extern bus_space_tag_t uart_bus_space_mem; static struct resource * iodi_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static int iodi_activate_resource(device_t, device_t, int, int, struct resource *); static int iodi_setup_intr(device_t, device_t, struct resource *, int, driver_filter_t *, driver_intr_t *, void *, void **); struct iodi_softc *iodi_softc; /* There can be only one. */ /* * We will manage the Flash/PCMCIA devices in IODI for now. * The NOR flash, Compact flash etc. which can be connected on * various chip selects on the peripheral IO, should have a * separate bus later. */ static void bridge_pcmcia_ack(int irq) { xlr_reg_t *mmio = xlr_io_mmio(XLR_IO_FLASH_OFFSET); xlr_write_reg(mmio, 0x60, 0xffffffff); } static int iodi_setup_intr(device_t dev, device_t child, struct resource *ires, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { const char *name = device_get_name(child); if (strcmp(name, "uart") == 0) { /* FIXME uart 1? */ cpu_establish_hardintr("uart", filt, intr, arg, PIC_UART_0_IRQ, flags, cookiep); pic_setup_intr(PIC_IRT_UART_0_INDEX, PIC_UART_0_IRQ, 0x1, 1); } else if (strcmp(name, "nlge") == 0) { int irq; /* This is a hack to pass in the irq */ irq = (intptr_t)ires->__r_i; cpu_establish_hardintr("nlge", filt, intr, arg, irq, flags, cookiep); pic_setup_intr(irq - PIC_IRQ_BASE, irq, 0x1, 1); } else if (strcmp(name, "ehci") == 0) { cpu_establish_hardintr("ehci", filt, intr, arg, PIC_USB_IRQ, flags, cookiep); pic_setup_intr(PIC_USB_IRQ - PIC_IRQ_BASE, PIC_USB_IRQ, 0x1, 1); } else if (strcmp(name, "ata") == 0) { xlr_establish_intr("ata", filt, intr, arg, PIC_PCMCIA_IRQ, flags, cookiep, bridge_pcmcia_ack); pic_setup_intr(PIC_PCMCIA_IRQ - PIC_IRQ_BASE, PIC_PCMCIA_IRQ, 0x1, 1); } return (0); } static struct resource * iodi_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res = malloc(sizeof(*res), M_DEVBUF, M_WAITOK); const char *name = device_get_name(child); int unit; #ifdef DEBUG switch (type) { case SYS_RES_IRQ: device_printf(bus, "IRQ resource - for %s %lx-%lx\n", device_get_nameunit(child), start, end); break; case SYS_RES_IOPORT: device_printf(bus, "IOPORT resource - for %s %lx-%lx\n", device_get_nameunit(child), start, end); break; case SYS_RES_MEMORY: device_printf(bus, "MEMORY resource - for %s %lx-%lx\n", device_get_nameunit(child), start, end); break; } #endif if (strcmp(name, "uart") == 0) { if ((unit = device_get_unit(child)) == 0) { /* uart 0 */ res->r_bushandle = (xlr_io_base + XLR_IO_UART_0_OFFSET); } else if (unit == 1) { res->r_bushandle = (xlr_io_base + XLR_IO_UART_1_OFFSET); } else printf("%s: Unknown uart unit\n", __FUNCTION__); res->r_bustag = uart_bus_space_mem; } else if (strcmp(name, "ehci") == 0) { res->r_bushandle = MIPS_PHYS_TO_KSEG1(0x1ef24000); res->r_bustag = rmi_pci_bus_space; } else if (strcmp(name, "cfi") == 0) { res->r_bushandle = MIPS_PHYS_TO_KSEG1(0x1c000000); res->r_bustag = 0; } else if (strcmp(name, "ata") == 0) { res->r_bushandle = MIPS_PHYS_TO_KSEG1(0x1d000000); res->r_bustag = rmi_pci_bus_space; /* byte swapping (not really PCI) */ } /* res->r_start = *rid; */ return (res); } static int iodi_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { return (0); } /* prototypes */ static int iodi_probe(device_t); static int iodi_attach(device_t); static int iodi_detach(device_t); static void iodi_identify(driver_t *, device_t); int iodi_probe(device_t dev) { return (BUS_PROBE_NOWILDCARD); } void iodi_identify(driver_t * driver, device_t parent) { BUS_ADD_CHILD(parent, 0, "iodi", 0); } int iodi_attach(device_t dev) { device_t tmpd; int i; /* * Attach each devices */ device_add_child(dev, "uart", 0); device_add_child(dev, "xlr_i2c", 0); device_add_child(dev, "xlr_i2c", 1); device_add_child(dev, "pcib", 0); device_add_child(dev, "rmisec", -1); if (xlr_board_info.usb) device_add_child(dev, "ehci", 0); if (xlr_board_info.cfi) device_add_child(dev, "cfi", 0); if (xlr_board_info.ata) device_add_child(dev, "ata", 0); for (i = 0; i < 3; i++) { if (xlr_board_info.gmac_block[i].enabled == 0) continue; tmpd = device_add_child(dev, "nlna", i); device_set_ivars(tmpd, &xlr_board_info.gmac_block[i]); } bus_generic_probe(dev); bus_generic_attach(dev); return 0; } int iodi_detach(device_t dev) { device_t nlna_dev; int error, i, ret; error = 0; ret = 0; for (i = 0; i < 3; i++) { nlna_dev = device_find_child(dev, "nlna", i); if (nlna_dev != NULL) error = bus_generic_detach(nlna_dev); if (error) ret = error; } return ret; } static device_method_t iodi_methods[] = { DEVMETHOD(device_probe, iodi_probe), DEVMETHOD(device_attach, iodi_attach), DEVMETHOD(device_detach, iodi_detach), DEVMETHOD(device_identify, iodi_identify), DEVMETHOD(bus_alloc_resource, iodi_alloc_resource), DEVMETHOD(bus_activate_resource, iodi_activate_resource), DEVMETHOD(bus_add_child, bus_generic_add_child), DEVMETHOD(bus_setup_intr, iodi_setup_intr), {0, 0}, }; static driver_t iodi_driver = { "iodi", iodi_methods, 1 /* no softc */ }; static devclass_t iodi_devclass; DRIVER_MODULE(iodi, nexus, iodi_driver, iodi_devclass, 0, 0);