diff --git a/sys/arm/arm/trap-v6.c b/sys/arm/arm/trap-v6.c index cd141376d381..7bd5780af338 100644 --- a/sys/arm/arm/trap-v6.c +++ b/sys/arm/arm/trap-v6.c @@ -1,670 +1,670 @@ /*- * Copyright 2014 Olivier Houchard * Copyright 2014 Svatopluk Kraus * Copyright 2014 Michal Meloun * Copyright 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ktrace.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #ifdef KTRACE #include #include #endif #include #include #include #include #include #include #include #include #include #include #ifdef KDB #include #include #endif #ifdef KDTRACE_HOOKS #include #endif extern char cachebailout[]; struct ksig { int sig; u_long code; vm_offset_t addr; }; typedef int abort_func_t(struct trapframe *, u_int, u_int, u_int, u_int, struct thread *, struct ksig *); static abort_func_t abort_fatal; static abort_func_t abort_align; static abort_func_t abort_icache; struct abort { abort_func_t *func; const char *desc; }; /* * How are the aborts handled? * * Undefined Code: * - Always fatal as we do not know what does it mean. * Imprecise External Abort: * - Always fatal, but can be handled somehow in the future. * Now, due to PCIe buggy hardware, ignored. * Precise External Abort: * - Always fatal, but who knows in the future??? * Debug Event: * - Special handling. * External Translation Abort (L1 & L2) * - Always fatal as something is screwed up in page tables or hardware. * Domain Fault (L1 & L2): * - Always fatal as we do not play game with domains. * Alignment Fault: * - Everything should be aligned in kernel with exception of user to kernel * and vice versa data copying, so if pcb_onfault is not set, it's fatal. * We generate signal in case of abort from user mode. * Instruction cache maintenance: * - According to manual, this is translation fault during cache maintenance * operation. So, it could be really complex in SMP case and fuzzy too * for cache operations working on virtual addresses. For now, we will * consider this abort as fatal. In fact, no cache maintenance on * not mapped virtual addresses should be called. As cache maintenance - * operation (except DMB, DSB, and Flush Prefetch Buffer) are priviledged, + * operation (except DMB, DSB, and Flush Prefetch Buffer) are privileged, * the abort is fatal for user mode as well for now. (This is good place to * note that cache maintenance on virtual address fill TLB.) * Acces Bit (L1 & L2): * - Fast hardware emulation for kernel and user mode. * Translation Fault (L1 & L2): * - Standard fault mechanism is held including vm_fault(). * Permission Fault (L1 & L2): * - Fast hardware emulation of modify bits and in other cases, standard * fault mechanism is held including vm_fault(). */ static const struct abort aborts[] = { {abort_fatal, "Undefined Code (0x000)"}, {abort_align, "Alignment Fault"}, {abort_fatal, "Debug Event"}, {NULL, "Access Bit (L1)"}, {NULL, "Instruction cache maintenance"}, {NULL, "Translation Fault (L1)"}, {NULL, "Access Bit (L2)"}, {NULL, "Translation Fault (L2)"}, {abort_fatal, "External Abort"}, {abort_fatal, "Domain Fault (L1)"}, {abort_fatal, "Undefined Code (0x00A)"}, {abort_fatal, "Domain Fault (L2)"}, {abort_fatal, "External Translation Abort (L1)"}, {NULL, "Permission Fault (L1)"}, {abort_fatal, "External Translation Abort (L2)"}, {NULL, "Permission Fault (L2)"}, {abort_fatal, "TLB Conflict Abort"}, {abort_fatal, "Undefined Code (0x401)"}, {abort_fatal, "Undefined Code (0x402)"}, {abort_fatal, "Undefined Code (0x403)"}, {abort_fatal, "Undefined Code (0x404)"}, {abort_fatal, "Undefined Code (0x405)"}, {abort_fatal, "Asynchronous External Abort"}, {abort_fatal, "Undefined Code (0x407)"}, {abort_fatal, "Asynchronous Parity Error on Memory Access"}, {abort_fatal, "Parity Error on Memory Access"}, {abort_fatal, "Undefined Code (0x40A)"}, {abort_fatal, "Undefined Code (0x40B)"}, {abort_fatal, "Parity Error on Translation (L1)"}, {abort_fatal, "Undefined Code (0x40D)"}, {abort_fatal, "Parity Error on Translation (L2)"}, {abort_fatal, "Undefined Code (0x40F)"} }; static __inline void call_trapsignal(struct thread *td, int sig, int code, vm_offset_t addr, int trapno) { ksiginfo_t ksi; CTR4(KTR_TRAP, "%s: addr: %#x, sig: %d, code: %d", __func__, addr, sig, code); /* * TODO: some info would be nice to know * if we are serving data or prefetch abort. */ ksiginfo_init_trap(&ksi); ksi.ksi_signo = sig; ksi.ksi_code = code; ksi.ksi_addr = (void *)addr; ksi.ksi_trapno = trapno; trapsignal(td, &ksi); } /* * abort_imprecise() handles the following abort: * * FAULT_EA_IMPREC - Imprecise External Abort * * The imprecise means that we don't know where the abort happened, * thus FAR is undefined. The abort should not never fire, but hot * plugging or accidental hardware failure can be the cause of it. * If the abort happens, it can even be on different (thread) context. * Without any additional support, the abort is fatal, as we do not * know what really happened. * * QQQ: Some additional functionality, like pcb_onfault but global, * can be implemented. Imprecise handlers could be registered * which tell us if the abort is caused by something they know * about. They should return one of three codes like: * FAULT_IS_MINE, * FAULT_CAN_BE_MINE, * FAULT_IS_NOT_MINE. * The handlers should be called until some of them returns * FAULT_IS_MINE value or all was called. If all handlers return * FAULT_IS_NOT_MINE value, then the abort is fatal. */ static __inline void abort_imprecise(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode) { /* * XXX - We can got imprecise abort as result of access * to not-present PCI/PCIe configuration space. */ #if 0 goto out; #endif abort_fatal(tf, FAULT_EA_IMPREC, fsr, 0, prefetch, curthread, NULL); /* * Returning from this function means that we ignore * the abort for good reason. Note that imprecise abort * could fire any time even in user mode. */ #if 0 out: if (usermode) userret(curthread, tf); #endif } /* * abort_debug() handles the following abort: * * FAULT_DEBUG - Debug Event * */ static __inline void abort_debug(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode, u_int far) { if (usermode) { struct thread *td; td = curthread; call_trapsignal(td, SIGTRAP, TRAP_BRKPT, far, FAULT_DEBUG); userret(td, tf); } else { #ifdef KDB kdb_trap((prefetch) ? T_BREAKPOINT : T_WATCHPOINT, 0, tf); #else printf("No debugger in kernel.\n"); #endif } } /* * Abort handler. * * FAR, FSR, and everything what can be lost after enabling * interrupts must be grabbed before the interrupts will be * enabled. Note that when interrupts will be enabled, we * could even migrate to another CPU ... * * TODO: move quick cases to ASM */ void abort_handler(struct trapframe *tf, int prefetch) { struct thread *td; vm_offset_t far, va; int idx, rv; uint32_t fsr; struct ksig ksig; struct proc *p; struct pcb *pcb; struct vm_map *map; struct vmspace *vm; vm_prot_t ftype; bool usermode; int bp_harden, ucode; #ifdef INVARIANTS void *onfault; #endif VM_CNT_INC(v_trap); td = curthread; fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get(); #if __ARM_ARCH >= 7 far = (prefetch) ? cp15_ifar_get() : cp15_dfar_get(); #else far = (prefetch) ? TRAPF_PC(tf) : cp15_dfar_get(); #endif idx = FSR_TO_FAULT(fsr); usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */ /* * Apply BP hardening by flushing the branch prediction cache * for prefaults on kernel addresses. */ if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS && (idx == FAULT_TRAN_L2 || idx == FAULT_PERM_L2))) { bp_harden = PCPU_GET(bp_harden_kind); if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL) _CP15_BPIALL(); else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU) _CP15_ICIALLU(); } if (usermode) td->td_frame = tf; CTR6(KTR_TRAP, "%s: fsr %#x (idx %u) far %#x prefetch %u usermode %d", __func__, fsr, idx, far, prefetch, usermode); /* * Firstly, handle aborts that are not directly related to mapping. */ if (__predict_false(idx == FAULT_EA_IMPREC)) { abort_imprecise(tf, fsr, prefetch, usermode); return; } if (__predict_false(idx == FAULT_DEBUG)) { abort_debug(tf, fsr, prefetch, usermode, far); return; } /* * ARM has a set of unprivileged load and store instructions * (LDRT/LDRBT/STRT/STRBT ...) which are supposed to be used in other * than user mode and OS should recognize their aborts and behave * appropriately. However, there is no way how to do that reasonably * in general unless we restrict the handling somehow. * * For now, these instructions are used only in copyin()/copyout() * like functions where usermode buffers are checked in advance that * they are not from KVA space. Thus, no action is needed here. */ /* * (1) Handle access and R/W hardware emulation aborts. * (2) Check that abort is not on pmap essential address ranges. * There is no way how to fix it, so we don't even try. */ rv = pmap_fault(PCPU_GET(curpmap), far, fsr, idx, usermode); if (rv == KERN_SUCCESS) return; #ifdef KDB if (kdb_active) { kdb_reenter(); goto out; } #endif if (rv == KERN_INVALID_ADDRESS) goto nogo; if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) { /* * Due to both processor errata and lazy TLB invalidation when * access restrictions are removed from virtual pages, memory * accesses that are allowed by the physical mapping layer may * nonetheless cause one spurious page fault per virtual page. * When the thread is executing a "no faulting" section that * is bracketed by vm_fault_{disable,enable}_pagefaults(), * every page fault is treated as a spurious page fault, * unless it accesses the same virtual address as the most * recent page fault within the same "no faulting" section. */ if (td->td_md.md_spurflt_addr != far || (td->td_pflags & TDP_RESETSPUR) != 0) { td->td_md.md_spurflt_addr = far; td->td_pflags &= ~TDP_RESETSPUR; tlb_flush_local(far & ~PAGE_MASK); return; } } else { /* * If we get a page fault while in a critical section, then * it is most likely a fatal kernel page fault. The kernel * is already going to panic trying to get a sleep lock to * do the VM lookup, so just consider it a fatal trap so the * kernel can print out a useful trap message and even get * to the debugger. * * If we get a page fault while holding a non-sleepable * lock, then it is most likely a fatal kernel page fault. * If WITNESS is enabled, then it's going to whine about * bogus LORs with various VM locks, so just skip to the * fatal trap handling directly. */ if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, "Kernel page fault") != 0) { abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig); return; } } /* Re-enable interrupts if they were enabled previously. */ if (td->td_md.md_spinlock_count == 0) { if (__predict_true(tf->tf_spsr & PSR_I) == 0) enable_interrupts(PSR_I); if (__predict_true(tf->tf_spsr & PSR_F) == 0) enable_interrupts(PSR_F); } p = td->td_proc; if (usermode) { td->td_pticks = 0; if (td->td_cowgen != p->p_cowgen) thread_cow_update(td); } /* Invoke the appropriate handler, if necessary. */ if (__predict_false(aborts[idx].func != NULL)) { if ((aborts[idx].func)(tf, idx, fsr, far, prefetch, td, &ksig)) goto do_trapsignal; goto out; } /* * At this point, we're dealing with one of the following aborts: * * FAULT_ICACHE - I-cache maintenance * FAULT_TRAN_xx - Translation * FAULT_PERM_xx - Permission */ /* * Don't pass faulting cache operation to vm_fault(). We don't want * to handle all vm stuff at this moment. */ pcb = td->td_pcb; if (__predict_false(pcb->pcb_onfault == cachebailout)) { tf->tf_r0 = far; /* return failing address */ tf->tf_pc = (register_t)pcb->pcb_onfault; return; } /* Handle remaining I-cache aborts. */ if (idx == FAULT_ICACHE) { if (abort_icache(tf, idx, fsr, far, prefetch, td, &ksig)) goto do_trapsignal; goto out; } va = trunc_page(far); if (va >= KERNBASE) { /* * Don't allow user-mode faults in kernel address space. */ if (usermode) { ksig.sig = SIGSEGV; ksig.code = SEGV_ACCERR; goto nogo; } map = kernel_map; } else { /* * This is a fault on non-kernel virtual memory. If curproc * is NULL or curproc->p_vmspace is NULL the fault is fatal. */ vm = (p != NULL) ? p->p_vmspace : NULL; if (vm == NULL) { ksig.sig = SIGSEGV; ksig.code = 0; goto nogo; } map = &vm->vm_map; if (!usermode && (td->td_intr_nesting_level != 0 || pcb->pcb_onfault == NULL)) { abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig); return; } } ftype = (fsr & FSR_WNR) ? VM_PROT_WRITE : VM_PROT_READ; if (prefetch) ftype |= VM_PROT_EXECUTE; #ifdef INVARIANTS onfault = pcb->pcb_onfault; pcb->pcb_onfault = NULL; #endif /* Fault in the page. */ rv = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &ksig.sig, &ucode); ksig.code = ucode; #ifdef INVARIANTS pcb->pcb_onfault = onfault; #endif if (__predict_true(rv == KERN_SUCCESS)) goto out; nogo: if (!usermode) { if (td->td_intr_nesting_level == 0 && pcb->pcb_onfault != NULL) { tf->tf_r0 = rv; tf->tf_pc = (int)pcb->pcb_onfault; return; } CTR2(KTR_TRAP, "%s: vm_fault() failed with %d", __func__, rv); abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig); return; } ksig.addr = far; do_trapsignal: call_trapsignal(td, ksig.sig, ksig.code, ksig.addr, idx); out: if (usermode) userret(td, tf); } /* * abort_fatal() handles the following data aborts: * * FAULT_DEBUG - Debug Event * FAULT_ACCESS_xx - Acces Bit * FAULT_EA_PREC - Precise External Abort * FAULT_DOMAIN_xx - Domain Fault * FAULT_EA_TRAN_xx - External Translation Abort * FAULT_EA_IMPREC - Imprecise External Abort * + all undefined codes for ABORT * * We should never see these on a properly functioning system. * * This function is also called by the other handlers if they * detect a fatal problem. * * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort. */ static int abort_fatal(struct trapframe *tf, u_int idx, u_int fsr, u_int far, u_int prefetch, struct thread *td, struct ksig *ksig) { bool usermode; const char *mode; const char *rw_mode; #ifdef KDB bool handled; #endif usermode = TRAPF_USERMODE(tf); #ifdef KDTRACE_HOOKS if (!usermode) { if (dtrace_trap_func != NULL && (*dtrace_trap_func)(tf, far)) return (0); } #endif mode = usermode ? "user" : "kernel"; rw_mode = fsr & FSR_WNR ? "write" : "read"; disable_interrupts(PSR_I|PSR_F); if (td != NULL) { printf("Fatal %s mode data abort: '%s' on %s\n", mode, aborts[idx].desc, rw_mode); printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr); if (idx != FAULT_EA_IMPREC) printf("%08x, ", far); else printf("Invalid, "); printf("spsr=%08x\n", tf->tf_spsr); } else { printf("Fatal %s mode prefetch abort at 0x%08x\n", mode, tf->tf_pc); printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr); } printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n", tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3); printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n", tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7); printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n", tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11); printf("r12=%08x, ", tf->tf_r12); if (usermode) printf("usp=%08x, ulr=%08x", tf->tf_usr_sp, tf->tf_usr_lr); else printf("ssp=%08x, slr=%08x", tf->tf_svc_sp, tf->tf_svc_lr); printf(", pc =%08x\n\n", tf->tf_pc); #ifdef KDB if (debugger_on_trap) { kdb_why = KDB_WHY_TRAP; handled = kdb_trap(fsr, 0, tf); kdb_why = KDB_WHY_UNSET; if (handled) return (0); } #endif panic("Fatal abort"); /*NOTREACHED*/ } /* * abort_align() handles the following data abort: * * FAULT_ALIGN - Alignment fault * * Everything should be aligned in kernel with exception of user to kernel * and vice versa data copying, so if pcb_onfault is not set, it's fatal. * We generate signal in case of abort from user mode. */ static int abort_align(struct trapframe *tf, u_int idx, u_int fsr, u_int far, u_int prefetch, struct thread *td, struct ksig *ksig) { bool usermode; usermode = TRAPF_USERMODE(tf); if (!usermode) { if (td->td_intr_nesting_level == 0 && td != NULL && td->td_pcb->pcb_onfault != NULL) { tf->tf_r0 = EFAULT; tf->tf_pc = (int)td->td_pcb->pcb_onfault; return (0); } abort_fatal(tf, idx, fsr, far, prefetch, td, ksig); } /* Deliver a bus error signal to the process */ ksig->code = BUS_ADRALN; ksig->sig = SIGBUS; ksig->addr = far; return (1); } /* * abort_icache() handles the following data abort: * * FAULT_ICACHE - Instruction cache maintenance * * According to manual, FAULT_ICACHE is translation fault during cache * maintenance operation. In fact, no cache maintenance operation on * not mapped virtual addresses should be called. As cache maintenance - * operation (except DMB, DSB, and Flush Prefetch Buffer) are priviledged, + * operation (except DMB, DSB, and Flush Prefetch Buffer) are privileged, * the abort is concider as fatal for now. However, all the matter with * cache maintenance operation on virtual addresses could be really complex * and fuzzy in SMP case, so maybe in future standard fault mechanism * should be held here including vm_fault() calling. */ static int abort_icache(struct trapframe *tf, u_int idx, u_int fsr, u_int far, u_int prefetch, struct thread *td, struct ksig *ksig) { abort_fatal(tf, idx, fsr, far, prefetch, td, ksig); return(0); } diff --git a/sys/dev/cxgbe/cudbg/fastlz_api.c b/sys/dev/cxgbe/cudbg/fastlz_api.c index a513557ad352..7a8131f6da3e 100644 --- a/sys/dev/cxgbe/cudbg/fastlz_api.c +++ b/sys/dev/cxgbe/cudbg/fastlz_api.c @@ -1,531 +1,531 @@ /* FastLZ - lightning-fast lossless compression library Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "osdep.h" #include "cudbg.h" #include "cudbg_lib_common.h" #include "fastlz.h" static unsigned char sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10}; #define CUDBG_BLOCK_SIZE (63*1024) #define CUDBG_CHUNK_BUF_LEN 16 #define CUDBG_MIN_COMPR_LEN 32 /*min data length for applying compression*/ /* for Adler-32 checksum algorithm, see RFC 1950 Section 8.2 */ #define ADLER32_BASE 65521 static inline unsigned long update_adler32(unsigned long checksum, const void *buf, int len) { const unsigned char *ptr = (const unsigned char *)buf; unsigned long s1 = checksum & 0xffff; unsigned long s2 = (checksum >> 16) & 0xffff; while (len > 0) { unsigned k = len < 5552 ? len : 5552; len -= k; while (k >= 8) { s1 += *ptr++; s2 += s1; s1 += *ptr++; s2 += s1; s1 += *ptr++; s2 += s1; s1 += *ptr++; s2 += s1; s1 += *ptr++; s2 += s1; s1 += *ptr++; s2 += s1; s1 += *ptr++; s2 += s1; s1 += *ptr++; s2 += s1; k -= 8; } while (k-- > 0) { s1 += *ptr++; s2 += s1; } s1 = s1 % ADLER32_BASE; s2 = s2 % ADLER32_BASE; } return (s2 << 16) + s1; } int write_magic(struct cudbg_buffer *_out_buff) { int rc; rc = write_to_buf(_out_buff->data, _out_buff->size, &_out_buff->offset, sixpack_magic, 8); return rc; } int write_to_buf(void *out_buf, u32 out_buf_size, u32 *offset, void *in_buf, u32 in_buf_size) { int rc = 0; if (*offset >= out_buf_size) { rc = CUDBG_STATUS_OUTBUFF_OVERFLOW; goto err; } memcpy((char *)out_buf + *offset, in_buf, in_buf_size); *offset = *offset + in_buf_size; err: return rc; } int read_from_buf(void *in_buf, u32 in_buf_size, u32 *offset, void *out_buf, u32 out_buf_size) { if (in_buf_size - *offset < out_buf_size) return 0; memcpy((char *)out_buf, (char *)in_buf + *offset, out_buf_size); *offset = *offset + out_buf_size; return out_buf_size; } int write_chunk_header(struct cudbg_buffer *_outbuf, int id, int options, unsigned long size, unsigned long checksum, unsigned long extra) { unsigned char buffer[CUDBG_CHUNK_BUF_LEN]; int rc = 0; buffer[0] = id & 255; buffer[1] = (unsigned char)(id >> 8); buffer[2] = options & 255; buffer[3] = (unsigned char)(options >> 8); buffer[4] = size & 255; buffer[5] = (size >> 8) & 255; buffer[6] = (size >> 16) & 255; buffer[7] = (size >> 24) & 255; buffer[8] = checksum & 255; buffer[9] = (checksum >> 8) & 255; buffer[10] = (checksum >> 16) & 255; buffer[11] = (checksum >> 24) & 255; buffer[12] = extra & 255; buffer[13] = (extra >> 8) & 255; buffer[14] = (extra >> 16) & 255; buffer[15] = (extra >> 24) & 255; rc = write_to_buf(_outbuf->data, _outbuf->size, &_outbuf->offset, buffer, 16); return rc; } int write_compression_hdr(struct cudbg_buffer *pin_buff, struct cudbg_buffer *pout_buff) { struct cudbg_buffer tmp_buffer; unsigned long fsize = pin_buff->size; unsigned char *buffer; unsigned long checksum; int rc; char *shown_name = "abc"; /* Always release inner scratch buffer, before releasing outer. */ rc = get_scratch_buff(pout_buff, 10, &tmp_buffer); if (rc) goto err; buffer = (unsigned char *)tmp_buffer.data; rc = write_magic(pout_buff); if (rc) goto err1; /* chunk for File Entry */ buffer[0] = fsize & 255; buffer[1] = (fsize >> 8) & 255; buffer[2] = (fsize >> 16) & 255; buffer[3] = (fsize >> 24) & 255; buffer[4] = 0; buffer[5] = 0; buffer[6] = 0; buffer[7] = 0; buffer[8] = (strlen(shown_name)+1) & 255; buffer[9] = (unsigned char)((strlen(shown_name)+1) >> 8); checksum = 1L; checksum = update_adler32(checksum, buffer, 10); checksum = update_adler32(checksum, shown_name, (int)strlen(shown_name)+1); rc = write_chunk_header(pout_buff, 1, 0, 10+(unsigned long)strlen(shown_name)+1, checksum, 0); if (rc) goto err1; rc = write_to_buf(pout_buff->data, pout_buff->size, &(pout_buff->offset), buffer, 10); if (rc) goto err1; rc = write_to_buf(pout_buff->data, pout_buff->size, &(pout_buff->offset), shown_name, (u32)strlen(shown_name)+1); if (rc) goto err1; err1: release_scratch_buff(&tmp_buffer, pout_buff); err: return rc; } int compress_buff(struct cudbg_buffer *pin_buff, struct cudbg_buffer *pout_buff) { struct cudbg_buffer tmp_buffer; struct cudbg_hdr *cudbg_hdr; unsigned long checksum; unsigned char *result; unsigned int bytes_read; int chunk_size, level = 2, rc = 0; int compress_method = 1; bytes_read = pin_buff->size; rc = get_scratch_buff(pout_buff, CUDBG_BLOCK_SIZE, &tmp_buffer); if (rc) goto err; result = (unsigned char *)tmp_buffer.data; if (bytes_read < 32) compress_method = 0; cudbg_hdr = (struct cudbg_hdr *) pout_buff->data; switch (compress_method) { case 1: chunk_size = fastlz_compress_level(level, pin_buff->data, bytes_read, result); checksum = update_adler32(1L, result, chunk_size); if ((chunk_size > 62000) && (cudbg_hdr->reserved[7] < (u32) chunk_size)) /* 64512 */ cudbg_hdr->reserved[7] = (u32) chunk_size; rc = write_chunk_header(pout_buff, 17, 1, chunk_size, checksum, bytes_read); if (rc) goto err_put_buff; rc = write_to_buf(pout_buff->data, pout_buff->size, &pout_buff->offset, result, chunk_size); if (rc) goto err_put_buff; break; /* uncompressed, also fallback method */ case 0: default: checksum = update_adler32(1L, pin_buff->data, bytes_read); rc = write_chunk_header(pout_buff, 17, 0, bytes_read, checksum, bytes_read); if (rc) goto err_put_buff; rc = write_to_buf(pout_buff->data, pout_buff->size, &pout_buff->offset, pin_buff->data, bytes_read); if (rc) goto err_put_buff; break; } err_put_buff: release_scratch_buff(&tmp_buffer, pout_buff); err: return rc; } /* return non-zero if magic sequence is detected */ /* warning: reset the read pointer to the beginning of the file */ int detect_magic(struct cudbg_buffer *_c_buff) { unsigned char buffer[8]; size_t bytes_read; int c; bytes_read = read_from_buf(_c_buff->data, _c_buff->size, &_c_buff->offset, buffer, 8); if (bytes_read < 8) return 0; for (c = 0; c < 8; c++) if (buffer[c] != sixpack_magic[c]) return 0; return -1; } static inline unsigned long readU16(const unsigned char *ptr) { return ptr[0]+(ptr[1]<<8); } static inline unsigned long readU32(const unsigned char *ptr) { return ptr[0]+(ptr[1]<<8)+(ptr[2]<<16)+(ptr[3]<<24); } int read_chunk_header(struct cudbg_buffer *pc_buff, int *pid, int *poptions, unsigned long *psize, unsigned long *pchecksum, unsigned long *pextra) { unsigned char buffer[CUDBG_CHUNK_BUF_LEN]; int byte_r = read_from_buf(pc_buff->data, pc_buff->size, &pc_buff->offset, buffer, 16); if (byte_r == 0) return 0; *pid = readU16(buffer) & 0xffff; *poptions = readU16(buffer+2) & 0xffff; *psize = readU32(buffer+4) & 0xffffffff; *pchecksum = readU32(buffer+8) & 0xffffffff; *pextra = readU32(buffer+12) & 0xffffffff; return 0; } int validate_buffer(struct cudbg_buffer *compressed_buffer) { if (!detect_magic(compressed_buffer)) return CUDBG_STATUS_INVALID_BUFF; return 0; } int decompress_buffer(struct cudbg_buffer *pc_buff, struct cudbg_buffer *pd_buff) { struct cudbg_buffer tmp_compressed_buffer; struct cudbg_buffer tmp_decompressed_buffer; unsigned char *compressed_buffer; unsigned char *decompressed_buffer; unsigned char buffer[CUDBG_MIN_COMPR_LEN]; unsigned long chunk_size; unsigned long chunk_checksum; unsigned long chunk_extra; unsigned long checksum; unsigned long total_extracted = 0; unsigned long r; unsigned long remaining; unsigned long bytes_read; u32 decompressed_size = 0; int chunk_id, chunk_options, rc; if (pd_buff->size < 2 * CUDBG_BLOCK_SIZE) return CUDBG_STATUS_SMALL_BUFF; rc = get_scratch_buff(pd_buff, CUDBG_BLOCK_SIZE, &tmp_compressed_buffer); if (rc) goto err_cbuff; rc = get_scratch_buff(pd_buff, CUDBG_BLOCK_SIZE, &tmp_decompressed_buffer); if (rc) goto err_dcbuff; compressed_buffer = (unsigned char *)tmp_compressed_buffer.data; decompressed_buffer = (unsigned char *)tmp_decompressed_buffer.data; /* main loop */ for (;;) { if (pc_buff->offset > pc_buff->size) break; rc = read_chunk_header(pc_buff, &chunk_id, &chunk_options, &chunk_size, &chunk_checksum, &chunk_extra); if (rc != 0) break; /* skip 8+16 */ if ((chunk_id == 1) && (chunk_size > 10) && (chunk_size < CUDBG_BLOCK_SIZE)) { bytes_read = read_from_buf(pc_buff->data, pc_buff->size, &pc_buff->offset, buffer, chunk_size); if (bytes_read == 0) return 0; checksum = update_adler32(1L, buffer, chunk_size); if (checksum != chunk_checksum) return CUDBG_STATUS_CHKSUM_MISSMATCH; decompressed_size = (u32)readU32(buffer); if (pd_buff->size < decompressed_size) { pd_buff->size = 2 * CUDBG_BLOCK_SIZE + decompressed_size; pc_buff->offset -= chunk_size + 16; return CUDBG_STATUS_SMALL_BUFF; } total_extracted = 0; } if (chunk_size > CUDBG_BLOCK_SIZE) { /* Release old allocated memory */ release_scratch_buff(&tmp_decompressed_buffer, pd_buff); release_scratch_buff(&tmp_compressed_buffer, pd_buff); /* allocate new memory with chunk_size size */ rc = get_scratch_buff(pd_buff, chunk_size, &tmp_compressed_buffer); if (rc) goto err_cbuff; rc = get_scratch_buff(pd_buff, chunk_size, &tmp_decompressed_buffer); if (rc) goto err_dcbuff; compressed_buffer = (unsigned char *)tmp_compressed_buffer.data; decompressed_buffer = (unsigned char *)tmp_decompressed_buffer.data; } if ((chunk_id == 17) && decompressed_size) { /* uncompressed */ switch (chunk_options) { /* stored, simply copy to output */ case 0: total_extracted += chunk_size; remaining = chunk_size; checksum = 1L; for (;;) { - /* Write a funtion for this */ + /* Write a function for this */ r = (CUDBG_BLOCK_SIZE < remaining) ? CUDBG_BLOCK_SIZE : remaining; bytes_read = read_from_buf(pc_buff->data, pc_buff->size, &pc_buff->offset, buffer, r); if (bytes_read == 0) return 0; write_to_buf(pd_buff->data, pd_buff->size, &pd_buff->offset, buffer, bytes_read); checksum = update_adler32(checksum, buffer, bytes_read); remaining -= bytes_read; /* verify everything is written * correctly */ if (checksum != chunk_checksum) return CUDBG_STATUS_CHKSUM_MISSMATCH; } break; /* compressed using FastLZ */ case 1: bytes_read = read_from_buf(pc_buff->data, pc_buff->size, &pc_buff->offset, compressed_buffer, chunk_size); if (bytes_read == 0) return 0; checksum = update_adler32(1L, compressed_buffer, chunk_size); total_extracted += chunk_extra; /* verify that the chunk data is correct */ if (checksum != chunk_checksum) { return CUDBG_STATUS_CHKSUM_MISSMATCH; } else { /* decompress and verify */ remaining = fastlz_decompress(compressed_buffer, chunk_size, decompressed_buffer, chunk_extra); if (remaining != chunk_extra) { rc = CUDBG_STATUS_DECOMPRESS_FAIL; goto err; } else { write_to_buf(pd_buff->data, pd_buff->size, &pd_buff->offset, decompressed_buffer, chunk_extra); } } break; default: break; } } } err: release_scratch_buff(&tmp_decompressed_buffer, pd_buff); err_dcbuff: release_scratch_buff(&tmp_compressed_buffer, pd_buff); err_cbuff: return rc; } diff --git a/sys/dev/mii/lxtphy.c b/sys/dev/mii/lxtphy.c index 699c1d6f83b6..31900cc5a34b 100644 --- a/sys/dev/mii/lxtphy.c +++ b/sys/dev/mii/lxtphy.c @@ -1,269 +1,269 @@ /* OpenBSD: lxtphy.c,v 1.5 2000/08/26 20:04:17 nate Exp */ /* NetBSD: lxtphy.c,v 1.19 2000/02/02 23:34:57 thorpej Exp */ /*- * SPDX-License-Identifier: BSD-2-Clause-NetBSD * * Copyright (c) 1998, 1999 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1997 Manuel Bouyer. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * driver for Level One's LXT-970 ethernet 10/100 PHY * datasheet from www.level1.com */ #include #include #include #include #include #include #include #include #include #include #include #include "miidevs.h" #include #include "miibus_if.h" static int lxtphy_probe(device_t); static int lxtphy_attach(device_t); static device_method_t lxtphy_methods[] = { /* device interface */ DEVMETHOD(device_probe, lxtphy_probe), DEVMETHOD(device_attach, lxtphy_attach), DEVMETHOD(device_detach, mii_phy_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD_END }; static devclass_t lxtphy_devclass; static driver_t lxtphy_driver = { "lxtphy", lxtphy_methods, sizeof(struct mii_softc) }; DRIVER_MODULE(lxtphy, miibus, lxtphy_driver, lxtphy_devclass, 0, 0); static int lxtphy_service(struct mii_softc *, struct mii_data *, int); static void lxtphy_status(struct mii_softc *); static void lxtphy_reset(struct mii_softc *); static void lxtphy_set_tp(struct mii_softc *); static void lxtphy_set_fx(struct mii_softc *); static const struct mii_phydesc lxtphys[] = { MII_PHY_DESC(xxLEVEL1, LXT970), MII_PHY_END }; static const struct mii_phy_funcs lxtphy_funcs = { lxtphy_service, lxtphy_status, lxtphy_reset }; static int lxtphy_probe(device_t dev) { return (mii_phy_dev_probe(dev, lxtphys, BUS_PROBE_DEFAULT)); } static int lxtphy_attach(device_t dev) { struct mii_softc *sc; sc = device_get_softc(dev); mii_phy_dev_attach(dev, MIIF_NOMANPAUSE, &lxtphy_funcs, 0); PHY_RESET(sc); sc->mii_capabilities = PHY_READ(sc, MII_BMSR) & sc->mii_capmask; device_printf(dev, " "); #define ADD(m) ifmedia_add(&sc->mii_pdata->mii_media, (m), 0, NULL) ADD(IFM_MAKEWORD(IFM_ETHER, IFM_100_FX, 0, sc->mii_inst)); printf("100baseFX, "); ADD(IFM_MAKEWORD(IFM_ETHER, IFM_100_FX, IFM_FDX, sc->mii_inst)); printf("100baseFX-FDX, "); #undef ADD mii_phy_add_media(sc); printf("\n"); MIIBUS_MEDIAINIT(sc->mii_dev); return (0); } static int lxtphy_service(struct mii_softc *sc, struct mii_data *mii, int cmd) { struct ifmedia_entry *ife = mii->mii_media.ifm_cur; switch (cmd) { case MII_POLLSTAT: break; case MII_MEDIACHG: if (IFM_SUBTYPE(ife->ifm_media) == IFM_100_FX) lxtphy_set_fx(sc); else lxtphy_set_tp(sc); mii_phy_setmedia(sc); break; case MII_TICK: if (mii_phy_tick(sc) == EJUSTRETURN) return (0); break; } /* Update the media status. */ PHY_STATUS(sc); /* Callback if something changed. */ mii_phy_update(sc, cmd); return (0); } static void lxtphy_status(struct mii_softc *sc) { struct mii_data *mii = sc->mii_pdata; struct ifmedia_entry *ife = mii->mii_media.ifm_cur; int bmcr, bmsr, csr; mii->mii_media_status = IFM_AVALID; mii->mii_media_active = IFM_ETHER; /* * Get link status from the CSR; we need to read the CSR * for media type anyhow, and the link status in the CSR - * doens't latch, so fewer register reads are required. + * doesn't latch, so fewer register reads are required. */ csr = PHY_READ(sc, MII_LXTPHY_CSR); if (csr & CSR_LINK) mii->mii_media_status |= IFM_ACTIVE; bmcr = PHY_READ(sc, MII_BMCR); if (bmcr & BMCR_ISO) { mii->mii_media_active |= IFM_NONE; mii->mii_media_status = 0; return; } if (bmcr & BMCR_LOOP) mii->mii_media_active |= IFM_LOOP; if (bmcr & BMCR_AUTOEN) { bmsr = PHY_READ(sc, MII_BMSR) | PHY_READ(sc, MII_BMSR); if ((bmsr & BMSR_ACOMP) == 0) { /* Erg, still trying, I guess... */ mii->mii_media_active |= IFM_NONE; return; } if (csr & CSR_SPEED) mii->mii_media_active |= IFM_100_TX; else mii->mii_media_active |= IFM_10_T; if (csr & CSR_DUPLEX) mii->mii_media_active |= IFM_FDX | mii_phy_flowstatus(sc); else mii->mii_media_active |= IFM_HDX; } else mii->mii_media_active = ife->ifm_media; } static void lxtphy_reset(struct mii_softc *sc) { mii_phy_reset(sc); PHY_WRITE(sc, MII_LXTPHY_IER, PHY_READ(sc, MII_LXTPHY_IER) & ~IER_INTEN); } static void lxtphy_set_tp(struct mii_softc *sc) { int cfg; cfg = PHY_READ(sc, MII_LXTPHY_CONFIG); cfg &= ~CONFIG_100BASEFX; PHY_WRITE(sc, MII_LXTPHY_CONFIG, cfg); } static void lxtphy_set_fx(struct mii_softc *sc) { int cfg; cfg = PHY_READ(sc, MII_LXTPHY_CONFIG); cfg |= CONFIG_100BASEFX; PHY_WRITE(sc, MII_LXTPHY_CONFIG, cfg); } diff --git a/sys/dev/oce/oce_mbox.c b/sys/dev/oce/oce_mbox.c index 7470d7a13b07..321775c13d1e 100644 --- a/sys/dev/oce/oce_mbox.c +++ b/sys/dev/oce/oce_mbox.c @@ -1,2332 +1,2332 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2013 Emulex * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Emulex Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * Contact Information: * freebsd-drivers@emulex.com * * Emulex * 3333 Susan Street * Costa Mesa, CA 92626 */ /* $FreeBSD$ */ #include "oce_if.h" int oce_wait_ready(POCE_SOFTC sc) { #define SLIPORT_READY_TIMEOUT 30000 uint32_t sliport_status, i; if (!IS_XE201(sc)) return (-1); for (i = 0; i < SLIPORT_READY_TIMEOUT; i++) { sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); if (sliport_status & SLIPORT_STATUS_RDY_MASK) return 0; if (sliport_status & SLIPORT_STATUS_ERR_MASK && !(sliport_status & SLIPORT_STATUS_RN_MASK)) { device_printf(sc->dev, "Error detected in the card\n"); return EIO; } DELAY(1000); } device_printf(sc->dev, "Firmware wait timed out\n"); return (-1); } /** * @brief Reset (firmware) common function * @param sc software handle to the device * @returns 0 on success, ETIMEDOUT on failure */ int oce_reset_fun(POCE_SOFTC sc) { struct oce_mbx *mbx; struct oce_bmbx *mb; struct ioctl_common_function_reset *fwcmd; int rc = 0; if (IS_XE201(sc)) { OCE_WRITE_REG32(sc, db, SLIPORT_CONTROL_OFFSET, SLI_PORT_CONTROL_IP_MASK); rc = oce_wait_ready(sc); if (rc) { device_printf(sc->dev, "Firmware reset Failed\n"); } return rc; } mb = OCE_DMAPTR(&sc->bsmbx, struct oce_bmbx); mbx = &mb->mbx; bzero(mbx, sizeof(struct oce_mbx)); fwcmd = (struct ioctl_common_function_reset *)&mbx->payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_FUNCTION_RESET, 10, /* MBX_TIMEOUT_SEC */ sizeof(struct ioctl_common_function_reset), OCE_MBX_VER_V0); mbx->u0.s.embedded = 1; mbx->payload_length = sizeof(struct ioctl_common_function_reset); rc = oce_mbox_dispatch(sc, 2); return rc; } /** - * @brief This funtions tells firmware we are + * @brief This functions tells firmware we are * done with commands. * @param sc software handle to the device * @returns 0 on success, ETIMEDOUT on failure */ int oce_fw_clean(POCE_SOFTC sc) { struct oce_bmbx *mbx; uint8_t *ptr; int ret = 0; mbx = OCE_DMAPTR(&sc->bsmbx, struct oce_bmbx); ptr = (uint8_t *) &mbx->mbx; /* Endian Signature */ *ptr++ = 0xff; *ptr++ = 0xaa; *ptr++ = 0xbb; *ptr++ = 0xff; *ptr++ = 0xff; *ptr++ = 0xcc; *ptr++ = 0xdd; *ptr = 0xff; ret = oce_mbox_dispatch(sc, 2); return ret; } /** * @brief Mailbox wait * @param sc software handle to the device * @param tmo_sec timeout in seconds */ static int oce_mbox_wait(POCE_SOFTC sc, uint32_t tmo_sec) { tmo_sec *= 10000; pd_mpu_mbox_db_t mbox_db; for (;;) { if (tmo_sec != 0) { if (--tmo_sec == 0) break; } mbox_db.dw0 = OCE_READ_REG32(sc, db, PD_MPU_MBOX_DB); if (mbox_db.bits.ready) return 0; DELAY(100); } device_printf(sc->dev, "Mailbox timed out\n"); return ETIMEDOUT; } /** * @brief Mailbox dispatch * @param sc software handle to the device * @param tmo_sec timeout in seconds */ int oce_mbox_dispatch(POCE_SOFTC sc, uint32_t tmo_sec) { pd_mpu_mbox_db_t mbox_db; uint32_t pa; int rc; oce_dma_sync(&sc->bsmbx, BUS_DMASYNC_PREWRITE); pa = (uint32_t) ((uint64_t) sc->bsmbx.paddr >> 34); bzero(&mbox_db, sizeof(pd_mpu_mbox_db_t)); mbox_db.bits.ready = 0; mbox_db.bits.hi = 1; mbox_db.bits.address = pa; rc = oce_mbox_wait(sc, tmo_sec); if (rc == 0) { OCE_WRITE_REG32(sc, db, PD_MPU_MBOX_DB, mbox_db.dw0); pa = (uint32_t) ((uint64_t) sc->bsmbx.paddr >> 4) & 0x3fffffff; mbox_db.bits.ready = 0; mbox_db.bits.hi = 0; mbox_db.bits.address = pa; rc = oce_mbox_wait(sc, tmo_sec); if (rc == 0) { OCE_WRITE_REG32(sc, db, PD_MPU_MBOX_DB, mbox_db.dw0); rc = oce_mbox_wait(sc, tmo_sec); oce_dma_sync(&sc->bsmbx, BUS_DMASYNC_POSTWRITE); } } return rc; } /** * @brief Mailbox common request header initialization * @param hdr mailbox header * @param dom domain * @param port port * @param subsys subsystem * @param opcode opcode * @param timeout timeout * @param pyld_len payload length */ void mbx_common_req_hdr_init(struct mbx_hdr *hdr, uint8_t dom, uint8_t port, uint8_t subsys, uint8_t opcode, uint32_t timeout, uint32_t pyld_len, uint8_t version) { hdr->u0.req.opcode = opcode; hdr->u0.req.subsystem = subsys; hdr->u0.req.port_number = port; hdr->u0.req.domain = dom; hdr->u0.req.timeout = timeout; hdr->u0.req.request_length = pyld_len - sizeof(struct mbx_hdr); hdr->u0.req.version = version; } /** * @brief Function to initialize the hw with host endian information * @param sc software handle to the device * @returns 0 on success, ETIMEDOUT on failure */ int oce_mbox_init(POCE_SOFTC sc) { struct oce_bmbx *mbx; uint8_t *ptr; int ret = 0; if (sc->flags & OCE_FLAGS_MBOX_ENDIAN_RQD) { mbx = OCE_DMAPTR(&sc->bsmbx, struct oce_bmbx); ptr = (uint8_t *) &mbx->mbx; /* Endian Signature */ *ptr++ = 0xff; *ptr++ = 0x12; *ptr++ = 0x34; *ptr++ = 0xff; *ptr++ = 0xff; *ptr++ = 0x56; *ptr++ = 0x78; *ptr = 0xff; ret = oce_mbox_dispatch(sc, 0); } return ret; } /** * @brief Function to get the firmware version * @param sc software handle to the device * @returns 0 on success, EIO on failure */ int oce_get_fw_version(POCE_SOFTC sc) { struct oce_mbx mbx; struct mbx_get_common_fw_version *fwcmd; int ret = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_get_common_fw_version *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_GET_FW_VERSION, MBX_TIMEOUT_SEC, sizeof(struct mbx_get_common_fw_version), OCE_MBX_VER_V0); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_get_common_fw_version); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); ret = oce_mbox_post(sc, &mbx, NULL); if (!ret) ret = fwcmd->hdr.u0.rsp.status; if (ret) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, ret, fwcmd->hdr.u0.rsp.additional_status); goto error; } bcopy(fwcmd->params.rsp.fw_ver_str, sc->fw_version, 32); error: return ret; } /** * @brief Firmware will send gracious notifications during * attach only after sending first mcc commnad. We * use MCC queue only for getting async and mailbox * for sending cmds. So to get gracious notifications * atleast send one dummy command on mcc. */ int oce_first_mcc_cmd(POCE_SOFTC sc) { struct oce_mbx *mbx; struct oce_mq *mq = sc->mq; struct mbx_get_common_fw_version *fwcmd; uint32_t reg_value; mbx = RING_GET_PRODUCER_ITEM_VA(mq->ring, struct oce_mbx); bzero(mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_get_common_fw_version *)&mbx->payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_GET_FW_VERSION, MBX_TIMEOUT_SEC, sizeof(struct mbx_get_common_fw_version), OCE_MBX_VER_V0); mbx->u0.s.embedded = 1; mbx->payload_length = sizeof(struct mbx_get_common_fw_version); bus_dmamap_sync(mq->ring->dma.tag, mq->ring->dma.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); RING_PUT(mq->ring, 1); reg_value = (1 << 16) | mq->mq_id; OCE_WRITE_REG32(sc, db, PD_MQ_DB, reg_value); return 0; } /** * @brief Function to post a MBX to the mbox * @param sc software handle to the device * @param mbx pointer to the MBX to send * @param mbxctx pointer to the mbx context structure * @returns 0 on success, error on failure */ int oce_mbox_post(POCE_SOFTC sc, struct oce_mbx *mbx, struct oce_mbx_ctx *mbxctx) { struct oce_mbx *mb_mbx = NULL; struct oce_mq_cqe *mb_cqe = NULL; struct oce_bmbx *mb = NULL; int rc = 0; uint32_t tmo = 0; uint32_t cstatus = 0; uint32_t xstatus = 0; LOCK(&sc->bmbx_lock); mb = OCE_DMAPTR(&sc->bsmbx, struct oce_bmbx); mb_mbx = &mb->mbx; /* get the tmo */ tmo = mbx->tag[0]; mbx->tag[0] = 0; /* copy mbx into mbox */ bcopy(mbx, mb_mbx, sizeof(struct oce_mbx)); /* now dispatch */ rc = oce_mbox_dispatch(sc, tmo); if (rc == 0) { /* * the command completed successfully. Now get the * completion queue entry */ mb_cqe = &mb->cqe; DW_SWAP(u32ptr(&mb_cqe->u0.dw[0]), sizeof(struct oce_mq_cqe)); /* copy mbox mbx back */ bcopy(mb_mbx, mbx, sizeof(struct oce_mbx)); /* pick up the mailbox status */ cstatus = mb_cqe->u0.s.completion_status; xstatus = mb_cqe->u0.s.extended_status; /* * store the mbx context in the cqe tag section so that * the upper layer handling the cqe can associate the mbx * with the response */ if (cstatus == 0 && mbxctx) { /* save context */ mbxctx->mbx = mb_mbx; bcopy(&mbxctx, mb_cqe->u0.s.mq_tag, sizeof(struct oce_mbx_ctx *)); } } UNLOCK(&sc->bmbx_lock); return rc; } /** * @brief Function to read the mac address associated with an interface * @param sc software handle to the device * @param if_id interface id to read the address from * @param perm set to 1 if reading the factory mac address. * In this case if_id is ignored * @param type type of the mac address, whether network or storage * @param[out] mac [OUTPUT] pointer to a buffer containing the * mac address when the command succeeds. * @returns 0 on success, EIO on failure */ int oce_read_mac_addr(POCE_SOFTC sc, uint32_t if_id, uint8_t perm, uint8_t type, struct mac_address_format *mac) { struct oce_mbx mbx; struct mbx_query_common_iface_mac *fwcmd; int ret = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_query_common_iface_mac *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_QUERY_IFACE_MAC, MBX_TIMEOUT_SEC, sizeof(struct mbx_query_common_iface_mac), OCE_MBX_VER_V0); fwcmd->params.req.permanent = perm; if (!perm) fwcmd->params.req.if_id = (uint16_t) if_id; else fwcmd->params.req.if_id = 0; fwcmd->params.req.type = type; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_query_common_iface_mac); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); ret = oce_mbox_post(sc, &mbx, NULL); if (!ret) ret = fwcmd->hdr.u0.rsp.status; if (ret) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, ret, fwcmd->hdr.u0.rsp.additional_status); goto error; } /* copy the mac addres in the output parameter */ mac->size_of_struct = fwcmd->params.rsp.mac.size_of_struct; bcopy(&fwcmd->params.rsp.mac.mac_addr[0], &mac->mac_addr[0], mac->size_of_struct); error: return ret; } /** * @brief Function to query the fw attributes from the hw * @param sc software handle to the device * @returns 0 on success, EIO on failure */ int oce_get_fw_config(POCE_SOFTC sc) { struct oce_mbx mbx; struct mbx_common_query_fw_config *fwcmd; int ret = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_common_query_fw_config *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_QUERY_FIRMWARE_CONFIG, MBX_TIMEOUT_SEC, sizeof(struct mbx_common_query_fw_config), OCE_MBX_VER_V0); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_common_query_fw_config); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); ret = oce_mbox_post(sc, &mbx, NULL); if (!ret) ret = fwcmd->hdr.u0.rsp.status; if (ret) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, ret, fwcmd->hdr.u0.rsp.additional_status); goto error; } DW_SWAP(u32ptr(fwcmd), sizeof(struct mbx_common_query_fw_config)); sc->config_number = HOST_32(fwcmd->params.rsp.config_number); sc->asic_revision = HOST_32(fwcmd->params.rsp.asic_revision); sc->port_id = HOST_32(fwcmd->params.rsp.port_id); sc->function_mode = HOST_32(fwcmd->params.rsp.function_mode); if ((sc->function_mode & (ULP_NIC_MODE | ULP_RDMA_MODE)) == (ULP_NIC_MODE | ULP_RDMA_MODE)) { sc->rdma_flags = OCE_RDMA_FLAG_SUPPORTED; } sc->function_caps = HOST_32(fwcmd->params.rsp.function_caps); if (fwcmd->params.rsp.ulp[0].ulp_mode & ULP_NIC_MODE) { sc->max_tx_rings = HOST_32(fwcmd->params.rsp.ulp[0].nic_wq_tot); sc->max_rx_rings = HOST_32(fwcmd->params.rsp.ulp[0].lro_rqid_tot); } else { sc->max_tx_rings = HOST_32(fwcmd->params.rsp.ulp[1].nic_wq_tot); sc->max_rx_rings = HOST_32(fwcmd->params.rsp.ulp[1].lro_rqid_tot); } error: return ret; } /** * * @brief function to create a device interface * @param sc software handle to the device * @param cap_flags capability flags * @param en_flags enable capability flags * @param vlan_tag optional vlan tag to associate with the if * @param mac_addr pointer to a buffer containing the mac address * @param[out] if_id [OUTPUT] pointer to an integer to hold the ID of the interface created * @returns 0 on success, EIO on failure */ int oce_if_create(POCE_SOFTC sc, uint32_t cap_flags, uint32_t en_flags, uint16_t vlan_tag, uint8_t *mac_addr, uint32_t *if_id) { struct oce_mbx mbx; struct mbx_create_common_iface *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_create_common_iface *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_CREATE_IFACE, MBX_TIMEOUT_SEC, sizeof(struct mbx_create_common_iface), OCE_MBX_VER_V0); DW_SWAP(u32ptr(&fwcmd->hdr), sizeof(struct mbx_hdr)); fwcmd->params.req.version = 0; fwcmd->params.req.cap_flags = LE_32(cap_flags); fwcmd->params.req.enable_flags = LE_32(en_flags); if (mac_addr != NULL) { bcopy(mac_addr, &fwcmd->params.req.mac_addr[0], 6); fwcmd->params.req.vlan_tag.u0.normal.vtag = LE_16(vlan_tag); fwcmd->params.req.mac_invalid = 0; } else { fwcmd->params.req.mac_invalid = 1; } mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_common_iface); DW_SWAP(u32ptr(&mbx), OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } *if_id = HOST_32(fwcmd->params.rsp.if_id); if (mac_addr != NULL) sc->pmac_id = HOST_32(fwcmd->params.rsp.pmac_id); error: return rc; } /** * @brief Function to delete an interface * @param sc software handle to the device * @param if_id ID of the interface to delete * @returns 0 on success, EIO on failure */ int oce_if_del(POCE_SOFTC sc, uint32_t if_id) { struct oce_mbx mbx; struct mbx_destroy_common_iface *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_destroy_common_iface *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_DESTROY_IFACE, MBX_TIMEOUT_SEC, sizeof(struct mbx_destroy_common_iface), OCE_MBX_VER_V0); fwcmd->params.req.if_id = if_id; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_destroy_common_iface); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } /** * @brief Function to send the mbx command to configure vlan * @param sc software handle to the device * @param if_id interface identifier index * @param vtag_arr array of vlan tags * @param vtag_cnt number of elements in array * @param untagged boolean TRUE/FLASE * @param enable_promisc flag to enable/disable VLAN promiscuous mode * @returns 0 on success, EIO on failure */ int oce_config_vlan(POCE_SOFTC sc, uint32_t if_id, struct normal_vlan *vtag_arr, uint8_t vtag_cnt, uint32_t untagged, uint32_t enable_promisc) { struct oce_mbx mbx; struct mbx_common_config_vlan *fwcmd; int rc = 0; if (sc->vlans_added > sc->max_vlans) goto vlan_promisc; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_common_config_vlan *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_CONFIG_IFACE_VLAN, MBX_TIMEOUT_SEC, sizeof(struct mbx_common_config_vlan), OCE_MBX_VER_V0); fwcmd->params.req.if_id = (uint8_t) if_id; fwcmd->params.req.promisc = (uint8_t) enable_promisc; fwcmd->params.req.untagged = (uint8_t) untagged; fwcmd->params.req.num_vlans = vtag_cnt; if (!enable_promisc) { bcopy(vtag_arr, fwcmd->params.req.tags.normal_vlans, vtag_cnt * sizeof(struct normal_vlan)); } mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_common_config_vlan); DW_SWAP(u32ptr(&mbx), (OCE_BMBX_RHDR_SZ + mbx.payload_length)); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto done; vlan_promisc: /* Enable Vlan Promis */ oce_rxf_set_promiscuous(sc, (1 << 1)); device_printf(sc->dev,"Enabling Vlan Promisc Mode\n"); done: return rc; } /** * @brief Function to set flow control capability in the hardware * @param sc software handle to the device * @param flow_control flow control flags to set * @returns 0 on success, EIO on failure */ int oce_set_flow_control(POCE_SOFTC sc, uint32_t flow_control) { struct oce_mbx mbx; struct mbx_common_get_set_flow_control *fwcmd = (struct mbx_common_get_set_flow_control *)&mbx.payload; int rc; bzero(&mbx, sizeof(struct oce_mbx)); mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_SET_FLOW_CONTROL, MBX_TIMEOUT_SEC, sizeof(struct mbx_common_get_set_flow_control), OCE_MBX_VER_V0); if (flow_control & OCE_FC_TX) fwcmd->tx_flow_control = 1; if (flow_control & OCE_FC_RX) fwcmd->rx_flow_control = 1; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_common_get_set_flow_control); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } /** * @brief Initialize the RSS CPU indirection table * * The table is used to choose the queue to place the incomming packets. * Incomming packets are hashed. The lowest bits in the hash result * are used as the index into the CPU indirection table. * Each entry in the table contains the RSS CPU-ID returned by the NIC * create. Based on the CPU ID, the receive completion is routed to * the corresponding RSS CQs. (Non-RSS packets are always completed * on the default (0) CQ). * * @param sc software handle to the device * @param *fwcmd pointer to the rss mbox command * @returns none */ static int oce_rss_itbl_init(POCE_SOFTC sc, struct mbx_config_nic_rss *fwcmd) { int i = 0, j = 0, rc = 0; uint8_t *tbl = fwcmd->params.req.cputable; struct oce_rq *rq = NULL; for (j = 0; j < INDIRECTION_TABLE_ENTRIES ; j += (sc->nrqs - 1)) { for_all_rss_queues(sc, rq, i) { if ((j + i) >= INDIRECTION_TABLE_ENTRIES) break; tbl[j + i] = rq->rss_cpuid; } } if (i == 0) { device_printf(sc->dev, "error: Invalid number of RSS RQ's\n"); rc = ENXIO; } /* fill log2 value indicating the size of the CPU table */ if (rc == 0) fwcmd->params.req.cpu_tbl_sz_log2 = LE_16(OCE_LOG2(INDIRECTION_TABLE_ENTRIES)); return rc; } /** * @brief Function to set flow control capability in the hardware * @param sc software handle to the device * @param if_id interface id to read the address from * @param enable_rss 0=disable, RSS_ENABLE_xxx flags otherwise * @returns 0 on success, EIO on failure */ int oce_config_nic_rss(POCE_SOFTC sc, uint32_t if_id, uint16_t enable_rss) { int rc; struct oce_mbx mbx; struct mbx_config_nic_rss *fwcmd = (struct mbx_config_nic_rss *)&mbx.payload; int version; bzero(&mbx, sizeof(struct oce_mbx)); if (IS_XE201(sc) || IS_SH(sc)) { version = OCE_MBX_VER_V1; fwcmd->params.req.enable_rss = RSS_ENABLE_UDP_IPV4 | RSS_ENABLE_UDP_IPV6; } else version = OCE_MBX_VER_V0; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, NIC_CONFIG_RSS, MBX_TIMEOUT_SEC, sizeof(struct mbx_config_nic_rss), version); if (enable_rss) fwcmd->params.req.enable_rss |= (RSS_ENABLE_IPV4 | RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV6 | RSS_ENABLE_TCP_IPV6); if(!sc->enable_hwlro) fwcmd->params.req.flush = OCE_FLUSH; else fwcmd->params.req.flush = 0; fwcmd->params.req.if_id = LE_32(if_id); read_random(fwcmd->params.req.hash, sizeof(fwcmd->params.req.hash)); rc = oce_rss_itbl_init(sc, fwcmd); if (rc == 0) { mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_config_nic_rss); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); } return rc; } /** * @brief RXF function to enable/disable device promiscuous mode * @param sc software handle to the device * @param enable enable/disable flag * @returns 0 on success, EIO on failure * @note * The NIC_CONFIG_PROMISCUOUS command deprecated for Lancer. * This function uses the COMMON_SET_IFACE_RX_FILTER command instead. */ int oce_rxf_set_promiscuous(POCE_SOFTC sc, uint8_t enable) { struct mbx_set_common_iface_rx_filter *fwcmd; int sz = sizeof(struct mbx_set_common_iface_rx_filter); iface_rx_filter_ctx_t *req; OCE_DMA_MEM sgl; int rc; /* allocate mbx payload's dma scatter/gather memory */ rc = oce_dma_alloc(sc, sz, &sgl, 0); if (rc) return rc; fwcmd = OCE_DMAPTR(&sgl, struct mbx_set_common_iface_rx_filter); req = &fwcmd->params.req; req->iface_flags_mask = MBX_RX_IFACE_FLAGS_PROMISCUOUS | MBX_RX_IFACE_FLAGS_VLAN_PROMISCUOUS; /* Bit 0 Mac promisc, Bit 1 Vlan promisc */ if (enable & 0x01) req->iface_flags = MBX_RX_IFACE_FLAGS_PROMISCUOUS; if (enable & 0x02) req->iface_flags |= MBX_RX_IFACE_FLAGS_VLAN_PROMISCUOUS; req->if_id = sc->if_id; rc = oce_set_common_iface_rx_filter(sc, &sgl); oce_dma_free(sc, &sgl); return rc; } /** * @brief Function modify and select rx filter options * @param sc software handle to the device * @param sgl scatter/gather request/response * @returns 0 on success, error code on failure */ int oce_set_common_iface_rx_filter(POCE_SOFTC sc, POCE_DMA_MEM sgl) { struct oce_mbx mbx; int mbx_sz = sizeof(struct mbx_set_common_iface_rx_filter); struct mbx_set_common_iface_rx_filter *fwcmd; int rc; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = OCE_DMAPTR(sgl, struct mbx_set_common_iface_rx_filter); mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_SET_IFACE_RX_FILTER, MBX_TIMEOUT_SEC, mbx_sz, OCE_MBX_VER_V0); oce_dma_sync(sgl, BUS_DMASYNC_PREWRITE); mbx.u0.s.embedded = 0; mbx.u0.s.sge_count = 1; mbx.payload.u0.u1.sgl[0].pa_lo = ADDR_LO(sgl->paddr); mbx.payload.u0.u1.sgl[0].pa_hi = ADDR_HI(sgl->paddr); mbx.payload.u0.u1.sgl[0].length = mbx_sz; mbx.payload_length = mbx_sz; DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } /** * @brief Function to query the link status from the hardware * @param sc software handle to the device * @param[out] link pointer to the structure returning link attributes * @returns 0 on success, EIO on failure */ int oce_get_link_status(POCE_SOFTC sc, struct link_status *link) { struct oce_mbx mbx; struct mbx_query_common_link_config *fwcmd; int rc = 0, version; bzero(&mbx, sizeof(struct oce_mbx)); IS_BE2(sc) ? (version = OCE_MBX_VER_V0) : (version = OCE_MBX_VER_V1); fwcmd = (struct mbx_query_common_link_config *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_QUERY_LINK_CONFIG, MBX_TIMEOUT_SEC, sizeof(struct mbx_query_common_link_config), version); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_query_common_link_config); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } /* interpret response */ link->qos_link_speed = HOST_16(fwcmd->params.rsp.qos_link_speed); link->phys_port_speed = fwcmd->params.rsp.physical_port_speed; link->logical_link_status = fwcmd->params.rsp.logical_link_status; error: return rc; } /** * @brief Function to get NIC statistics * @param sc software handle to the device * @param *stats pointer to where to store statistics * @param reset_stats resets statistics of set * @returns 0 on success, EIO on failure * @note command depricated in Lancer */ #define OCE_MBOX_GET_NIC_STATS(sc, pstats_dma_mem, version) \ int \ oce_mbox_get_nic_stats_v##version(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem) \ { \ struct oce_mbx mbx; \ struct mbx_get_nic_stats_v##version *fwcmd; \ int rc = 0; \ \ bzero(&mbx, sizeof(struct oce_mbx)); \ fwcmd = OCE_DMAPTR(pstats_dma_mem, struct mbx_get_nic_stats_v##version); \ bzero(fwcmd, sizeof(*fwcmd)); \ \ mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, \ MBX_SUBSYSTEM_NIC, \ NIC_GET_STATS, \ MBX_TIMEOUT_SEC, \ sizeof(*fwcmd), \ OCE_MBX_VER_V##version); \ \ mbx.u0.s.embedded = 0; /* stats too large for embedded mbx rsp */ \ mbx.u0.s.sge_count = 1; /* using scatter gather instead */ \ \ oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_PREWRITE); \ mbx.payload.u0.u1.sgl[0].pa_lo = ADDR_LO(pstats_dma_mem->paddr); \ mbx.payload.u0.u1.sgl[0].pa_hi = ADDR_HI(pstats_dma_mem->paddr); \ mbx.payload.u0.u1.sgl[0].length = sizeof(*fwcmd); \ mbx.payload_length = sizeof(*fwcmd); \ DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); \ \ rc = oce_mbox_post(sc, &mbx, NULL); \ oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_POSTWRITE); \ if (!rc) \ rc = fwcmd->hdr.u0.rsp.status; \ if (rc) \ device_printf(sc->dev, \ "%s failed - cmd status: %d addi status: %d\n", \ __FUNCTION__, rc, \ fwcmd->hdr.u0.rsp.additional_status); \ return rc; \ } OCE_MBOX_GET_NIC_STATS(sc, pstats_dma_mem, 0); OCE_MBOX_GET_NIC_STATS(sc, pstats_dma_mem, 1); OCE_MBOX_GET_NIC_STATS(sc, pstats_dma_mem, 2); /** * @brief Function to get pport (physical port) statistics * @param sc software handle to the device * @param *stats pointer to where to store statistics * @param reset_stats resets statistics of set * @returns 0 on success, EIO on failure */ int oce_mbox_get_pport_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem, uint32_t reset_stats) { struct oce_mbx mbx; struct mbx_get_pport_stats *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = OCE_DMAPTR(pstats_dma_mem, struct mbx_get_pport_stats); bzero(fwcmd, sizeof(struct mbx_get_pport_stats)); mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, NIC_GET_PPORT_STATS, MBX_TIMEOUT_SEC, sizeof(struct mbx_get_pport_stats), OCE_MBX_VER_V0); fwcmd->params.req.reset_stats = reset_stats; fwcmd->params.req.port_number = sc->port_id; mbx.u0.s.embedded = 0; /* stats too large for embedded mbx rsp */ mbx.u0.s.sge_count = 1; /* using scatter gather instead */ oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_PREWRITE); mbx.payload.u0.u1.sgl[0].pa_lo = ADDR_LO(pstats_dma_mem->paddr); mbx.payload.u0.u1.sgl[0].pa_hi = ADDR_HI(pstats_dma_mem->paddr); mbx.payload.u0.u1.sgl[0].length = sizeof(struct mbx_get_pport_stats); mbx.payload_length = sizeof(struct mbx_get_pport_stats); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_POSTWRITE); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } /** * @brief Function to get vport (virtual port) statistics * @param sc software handle to the device * @param *stats pointer to where to store statistics * @param reset_stats resets statistics of set * @returns 0 on success, EIO on failure */ int oce_mbox_get_vport_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem, uint32_t req_size, uint32_t reset_stats) { struct oce_mbx mbx; struct mbx_get_vport_stats *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = OCE_DMAPTR(pstats_dma_mem, struct mbx_get_vport_stats); bzero(fwcmd, sizeof(struct mbx_get_vport_stats)); mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, NIC_GET_VPORT_STATS, MBX_TIMEOUT_SEC, sizeof(struct mbx_get_vport_stats), OCE_MBX_VER_V0); fwcmd->params.req.reset_stats = reset_stats; fwcmd->params.req.vport_number = sc->if_id; mbx.u0.s.embedded = 0; /* stats too large for embedded mbx rsp */ mbx.u0.s.sge_count = 1; /* using scatter gather instead */ oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_PREWRITE); mbx.payload.u0.u1.sgl[0].pa_lo = ADDR_LO(pstats_dma_mem->paddr); mbx.payload.u0.u1.sgl[0].pa_hi = ADDR_HI(pstats_dma_mem->paddr); mbx.payload.u0.u1.sgl[0].length = sizeof(struct mbx_get_vport_stats); mbx.payload_length = sizeof(struct mbx_get_vport_stats); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_POSTWRITE); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } /** * @brief Function to update the muticast filter with * values in dma_mem * @param sc software handle to the device * @param dma_mem pointer to dma memory region * @returns 0 on success, EIO on failure */ int oce_update_multicast(POCE_SOFTC sc, POCE_DMA_MEM pdma_mem) { struct oce_mbx mbx; struct oce_mq_sge *sgl; struct mbx_set_common_iface_multicast *req = NULL; int rc = 0; req = OCE_DMAPTR(pdma_mem, struct mbx_set_common_iface_multicast); mbx_common_req_hdr_init(&req->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_SET_IFACE_MULTICAST, MBX_TIMEOUT_SEC, sizeof(struct mbx_set_common_iface_multicast), OCE_MBX_VER_V0); bzero(&mbx, sizeof(struct oce_mbx)); mbx.u0.s.embedded = 0; /*Non embeded*/ mbx.payload_length = sizeof(struct mbx_set_common_iface_multicast); mbx.u0.s.sge_count = 1; sgl = &mbx.payload.u0.u1.sgl[0]; sgl->pa_hi = htole32(upper_32_bits(pdma_mem->paddr)); sgl->pa_lo = htole32((pdma_mem->paddr) & 0xFFFFFFFF); sgl->length = htole32(mbx.payload_length); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = req->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, req->hdr.u0.rsp.additional_status); return rc; } /** * @brief Function to send passthrough Ioctls * @param sc software handle to the device * @param dma_mem pointer to dma memory region * @param req_size size of dma_mem * @returns 0 on success, EIO on failure */ int oce_pass_through_mbox(POCE_SOFTC sc, POCE_DMA_MEM dma_mem, uint32_t req_size) { struct oce_mbx mbx; struct oce_mq_sge *sgl; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); mbx.u0.s.embedded = 0; /*Non embeded*/ mbx.payload_length = req_size; mbx.u0.s.sge_count = 1; sgl = &mbx.payload.u0.u1.sgl[0]; sgl->pa_hi = htole32(upper_32_bits(dma_mem->paddr)); sgl->pa_lo = htole32((dma_mem->paddr) & 0xFFFFFFFF); sgl->length = htole32(req_size); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); return rc; } int oce_mbox_macaddr_add(POCE_SOFTC sc, uint8_t *mac_addr, uint32_t if_id, uint32_t *pmac_id) { struct oce_mbx mbx; struct mbx_add_common_iface_mac *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_add_common_iface_mac *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_ADD_IFACE_MAC, MBX_TIMEOUT_SEC, sizeof(struct mbx_add_common_iface_mac), OCE_MBX_VER_V0); fwcmd->params.req.if_id = (uint16_t) if_id; bcopy(mac_addr, fwcmd->params.req.mac_address, 6); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_add_common_iface_mac); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } *pmac_id = fwcmd->params.rsp.pmac_id; error: return rc; } int oce_mbox_macaddr_del(POCE_SOFTC sc, uint32_t if_id, uint32_t pmac_id) { struct oce_mbx mbx; struct mbx_del_common_iface_mac *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_del_common_iface_mac *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_DEL_IFACE_MAC, MBX_TIMEOUT_SEC, sizeof(struct mbx_del_common_iface_mac), OCE_MBX_VER_V0); fwcmd->params.req.if_id = (uint16_t)if_id; fwcmd->params.req.pmac_id = pmac_id; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_del_common_iface_mac); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } int oce_mbox_check_native_mode(POCE_SOFTC sc) { struct oce_mbx mbx; struct mbx_common_set_function_cap *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_common_set_function_cap *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_SET_FUNCTIONAL_CAPS, MBX_TIMEOUT_SEC, sizeof(struct mbx_common_set_function_cap), OCE_MBX_VER_V0); fwcmd->params.req.valid_capability_flags = CAP_SW_TIMESTAMPS | CAP_BE3_NATIVE_ERX_API; fwcmd->params.req.capability_flags = CAP_BE3_NATIVE_ERX_API; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_common_set_function_cap); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } sc->be3_native = HOST_32(fwcmd->params.rsp.capability_flags) & CAP_BE3_NATIVE_ERX_API; error: return 0; } int oce_mbox_cmd_set_loopback(POCE_SOFTC sc, uint8_t port_num, uint8_t loopback_type, uint8_t enable) { struct oce_mbx mbx; struct mbx_lowlevel_set_loopback_mode *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_lowlevel_set_loopback_mode *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_LOWLEVEL, OPCODE_LOWLEVEL_SET_LOOPBACK_MODE, MBX_TIMEOUT_SEC, sizeof(struct mbx_lowlevel_set_loopback_mode), OCE_MBX_VER_V0); fwcmd->params.req.src_port = port_num; fwcmd->params.req.dest_port = port_num; fwcmd->params.req.loopback_type = loopback_type; fwcmd->params.req.loopback_state = enable; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_lowlevel_set_loopback_mode); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } int oce_mbox_cmd_test_loopback(POCE_SOFTC sc, uint32_t port_num, uint32_t loopback_type, uint32_t pkt_size, uint32_t num_pkts, uint64_t pattern) { struct oce_mbx mbx; struct mbx_lowlevel_test_loopback_mode *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_lowlevel_test_loopback_mode *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_LOWLEVEL, OPCODE_LOWLEVEL_TEST_LOOPBACK, MBX_TIMEOUT_SEC, sizeof(struct mbx_lowlevel_test_loopback_mode), OCE_MBX_VER_V0); fwcmd->params.req.pattern = pattern; fwcmd->params.req.src_port = port_num; fwcmd->params.req.dest_port = port_num; fwcmd->params.req.pkt_size = pkt_size; fwcmd->params.req.num_pkts = num_pkts; fwcmd->params.req.loopback_type = loopback_type; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_lowlevel_test_loopback_mode); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } int oce_mbox_write_flashrom(POCE_SOFTC sc, uint32_t optype,uint32_t opcode, POCE_DMA_MEM pdma_mem, uint32_t num_bytes) { struct oce_mbx mbx; struct oce_mq_sge *sgl = NULL; struct mbx_common_read_write_flashrom *fwcmd = NULL; int rc = 0, payload_len = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = OCE_DMAPTR(pdma_mem, struct mbx_common_read_write_flashrom); payload_len = sizeof(struct mbx_common_read_write_flashrom) + 32*1024; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_WRITE_FLASHROM, LONG_TIMEOUT, payload_len, OCE_MBX_VER_V0); fwcmd->flash_op_type = LE_32(optype); fwcmd->flash_op_code = LE_32(opcode); fwcmd->data_buffer_size = LE_32(num_bytes); mbx.u0.s.embedded = 0; /*Non embeded*/ mbx.payload_length = payload_len; mbx.u0.s.sge_count = 1; sgl = &mbx.payload.u0.u1.sgl[0]; sgl->pa_hi = upper_32_bits(pdma_mem->paddr); sgl->pa_lo = pdma_mem->paddr & 0xFFFFFFFF; sgl->length = payload_len; /* post the command */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } int oce_mbox_get_flashrom_crc(POCE_SOFTC sc, uint8_t *flash_crc, uint32_t offset, uint32_t optype) { int rc = 0, payload_len = 0; struct oce_mbx mbx; struct mbx_common_read_write_flashrom *fwcmd; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_common_read_write_flashrom *)&mbx.payload; /* Firmware requires extra 4 bytes with this ioctl. Since there is enough room in the mbx payload it should be good enough Reference: Bug 14853 */ payload_len = sizeof(struct mbx_common_read_write_flashrom) + 4; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_READ_FLASHROM, MBX_TIMEOUT_SEC, payload_len, OCE_MBX_VER_V0); fwcmd->flash_op_type = optype; fwcmd->flash_op_code = FLASHROM_OPER_REPORT; fwcmd->data_offset = offset; fwcmd->data_buffer_size = 0x4; mbx.u0.s.embedded = 1; mbx.payload_length = payload_len; /* post the command */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } bcopy(fwcmd->data_buffer, flash_crc, 4); error: return rc; } int oce_mbox_get_phy_info(POCE_SOFTC sc, struct oce_phy_info *phy_info) { struct oce_mbx mbx; struct mbx_common_phy_info *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_common_phy_info *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_GET_PHY_CONFIG, MBX_TIMEOUT_SEC, sizeof(struct mbx_common_phy_info), OCE_MBX_VER_V0); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_common_phy_info); /* now post the command */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } phy_info->phy_type = HOST_16(fwcmd->params.rsp.phy_info.phy_type); phy_info->interface_type = HOST_16(fwcmd->params.rsp.phy_info.interface_type); phy_info->auto_speeds_supported = HOST_16(fwcmd->params.rsp.phy_info.auto_speeds_supported); phy_info->fixed_speeds_supported = HOST_16(fwcmd->params.rsp.phy_info.fixed_speeds_supported); phy_info->misc_params = HOST_32(fwcmd->params.rsp.phy_info.misc_params); error: return rc; } int oce_mbox_lancer_write_flashrom(POCE_SOFTC sc, uint32_t data_size, uint32_t data_offset, POCE_DMA_MEM pdma_mem, uint32_t *written_data, uint32_t *additional_status) { struct oce_mbx mbx; struct mbx_lancer_common_write_object *fwcmd = NULL; int rc = 0, payload_len = 0; bzero(&mbx, sizeof(struct oce_mbx)); payload_len = sizeof(struct mbx_lancer_common_write_object); mbx.u0.s.embedded = 1;/* Embedded */ mbx.payload_length = payload_len; fwcmd = (struct mbx_lancer_common_write_object *)&mbx.payload; /* initialize the ioctl header */ mbx_common_req_hdr_init(&fwcmd->params.req.hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_WRITE_OBJECT, LONG_TIMEOUT, payload_len, OCE_MBX_VER_V0); fwcmd->params.req.write_length = data_size; if (data_size == 0) fwcmd->params.req.eof = 1; else fwcmd->params.req.eof = 0; strcpy(fwcmd->params.req.object_name, "/prg"); fwcmd->params.req.descriptor_count = 1; fwcmd->params.req.write_offset = data_offset; fwcmd->params.req.buffer_length = data_size; fwcmd->params.req.address_lower = pdma_mem->paddr & 0xFFFFFFFF; fwcmd->params.req.address_upper = upper_32_bits(pdma_mem->paddr); /* post the command */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->params.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->params.rsp.additional_status); goto error; } *written_data = HOST_32(fwcmd->params.rsp.actual_write_length); *additional_status = fwcmd->params.rsp.additional_status; error: return rc; } int oce_mbox_create_rq(struct oce_rq *rq) { struct oce_mbx mbx; struct mbx_create_nic_rq *fwcmd; POCE_SOFTC sc = rq->parent; int rc, num_pages = 0; if (rq->qstate == QCREATED) return 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_create_nic_rq *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, NIC_CREATE_RQ, MBX_TIMEOUT_SEC, sizeof(struct mbx_create_nic_rq), OCE_MBX_VER_V0); /* oce_page_list will also prepare pages */ num_pages = oce_page_list(rq->ring, &fwcmd->params.req.pages[0]); if (IS_XE201(sc)) { fwcmd->params.req.frag_size = rq->cfg.frag_size/2048; fwcmd->params.req.page_size = 1; fwcmd->hdr.u0.req.version = OCE_MBX_VER_V1; } else fwcmd->params.req.frag_size = OCE_LOG2(rq->cfg.frag_size); fwcmd->params.req.num_pages = num_pages; fwcmd->params.req.cq_id = rq->cq->cq_id; fwcmd->params.req.if_id = sc->if_id; fwcmd->params.req.max_frame_size = rq->cfg.mtu; fwcmd->params.req.is_rss_queue = rq->cfg.is_rss_queue; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_nic_rq); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } rq->rq_id = HOST_16(fwcmd->params.rsp.rq_id); rq->rss_cpuid = fwcmd->params.rsp.rss_cpuid; error: return rc; } int oce_mbox_create_wq(struct oce_wq *wq) { struct oce_mbx mbx; struct mbx_create_nic_wq *fwcmd; POCE_SOFTC sc = wq->parent; int rc = 0, version, num_pages; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_create_nic_wq *)&mbx.payload; if (IS_XE201(sc)) version = OCE_MBX_VER_V1; else if(IS_BE(sc)) IS_PROFILE_SUPER_NIC(sc) ? (version = OCE_MBX_VER_V2) : (version = OCE_MBX_VER_V0); else version = OCE_MBX_VER_V2; if (version > OCE_MBX_VER_V0) fwcmd->params.req.if_id = sc->if_id; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, NIC_CREATE_WQ, MBX_TIMEOUT_SEC, sizeof(struct mbx_create_nic_wq), version); num_pages = oce_page_list(wq->ring, &fwcmd->params.req.pages[0]); fwcmd->params.req.nic_wq_type = wq->cfg.wq_type; fwcmd->params.req.num_pages = num_pages; fwcmd->params.req.wq_size = OCE_LOG2(wq->cfg.q_len) + 1; fwcmd->params.req.cq_id = wq->cq->cq_id; fwcmd->params.req.ulp_num = 1; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_nic_wq); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } wq->wq_id = HOST_16(fwcmd->params.rsp.wq_id); if (version == OCE_MBX_VER_V2) wq->db_offset = HOST_32(fwcmd->params.rsp.db_offset); else wq->db_offset = PD_TXULP_DB; error: return rc; } int oce_mbox_create_eq(struct oce_eq *eq) { struct oce_mbx mbx; struct mbx_create_common_eq *fwcmd; POCE_SOFTC sc = eq->parent; int rc = 0; uint32_t num_pages; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_create_common_eq *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_CREATE_EQ, MBX_TIMEOUT_SEC, sizeof(struct mbx_create_common_eq), OCE_MBX_VER_V0); num_pages = oce_page_list(eq->ring, &fwcmd->params.req.pages[0]); fwcmd->params.req.ctx.num_pages = num_pages; fwcmd->params.req.ctx.valid = 1; fwcmd->params.req.ctx.size = (eq->eq_cfg.item_size == 4) ? 0 : 1; fwcmd->params.req.ctx.count = OCE_LOG2(eq->eq_cfg.q_len / 256); fwcmd->params.req.ctx.armed = 0; fwcmd->params.req.ctx.delay_mult = eq->eq_cfg.cur_eqd; mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_common_eq); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } eq->eq_id = HOST_16(fwcmd->params.rsp.eq_id); error: return rc; } int oce_mbox_cq_create(struct oce_cq *cq, uint32_t ncoalesce, uint32_t is_eventable) { struct oce_mbx mbx; struct mbx_create_common_cq *fwcmd; POCE_SOFTC sc = cq->parent; uint8_t version; oce_cq_ctx_t *ctx; uint32_t num_pages, page_size; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_create_common_cq *)&mbx.payload; if (IS_XE201(sc)) version = OCE_MBX_VER_V2; else version = OCE_MBX_VER_V0; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_CREATE_CQ, MBX_TIMEOUT_SEC, sizeof(struct mbx_create_common_cq), version); ctx = &fwcmd->params.req.cq_ctx; num_pages = oce_page_list(cq->ring, &fwcmd->params.req.pages[0]); page_size = 1; /* 1 for 4K */ if (version == OCE_MBX_VER_V2) { ctx->v2.num_pages = LE_16(num_pages); ctx->v2.page_size = page_size; ctx->v2.eventable = is_eventable; ctx->v2.valid = 1; ctx->v2.count = OCE_LOG2(cq->cq_cfg.q_len / 256); ctx->v2.nodelay = cq->cq_cfg.nodelay; ctx->v2.coalesce_wm = ncoalesce; ctx->v2.armed = 0; ctx->v2.eq_id = cq->eq->eq_id; if (ctx->v2.count == 3) { if ((u_int)cq->cq_cfg.q_len > (4*1024)-1) ctx->v2.cqe_count = (4*1024)-1; else ctx->v2.cqe_count = cq->cq_cfg.q_len; } } else { ctx->v0.num_pages = LE_16(num_pages); ctx->v0.eventable = is_eventable; ctx->v0.valid = 1; ctx->v0.count = OCE_LOG2(cq->cq_cfg.q_len / 256); ctx->v0.nodelay = cq->cq_cfg.nodelay; ctx->v0.coalesce_wm = ncoalesce; ctx->v0.armed = 0; ctx->v0.eq_id = cq->eq->eq_id; } mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_common_cq); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } cq->cq_id = HOST_16(fwcmd->params.rsp.cq_id); error: return rc; } int oce_mbox_read_transrecv_data(POCE_SOFTC sc, uint32_t page_num) { int rc = 0; struct oce_mbx mbx; struct mbx_read_common_transrecv_data *fwcmd; struct oce_mq_sge *sgl; OCE_DMA_MEM dma; /* Allocate DMA mem*/ if (oce_dma_alloc(sc, sizeof(struct mbx_read_common_transrecv_data), &dma, 0)) return ENOMEM; fwcmd = OCE_DMAPTR(&dma, struct mbx_read_common_transrecv_data); bzero(fwcmd, sizeof(struct mbx_read_common_transrecv_data)); bzero(&mbx, sizeof(struct oce_mbx)); mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_READ_TRANSRECEIVER_DATA, MBX_TIMEOUT_SEC, sizeof(struct mbx_read_common_transrecv_data), OCE_MBX_VER_V0); /* fill rest of mbx */ mbx.u0.s.embedded = 0; mbx.payload_length = sizeof(struct mbx_read_common_transrecv_data); mbx.u0.s.sge_count = 1; sgl = &mbx.payload.u0.u1.sgl[0]; sgl->pa_hi = htole32(upper_32_bits(dma.paddr)); sgl->pa_lo = htole32((dma.paddr) & 0xFFFFFFFF); sgl->length = htole32(mbx.payload_length); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); fwcmd->params.req.port = LE_32(sc->port_id); fwcmd->params.req.page_num = LE_32(page_num); /* command post */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } if(fwcmd->params.rsp.page_num == PAGE_NUM_A0) { bcopy((char *)fwcmd->params.rsp.page_data, &sfp_vpd_dump_buffer[0], TRANSCEIVER_A0_SIZE); } if(fwcmd->params.rsp.page_num == PAGE_NUM_A2) { bcopy((char *)fwcmd->params.rsp.page_data, &sfp_vpd_dump_buffer[TRANSCEIVER_A0_SIZE], TRANSCEIVER_A2_SIZE); } error: oce_dma_free(sc, &dma); return rc; } void oce_mbox_eqd_modify_periodic(POCE_SOFTC sc, struct oce_set_eqd *set_eqd, int num) { struct oce_mbx mbx; struct mbx_modify_common_eq_delay *fwcmd; int rc = 0; int i = 0; bzero(&mbx, sizeof(struct oce_mbx)); /* Initialize MODIFY_EQ_DELAY ioctl header */ fwcmd = (struct mbx_modify_common_eq_delay *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_MODIFY_EQ_DELAY, MBX_TIMEOUT_SEC, sizeof(struct mbx_modify_common_eq_delay), OCE_MBX_VER_V0); /* fill rest of mbx */ mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_modify_common_eq_delay); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); fwcmd->params.req.num_eq = num; for (i = 0; i < num; i++) { fwcmd->params.req.delay[i].eq_id = htole32(set_eqd[i].eq_id); fwcmd->params.req.delay[i].phase = 0; fwcmd->params.req.delay[i].dm = htole32(set_eqd[i].delay_multiplier); } /* command post */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); } int oce_get_profile_config(POCE_SOFTC sc, uint32_t max_rss) { struct oce_mbx mbx; struct mbx_common_get_profile_config *fwcmd; int rc = 0; int version = 0; struct oce_mq_sge *sgl; OCE_DMA_MEM dma; uint32_t desc_count = 0; struct oce_nic_resc_desc *nic_desc = NULL; int i; boolean_t nic_desc_valid = FALSE; if (IS_BE2(sc)) return -1; /* Allocate DMA mem*/ if (oce_dma_alloc(sc, sizeof(struct mbx_common_get_profile_config), &dma, 0)) return ENOMEM; /* Initialize MODIFY_EQ_DELAY ioctl header */ fwcmd = OCE_DMAPTR(&dma, struct mbx_common_get_profile_config); bzero(fwcmd, sizeof(struct mbx_common_get_profile_config)); if (!IS_XE201(sc)) version = OCE_MBX_VER_V1; else version = OCE_MBX_VER_V0; bzero(&mbx, sizeof(struct oce_mbx)); mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_GET_PROFILE_CONFIG, MBX_TIMEOUT_SEC, sizeof(struct mbx_common_get_profile_config), version); /* fill rest of mbx */ mbx.u0.s.embedded = 0; mbx.payload_length = sizeof(struct mbx_common_get_profile_config); mbx.u0.s.sge_count = 1; sgl = &mbx.payload.u0.u1.sgl[0]; sgl->pa_hi = htole32(upper_32_bits(dma.paddr)); sgl->pa_lo = htole32((dma.paddr) & 0xFFFFFFFF); sgl->length = htole32(mbx.payload_length); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); fwcmd->params.req.type = ACTIVE_PROFILE; /* command post */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } nic_desc = (struct oce_nic_resc_desc *) fwcmd->params.rsp.resources; desc_count = HOST_32(fwcmd->params.rsp.desc_count); for (i = 0; i < desc_count; i++) { if ((nic_desc->desc_type == NIC_RESC_DESC_TYPE_V0) || (nic_desc->desc_type == NIC_RESC_DESC_TYPE_V1)) { nic_desc_valid = TRUE; break; } nic_desc = (struct oce_nic_resc_desc *) \ ((char *)nic_desc + nic_desc->desc_len); } if (!nic_desc_valid) { rc = -1; goto error; } else { sc->max_vlans = HOST_16(nic_desc->vlan_count); sc->nwqs = HOST_16(nic_desc->txq_count); if (sc->nwqs) sc->nwqs = MIN(sc->nwqs, OCE_MAX_WQ); else sc->nwqs = OCE_MAX_WQ; sc->nrssqs = HOST_16(nic_desc->rssq_count); if (sc->nrssqs) sc->nrssqs = MIN(sc->nrssqs, max_rss); else sc->nrssqs = max_rss; sc->nrqs = sc->nrssqs + 1; /* 1 for def RX */ } error: oce_dma_free(sc, &dma); return rc; } int oce_get_func_config(POCE_SOFTC sc) { struct oce_mbx mbx; struct mbx_common_get_func_config *fwcmd; int rc = 0; int version = 0; struct oce_mq_sge *sgl; OCE_DMA_MEM dma; uint32_t desc_count = 0; struct oce_nic_resc_desc *nic_desc = NULL; int i; boolean_t nic_desc_valid = FALSE; uint32_t max_rss = 0; if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) max_rss = OCE_LEGACY_MODE_RSS; else max_rss = OCE_MAX_RSS; /* Allocate DMA mem*/ if (oce_dma_alloc(sc, sizeof(struct mbx_common_get_func_config), &dma, 0)) return ENOMEM; /* Initialize MODIFY_EQ_DELAY ioctl header */ fwcmd = OCE_DMAPTR(&dma, struct mbx_common_get_func_config); bzero(fwcmd, sizeof(struct mbx_common_get_func_config)); if (IS_SH(sc)) version = OCE_MBX_VER_V1; else version = OCE_MBX_VER_V0; bzero(&mbx, sizeof(struct oce_mbx)); mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_GET_FUNCTION_CONFIG, MBX_TIMEOUT_SEC, sizeof(struct mbx_common_get_func_config), version); /* fill rest of mbx */ mbx.u0.s.embedded = 0; mbx.payload_length = sizeof(struct mbx_common_get_func_config); mbx.u0.s.sge_count = 1; sgl = &mbx.payload.u0.u1.sgl[0]; sgl->pa_hi = htole32(upper_32_bits(dma.paddr)); sgl->pa_lo = htole32((dma.paddr) & 0xFFFFFFFF); sgl->length = htole32(mbx.payload_length); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); /* command post */ rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } nic_desc = (struct oce_nic_resc_desc *) fwcmd->params.rsp.resources; desc_count = HOST_32(fwcmd->params.rsp.desc_count); for (i = 0; i < desc_count; i++) { if ((nic_desc->desc_type == NIC_RESC_DESC_TYPE_V0) || (nic_desc->desc_type == NIC_RESC_DESC_TYPE_V1)) { nic_desc_valid = TRUE; break; } nic_desc = (struct oce_nic_resc_desc *) \ ((char *)nic_desc + nic_desc->desc_len); } if (!nic_desc_valid) { rc = -1; goto error; } else { sc->max_vlans = nic_desc->vlan_count; sc->nwqs = HOST_32(nic_desc->txq_count); if (sc->nwqs) sc->nwqs = MIN(sc->nwqs, OCE_MAX_WQ); else sc->nwqs = OCE_MAX_WQ; sc->nrssqs = HOST_32(nic_desc->rssq_count); if (sc->nrssqs) sc->nrssqs = MIN(sc->nrssqs, max_rss); else sc->nrssqs = max_rss; sc->nrqs = sc->nrssqs + 1; /* 1 for def RX */ } error: oce_dma_free(sc, &dma); return rc; } /* hw lro functions */ int oce_mbox_nic_query_lro_capabilities(POCE_SOFTC sc, uint32_t *lro_rq_cnt, uint32_t *lro_flags) { struct oce_mbx mbx; struct mbx_nic_query_lro_capabilities *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_nic_query_lro_capabilities *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, 0x20,MBX_TIMEOUT_SEC, sizeof(struct mbx_nic_query_lro_capabilities), OCE_MBX_VER_V0); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_nic_query_lro_capabilities); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } if(lro_flags) *lro_flags = HOST_32(fwcmd->params.rsp.lro_flags); if(lro_rq_cnt) *lro_rq_cnt = HOST_16(fwcmd->params.rsp.lro_rq_cnt); return rc; } int oce_mbox_nic_set_iface_lro_config(POCE_SOFTC sc, int enable) { struct oce_mbx mbx; struct mbx_nic_set_iface_lro_config *fwcmd; int rc = 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_nic_set_iface_lro_config *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, 0x26,MBX_TIMEOUT_SEC, sizeof(struct mbx_nic_set_iface_lro_config), OCE_MBX_VER_V0); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_nic_set_iface_lro_config); fwcmd->params.req.iface_id = sc->if_id; fwcmd->params.req.lro_flags = 0; if(enable) { fwcmd->params.req.lro_flags = LRO_FLAGS_HASH_MODE | LRO_FLAGS_RSS_MODE; fwcmd->params.req.lro_flags |= LRO_FLAGS_CLSC_IPV4 | LRO_FLAGS_CLSC_IPV6; fwcmd->params.req.max_clsc_byte_cnt = 64*1024; /* min = 2974, max = 0xfa59 */ fwcmd->params.req.max_clsc_seg_cnt = 43; /* min = 2, max = 64 */ fwcmd->params.req.max_clsc_usec_delay = 18; /* min = 1, max = 256 */ fwcmd->params.req.min_clsc_frame_byte_cnt = 0; /* min = 1, max = 9014 */ } rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); return rc; } return rc; } int oce_mbox_create_rq_v2(struct oce_rq *rq) { struct oce_mbx mbx; struct mbx_create_nic_rq_v2 *fwcmd; POCE_SOFTC sc = rq->parent; int rc = 0, num_pages = 0; if (rq->qstate == QCREATED) return 0; bzero(&mbx, sizeof(struct oce_mbx)); fwcmd = (struct mbx_create_nic_rq_v2 *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, 0x08, MBX_TIMEOUT_SEC, sizeof(struct mbx_create_nic_rq_v2), OCE_MBX_VER_V2); /* oce_page_list will also prepare pages */ num_pages = oce_page_list(rq->ring, &fwcmd->params.req.pages[0]); fwcmd->params.req.cq_id = rq->cq->cq_id; fwcmd->params.req.frag_size = rq->cfg.frag_size/2048; fwcmd->params.req.num_pages = num_pages; fwcmd->params.req.if_id = sc->if_id; fwcmd->params.req.max_frame_size = rq->cfg.mtu; fwcmd->params.req.page_size = 1; if(rq->cfg.is_rss_queue) { fwcmd->params.req.rq_flags = (NIC_RQ_FLAGS_RSS | NIC_RQ_FLAGS_LRO); }else { device_printf(sc->dev, "non rss lro queue should not be created \n"); goto error; } mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_nic_rq_v2); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev, "%s failed - cmd status: %d addi status: %d\n", __FUNCTION__, rc, fwcmd->hdr.u0.rsp.additional_status); goto error; } rq->rq_id = HOST_16(fwcmd->params.rsp.rq_id); rq->rss_cpuid = fwcmd->params.rsp.rss_cpuid; error: return rc; } diff --git a/sys/dev/qlnx/qlnxe/ecore.h b/sys/dev/qlnx/qlnxe/ecore.h index 33af8f124658..32a6b98156c5 100644 --- a/sys/dev/qlnx/qlnxe/ecore.h +++ b/sys/dev/qlnx/qlnxe/ecore.h @@ -1,1075 +1,1075 @@ /* * Copyright (c) 2017-2018 Cavium, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __ECORE_H #define __ECORE_H #include "ecore_status.h" #include "ecore_hsi_common.h" #include "ecore_hsi_debug_tools.h" #include "ecore_hsi_init_func.h" #include "ecore_hsi_init_tool.h" #include "ecore_proto_if.h" #include "mcp_public.h" #define ECORE_MAJOR_VERSION 8 #define ECORE_MINOR_VERSION 33 #define ECORE_REVISION_VERSION 5 #define ECORE_ENGINEERING_VERSION 0 #define ECORE_VERSION \ ((ECORE_MAJOR_VERSION << 24) | (ECORE_MINOR_VERSION << 16) | \ (ECORE_REVISION_VERSION << 8) | ECORE_ENGINEERING_VERSION) #define STORM_FW_VERSION \ ((FW_MAJOR_VERSION << 24) | (FW_MINOR_VERSION << 16) | \ (FW_REVISION_VERSION << 8) | FW_ENGINEERING_VERSION) #define MAX_HWFNS_PER_DEVICE 2 #define NAME_SIZE 16 #define ARRAY_DECL static const #define ECORE_WFQ_UNIT 100 /* Constants */ #define ECORE_WID_SIZE (1024) #define ECORE_MIN_WIDS (4) /* Configurable */ #define ECORE_PF_DEMS_SIZE (4) /* cau states */ enum ecore_coalescing_mode { ECORE_COAL_MODE_DISABLE, ECORE_COAL_MODE_ENABLE }; enum ecore_nvm_cmd { ECORE_PUT_FILE_BEGIN = DRV_MSG_CODE_NVM_PUT_FILE_BEGIN, ECORE_PUT_FILE_DATA = DRV_MSG_CODE_NVM_PUT_FILE_DATA, ECORE_NVM_READ_NVRAM = DRV_MSG_CODE_NVM_READ_NVRAM, ECORE_NVM_WRITE_NVRAM = DRV_MSG_CODE_NVM_WRITE_NVRAM, ECORE_NVM_DEL_FILE = DRV_MSG_CODE_NVM_DEL_FILE, ECORE_EXT_PHY_FW_UPGRADE = DRV_MSG_CODE_EXT_PHY_FW_UPGRADE, ECORE_NVM_SET_SECURE_MODE = DRV_MSG_CODE_SET_SECURE_MODE, ECORE_PHY_RAW_READ = DRV_MSG_CODE_PHY_RAW_READ, ECORE_PHY_RAW_WRITE = DRV_MSG_CODE_PHY_RAW_WRITE, ECORE_PHY_CORE_READ = DRV_MSG_CODE_PHY_CORE_READ, ECORE_PHY_CORE_WRITE = DRV_MSG_CODE_PHY_CORE_WRITE, ECORE_ENCRYPT_PASSWORD = DRV_MSG_CODE_ENCRYPT_PASSWORD, ECORE_GET_MCP_NVM_RESP = 0xFFFFFF00 }; #ifndef LINUX_REMOVE #if !defined(CONFIG_ECORE_L2) && !defined(CONFIG_ECORE_ROCE) && \ !defined(CONFIG_ECORE_FCOE) && !defined(CONFIG_ECORE_ISCSI) && \ !defined(CONFIG_ECORE_IWARP) #define CONFIG_ECORE_L2 #define CONFIG_ECORE_SRIOV #define CONFIG_ECORE_ROCE #define CONFIG_ECORE_IWARP #define CONFIG_ECORE_FCOE #define CONFIG_ECORE_ISCSI #define CONFIG_ECORE_LL2 #endif #endif /* helpers */ #ifndef __EXTRACT__LINUX__IF__ #define MASK_FIELD(_name, _value) \ ((_value) &= (_name##_MASK)) #define FIELD_VALUE(_name, _value) \ ((_value & _name##_MASK) << _name##_SHIFT) #define SET_FIELD(value, name, flag) \ do { \ (value) &= ~(name##_MASK << name##_SHIFT); \ (value) |= ((((u64)flag) & (u64)name##_MASK) << (name##_SHIFT));\ } while (0) #define GET_FIELD(value, name) \ (((value) >> (name##_SHIFT)) & name##_MASK) #define GET_MFW_FIELD(name, field) \ (((name) & (field ## _MASK)) >> (field ## _OFFSET)) #define SET_MFW_FIELD(name, field, value) \ do { \ (name) &= ~(field ## _MASK); \ (name) |= (((value) << (field ## _OFFSET)) & (field ## _MASK)); \ } while (0) #endif static OSAL_INLINE u32 DB_ADDR(u32 cid, u32 DEMS) { u32 db_addr = FIELD_VALUE(DB_LEGACY_ADDR_DEMS, DEMS) | (cid * ECORE_PF_DEMS_SIZE); return db_addr; } static OSAL_INLINE u32 DB_ADDR_VF(u32 cid, u32 DEMS) { u32 db_addr = FIELD_VALUE(DB_LEGACY_ADDR_DEMS, DEMS) | FIELD_VALUE(DB_LEGACY_ADDR_ICID, cid); return db_addr; } #define ALIGNED_TYPE_SIZE(type_name, p_hwfn) \ ((sizeof(type_name) + (u32)(1<<(p_hwfn->p_dev->cache_shift))-1) & \ ~((1<<(p_hwfn->p_dev->cache_shift))-1)) #ifndef LINUX_REMOVE #ifndef U64_HI #define U64_HI(val) ((u32)(((u64)(val)) >> 32)) #endif #ifndef U64_LO #define U64_LO(val) ((u32)(((u64)(val)) & 0xffffffff)) #endif #endif #ifndef __EXTRACT__LINUX__IF__ #ifndef UEFI /* Debug print definitions */ #define DP_ERR(p_dev, fmt, ...) \ do { \ PRINT_ERR((p_dev)->dp_ctx, "[%s:%d(%s)]" fmt, \ __func__, __LINE__, \ (p_dev)->name ? (p_dev)->name : "", \ ##__VA_ARGS__); \ } while (0) #define DP_NOTICE(p_dev, is_assert, fmt, ...) \ do { \ if (OSAL_UNLIKELY((p_dev)->dp_level <= ECORE_LEVEL_NOTICE)) { \ PRINT((p_dev)->dp_ctx, "[%s:%d(%s)]" fmt, \ __func__, __LINE__, \ (p_dev)->name ? (p_dev)->name : "", \ ##__VA_ARGS__); \ OSAL_ASSERT(!is_assert); \ } \ } while (0) #define DP_INFO(p_dev, fmt, ...) \ do { \ if (OSAL_UNLIKELY((p_dev)->dp_level <= ECORE_LEVEL_INFO)) { \ PRINT((p_dev)->dp_ctx, "[%s:%d(%s)]" fmt, \ __func__, __LINE__, \ (p_dev)->name ? (p_dev)->name : "", \ ##__VA_ARGS__); \ } \ } while (0) #define DP_VERBOSE(p_dev, module, fmt, ...) \ do { \ if (OSAL_UNLIKELY(((p_dev)->dp_level <= ECORE_LEVEL_VERBOSE) && \ ((p_dev)->dp_module & module))) { \ PRINT((p_dev)->dp_ctx, "[%s:%d(%s)]" fmt, \ __func__, __LINE__, \ (p_dev)->name ? (p_dev)->name : "", \ ##__VA_ARGS__); \ } \ } while (0) #endif enum DP_LEVEL { ECORE_LEVEL_VERBOSE = 0x0, ECORE_LEVEL_INFO = 0x1, ECORE_LEVEL_NOTICE = 0x2, ECORE_LEVEL_ERR = 0x3, }; #define ECORE_LOG_LEVEL_SHIFT (30) #define ECORE_LOG_VERBOSE_MASK (0x3fffffff) #define ECORE_LOG_INFO_MASK (0x40000000) #define ECORE_LOG_NOTICE_MASK (0x80000000) enum DP_MODULE { #ifndef LINUX_REMOVE ECORE_MSG_DRV = 0x0001, ECORE_MSG_PROBE = 0x0002, ECORE_MSG_LINK = 0x0004, ECORE_MSG_TIMER = 0x0008, ECORE_MSG_IFDOWN = 0x0010, ECORE_MSG_IFUP = 0x0020, ECORE_MSG_RX_ERR = 0x0040, ECORE_MSG_TX_ERR = 0x0080, ECORE_MSG_TX_QUEUED = 0x0100, ECORE_MSG_INTR = 0x0200, ECORE_MSG_TX_DONE = 0x0400, ECORE_MSG_RX_STATUS = 0x0800, ECORE_MSG_PKTDATA = 0x1000, ECORE_MSG_HW = 0x2000, ECORE_MSG_WOL = 0x4000, #endif ECORE_MSG_SPQ = 0x10000, ECORE_MSG_STATS = 0x20000, ECORE_MSG_DCB = 0x40000, ECORE_MSG_IOV = 0x80000, ECORE_MSG_SP = 0x100000, ECORE_MSG_STORAGE = 0x200000, ECORE_MSG_OOO = 0x200000, ECORE_MSG_CXT = 0x800000, ECORE_MSG_LL2 = 0x1000000, ECORE_MSG_ILT = 0x2000000, ECORE_MSG_RDMA = 0x4000000, ECORE_MSG_DEBUG = 0x8000000, /* to be added...up to 0x8000000 */ }; #endif #define for_each_hwfn(p_dev, i) for (i = 0; i < p_dev->num_hwfns; i++) #define D_TRINE(val, cond1, cond2, true1, true2, def) \ (val == (cond1) ? true1 : \ (val == (cond2) ? true2 : def)) /* forward */ struct ecore_ptt_pool; struct ecore_spq; struct ecore_sb_info; struct ecore_sb_attn_info; struct ecore_cxt_mngr; struct ecore_dma_mem; struct ecore_sb_sp_info; struct ecore_ll2_info; struct ecore_l2_info; struct ecore_igu_info; struct ecore_mcp_info; struct ecore_dcbx_info; struct ecore_llh_info; struct ecore_rt_data { u32 *init_val; bool *b_valid; }; enum ecore_tunn_mode { ECORE_MODE_L2GENEVE_TUNN, ECORE_MODE_IPGENEVE_TUNN, ECORE_MODE_L2GRE_TUNN, ECORE_MODE_IPGRE_TUNN, ECORE_MODE_VXLAN_TUNN, }; enum ecore_tunn_clss { ECORE_TUNN_CLSS_MAC_VLAN, ECORE_TUNN_CLSS_MAC_VNI, ECORE_TUNN_CLSS_INNER_MAC_VLAN, ECORE_TUNN_CLSS_INNER_MAC_VNI, ECORE_TUNN_CLSS_MAC_VLAN_DUAL_STAGE, MAX_ECORE_TUNN_CLSS, }; struct ecore_tunn_update_type { bool b_update_mode; bool b_mode_enabled; enum ecore_tunn_clss tun_cls; }; struct ecore_tunn_update_udp_port { bool b_update_port; u16 port; }; struct ecore_tunnel_info { struct ecore_tunn_update_type vxlan; struct ecore_tunn_update_type l2_geneve; struct ecore_tunn_update_type ip_geneve; struct ecore_tunn_update_type l2_gre; struct ecore_tunn_update_type ip_gre; struct ecore_tunn_update_udp_port vxlan_port; struct ecore_tunn_update_udp_port geneve_port; bool b_update_rx_cls; bool b_update_tx_cls; }; /* The PCI personality is not quite synonymous to protocol ID: * 1. All personalities need CORE connections * 2. The Ethernet personality may support also the RoCE/iWARP protocol */ enum ecore_pci_personality { ECORE_PCI_ETH, ECORE_PCI_FCOE, ECORE_PCI_ISCSI, ECORE_PCI_ETH_ROCE, ECORE_PCI_ETH_IWARP, ECORE_PCI_ETH_RDMA, ECORE_PCI_DEFAULT /* default in shmem */ }; /* All VFs are symetric, all counters are PF + all VFs */ struct ecore_qm_iids { u32 cids; u32 vf_cids; u32 tids; }; /* The PCI relax ordering is either taken care by management FW or can be * enable/disable by ecore client. */ enum ecore_pci_rlx_odr { ECORE_DEFAULT_RLX_ODR, ECORE_ENABLE_RLX_ODR, ECORE_DISABLE_RLX_ODR }; #define MAX_PF_PER_PORT 8 /* HW / FW resources, output of features supported below, most information * is received from MFW. */ enum ecore_resources { ECORE_L2_QUEUE, ECORE_VPORT, ECORE_RSS_ENG, ECORE_PQ, ECORE_RL, ECORE_MAC, ECORE_VLAN, ECORE_RDMA_CNQ_RAM, ECORE_ILT, ECORE_LL2_QUEUE, ECORE_CMDQS_CQS, ECORE_RDMA_STATS_QUEUE, ECORE_BDQ, /* This is needed only internally for matching against the IGU. * In case of legacy MFW, would be set to `0'. */ ECORE_SB, ECORE_MAX_RESC, }; /* Features that require resources, given as input to the resource management * algorithm, the output are the resources above */ enum ecore_feature { ECORE_PF_L2_QUE, ECORE_PF_TC, ECORE_VF, ECORE_EXTRA_VF_QUE, ECORE_VMQ, ECORE_RDMA_CNQ, ECORE_ISCSI_CQ, ECORE_FCOE_CQ, ECORE_VF_L2_QUE, ECORE_MAX_FEATURES, }; enum ecore_port_mode { ECORE_PORT_MODE_DE_2X40G, ECORE_PORT_MODE_DE_2X50G, ECORE_PORT_MODE_DE_1X100G, ECORE_PORT_MODE_DE_4X10G_F, ECORE_PORT_MODE_DE_4X10G_E, ECORE_PORT_MODE_DE_4X20G, ECORE_PORT_MODE_DE_1X40G, ECORE_PORT_MODE_DE_2X25G, ECORE_PORT_MODE_DE_1X25G, ECORE_PORT_MODE_DE_4X25G, ECORE_PORT_MODE_DE_2X10G, }; enum ecore_dev_cap { ECORE_DEV_CAP_ETH, ECORE_DEV_CAP_FCOE, ECORE_DEV_CAP_ISCSI, ECORE_DEV_CAP_ROCE, ECORE_DEV_CAP_IWARP }; #ifndef __EXTRACT__LINUX__IF__ enum ecore_hw_err_type { ECORE_HW_ERR_FAN_FAIL, ECORE_HW_ERR_MFW_RESP_FAIL, ECORE_HW_ERR_HW_ATTN, ECORE_HW_ERR_DMAE_FAIL, ECORE_HW_ERR_RAMROD_FAIL, ECORE_HW_ERR_FW_ASSERT, }; #endif enum ecore_wol_support { ECORE_WOL_SUPPORT_NONE, ECORE_WOL_SUPPORT_PME, }; enum ecore_db_rec_exec { DB_REC_DRY_RUN, DB_REC_REAL_DEAL, DB_REC_ONCE, }; struct ecore_hw_info { /* PCI personality */ enum ecore_pci_personality personality; #define ECORE_IS_RDMA_PERSONALITY(dev) \ ((dev)->hw_info.personality == ECORE_PCI_ETH_ROCE || \ (dev)->hw_info.personality == ECORE_PCI_ETH_IWARP || \ (dev)->hw_info.personality == ECORE_PCI_ETH_RDMA) #define ECORE_IS_ROCE_PERSONALITY(dev) \ ((dev)->hw_info.personality == ECORE_PCI_ETH_ROCE || \ (dev)->hw_info.personality == ECORE_PCI_ETH_RDMA) #define ECORE_IS_IWARP_PERSONALITY(dev) \ ((dev)->hw_info.personality == ECORE_PCI_ETH_IWARP || \ (dev)->hw_info.personality == ECORE_PCI_ETH_RDMA) #define ECORE_IS_L2_PERSONALITY(dev) \ ((dev)->hw_info.personality == ECORE_PCI_ETH || \ ECORE_IS_RDMA_PERSONALITY(dev)) #define ECORE_IS_FCOE_PERSONALITY(dev) \ ((dev)->hw_info.personality == ECORE_PCI_FCOE) #define ECORE_IS_ISCSI_PERSONALITY(dev) \ ((dev)->hw_info.personality == ECORE_PCI_ISCSI) /* Resource Allocation scheme results */ u32 resc_start[ECORE_MAX_RESC]; u32 resc_num[ECORE_MAX_RESC]; u32 feat_num[ECORE_MAX_FEATURES]; #define RESC_START(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_start[resc]) #define RESC_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_num[resc]) #define RESC_END(_p_hwfn, resc) (RESC_START(_p_hwfn, resc) + \ RESC_NUM(_p_hwfn, resc)) #define FEAT_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.feat_num[resc]) /* Amount of traffic classes HW supports */ u8 num_hw_tc; /* Amount of TCs which should be active according to DCBx or upper layer driver configuration */ u8 num_active_tc; /* The traffic class used by PF for it's offloaded protocol */ u8 offload_tc; u32 concrete_fid; u16 opaque_fid; u16 ovlan; u32 part_num[4]; #ifndef ETH_ALEN #define ETH_ALEN 6 /* @@@ TBD - define somewhere else for Windows */ #endif unsigned char hw_mac_addr[ETH_ALEN]; u16 num_iscsi_conns; u16 num_fcoe_conns; struct ecore_igu_info *p_igu_info; /* Sriov */ u8 max_chains_per_vf; u32 port_mode; u32 hw_mode; unsigned long device_capabilities; #ifndef __EXTRACT__LINUX__THROW__ /* Default DCBX mode */ u8 dcbx_mode; #endif u16 mtu; enum ecore_wol_support b_wol_support; }; /* maximun size of read/write commands (HW limit) */ #define DMAE_MAX_RW_SIZE 0x2000 struct ecore_dmae_info { /* Spinlock for synchronizing access to functions */ osal_spinlock_t lock; bool b_mem_ready; u8 channel; dma_addr_t completion_word_phys_addr; /* The memory location where the DMAE writes the completion * value when an operation is finished on this context. */ u32 *p_completion_word; dma_addr_t intermediate_buffer_phys_addr; /* An intermediate buffer for DMAE operations that use virtual * addresses - data is DMA'd to/from this buffer and then * memcpy'd to/from the virtual address */ u32 *p_intermediate_buffer; dma_addr_t dmae_cmd_phys_addr; struct dmae_cmd *p_dmae_cmd; }; struct ecore_wfq_data { u32 default_min_speed; /* When wfq feature is not configured */ u32 min_speed; /* when feature is configured for any 1 vport */ bool configured; }; struct ecore_qm_info { struct init_qm_pq_params *qm_pq_params; struct init_qm_vport_params *qm_vport_params; struct init_qm_port_params *qm_port_params; u16 start_pq; u8 start_vport; u16 pure_lb_pq; u16 offload_pq; u16 low_latency_pq; u16 pure_ack_pq; u16 ooo_pq; u16 first_vf_pq; u16 first_mcos_pq; u16 first_rl_pq; u16 num_pqs; u16 num_vf_pqs; u8 num_vports; u8 max_phys_tcs_per_port; u8 ooo_tc; bool pf_rl_en; bool pf_wfq_en; bool vport_rl_en; bool vport_wfq_en; u8 pf_wfq; u32 pf_rl; struct ecore_wfq_data *wfq_data; u8 num_pf_rls; }; struct ecore_db_recovery_info { osal_list_t list; osal_spinlock_t lock; u32 db_recovery_counter; }; struct storm_stats { u32 address; u32 len; }; struct ecore_fw_data { #ifdef CONFIG_ECORE_BINARY_FW struct fw_ver_info *fw_ver_info; #endif const u8 *modes_tree_buf; union init_op *init_ops; const u32 *arr_data; u32 init_ops_size; }; enum ecore_mf_mode_bit { /* Supports PF-classification based on tag */ ECORE_MF_OVLAN_CLSS, /* Supports PF-classification based on MAC */ ECORE_MF_LLH_MAC_CLSS, /* Supports PF-classification based on protocol type */ ECORE_MF_LLH_PROTO_CLSS, /* Requires a default PF to be set */ ECORE_MF_NEED_DEF_PF, /* Allow LL2 to multicast/broadcast */ ECORE_MF_LL2_NON_UNICAST, /* Allow Cross-PF [& child VFs] Tx-switching */ ECORE_MF_INTER_PF_SWITCH, /* TODO - if we ever re-utilize any of this logic, we can rename */ ECORE_MF_UFP_SPECIFIC, ECORE_MF_DISABLE_ARFS, /* Use vlan for steering */ ECORE_MF_8021Q_TAGGING, /* Use stag for steering */ ECORE_MF_8021AD_TAGGING, }; enum ecore_ufp_mode { ECORE_UFP_MODE_ETS, ECORE_UFP_MODE_VNIC_BW, ECORE_UFP_MODE_UNKNOWN }; enum ecore_ufp_pri_type { ECORE_UFP_PRI_OS, ECORE_UFP_PRI_VNIC, ECORE_UFP_PRI_UNKNOWN }; struct ecore_ufp_info { enum ecore_ufp_pri_type pri_type; enum ecore_ufp_mode mode; u8 tc; }; enum BAR_ID { BAR_ID_0, /* used for GRC */ BAR_ID_1 /* Used for doorbells */ }; struct ecore_hwfn { struct ecore_dev *p_dev; u8 my_id; /* ID inside the PF */ #define IS_LEAD_HWFN(edev) (!((edev)->my_id)) u8 rel_pf_id; /* Relative to engine*/ u8 abs_pf_id; #define ECORE_PATH_ID(_p_hwfn) \ (ECORE_IS_BB((_p_hwfn)->p_dev) ? ((_p_hwfn)->abs_pf_id & 1) : 0) u8 port_id; bool b_active; u32 dp_module; u8 dp_level; char name[NAME_SIZE]; void *dp_ctx; bool hw_init_done; u8 num_funcs_on_engine; u8 enabled_func_idx; /* BAR access */ void OSAL_IOMEM *regview; void OSAL_IOMEM *doorbells; u64 db_phys_addr; unsigned long db_size; #ifndef LINUX_REMOVE u64 reg_offset; u64 db_offset; #endif /* PTT pool */ struct ecore_ptt_pool *p_ptt_pool; /* HW info */ struct ecore_hw_info hw_info; /* rt_array (for init-tool) */ struct ecore_rt_data rt_data; /* SPQ */ struct ecore_spq *p_spq; /* EQ */ struct ecore_eq *p_eq; /* Consolidate Q*/ struct ecore_consq *p_consq; /* Slow-Path definitions */ osal_dpc_t sp_dpc; bool b_sp_dpc_enabled; struct ecore_ptt *p_main_ptt; struct ecore_ptt *p_dpc_ptt; - /* PTP will be used only by the leading funtion. + /* PTP will be used only by the leading function. * Usage of all PTP-apis should be synchronized as result. */ struct ecore_ptt *p_ptp_ptt; struct ecore_sb_sp_info *p_sp_sb; struct ecore_sb_attn_info *p_sb_attn; /* Protocol related */ bool using_ll2; struct ecore_ll2_info *p_ll2_info; struct ecore_ooo_info *p_ooo_info; struct ecore_iscsi_info *p_iscsi_info; struct ecore_fcoe_info *p_fcoe_info; struct ecore_rdma_info *p_rdma_info; struct ecore_pf_params pf_params; bool b_rdma_enabled_in_prs; u32 rdma_prs_search_reg; struct ecore_cxt_mngr *p_cxt_mngr; /* Flag indicating whether interrupts are enabled or not*/ bool b_int_enabled; bool b_int_requested; /* True if the driver requests for the link */ bool b_drv_link_init; struct ecore_vf_iov *vf_iov_info; struct ecore_pf_iov *pf_iov_info; struct ecore_mcp_info *mcp_info; struct ecore_dcbx_info *p_dcbx_info; struct ecore_ufp_info ufp_info; struct ecore_dmae_info dmae_info; /* QM init */ struct ecore_qm_info qm_info; /* Buffer for unzipping firmware data */ #ifdef CONFIG_ECORE_ZIPPED_FW void *unzip_buf; #endif struct dbg_tools_data dbg_info; /* PWM region specific data */ u16 wid_count; u32 dpi_size; u32 dpi_count; u32 dpi_start_offset; /* this is used to * calculate th * doorbell address */ /* If one of the following is set then EDPM shouldn't be used */ u8 dcbx_no_edpm; u8 db_bar_no_edpm; /* L2-related */ struct ecore_l2_info *p_l2_info; /* Mechanism for recovering from doorbell drop */ struct ecore_db_recovery_info db_recovery_info; }; #ifndef __EXTRACT__LINUX__THROW__ enum ecore_mf_mode { ECORE_MF_DEFAULT, ECORE_MF_OVLAN, ECORE_MF_NPAR, ECORE_MF_UFP, }; #endif #ifndef __EXTRACT__LINUX__IF__ enum ecore_dev_type { ECORE_DEV_TYPE_BB, ECORE_DEV_TYPE_AH, ECORE_DEV_TYPE_E5, }; #endif struct ecore_dev { u32 dp_module; u8 dp_level; char name[NAME_SIZE]; void *dp_ctx; enum ecore_dev_type type; /* Translate type/revision combo into the proper conditions */ #define ECORE_IS_BB(dev) ((dev)->type == ECORE_DEV_TYPE_BB) #define ECORE_IS_BB_A0(dev) (ECORE_IS_BB(dev) && CHIP_REV_IS_A0(dev)) #ifndef ASIC_ONLY #define ECORE_IS_BB_B0(dev) ((ECORE_IS_BB(dev) && CHIP_REV_IS_B0(dev)) || \ (CHIP_REV_IS_TEDIBEAR(dev))) #else #define ECORE_IS_BB_B0(dev) (ECORE_IS_BB(dev) && CHIP_REV_IS_B0(dev)) #endif #define ECORE_IS_AH(dev) ((dev)->type == ECORE_DEV_TYPE_AH) #define ECORE_IS_K2(dev) ECORE_IS_AH(dev) #define ECORE_IS_E4(dev) (ECORE_IS_BB(dev) || ECORE_IS_AH(dev)) #define ECORE_IS_E5(dev) ((dev)->type == ECORE_DEV_TYPE_E5) #define ECORE_E5_MISSING_CODE OSAL_BUILD_BUG_ON(false) u16 vendor_id; u16 device_id; #define ECORE_DEV_ID_MASK 0xff00 #define ECORE_DEV_ID_MASK_BB 0x1600 #define ECORE_DEV_ID_MASK_AH 0x8000 #define ECORE_DEV_ID_MASK_E5 0x8100 u16 chip_num; #define CHIP_NUM_MASK 0xffff #define CHIP_NUM_SHIFT 0 u8 chip_rev; #define CHIP_REV_MASK 0xf #define CHIP_REV_SHIFT 0 #ifndef ASIC_ONLY #define CHIP_REV_IS_TEDIBEAR(_p_dev) ((_p_dev)->chip_rev == 0x5) #define CHIP_REV_IS_EMUL_A0(_p_dev) ((_p_dev)->chip_rev == 0xe) #define CHIP_REV_IS_EMUL_B0(_p_dev) ((_p_dev)->chip_rev == 0xc) #define CHIP_REV_IS_EMUL(_p_dev) \ (CHIP_REV_IS_EMUL_A0(_p_dev) || CHIP_REV_IS_EMUL_B0(_p_dev)) #define CHIP_REV_IS_FPGA_A0(_p_dev) ((_p_dev)->chip_rev == 0xf) #define CHIP_REV_IS_FPGA_B0(_p_dev) ((_p_dev)->chip_rev == 0xd) #define CHIP_REV_IS_FPGA(_p_dev) \ (CHIP_REV_IS_FPGA_A0(_p_dev) || CHIP_REV_IS_FPGA_B0(_p_dev)) #define CHIP_REV_IS_SLOW(_p_dev) \ (CHIP_REV_IS_EMUL(_p_dev) || CHIP_REV_IS_FPGA(_p_dev)) #define CHIP_REV_IS_A0(_p_dev) \ (CHIP_REV_IS_EMUL_A0(_p_dev) || CHIP_REV_IS_FPGA_A0(_p_dev) || \ (!(_p_dev)->chip_rev && !(_p_dev)->chip_metal)) #define CHIP_REV_IS_B0(_p_dev) \ (CHIP_REV_IS_EMUL_B0(_p_dev) || CHIP_REV_IS_FPGA_B0(_p_dev) || \ ((_p_dev)->chip_rev == 1 && !(_p_dev)->chip_metal)) #define CHIP_REV_IS_ASIC(_p_dev) !CHIP_REV_IS_SLOW(_p_dev) #else #define CHIP_REV_IS_A0(_p_dev) \ (!(_p_dev)->chip_rev && !(_p_dev)->chip_metal) #define CHIP_REV_IS_B0(_p_dev) \ ((_p_dev)->chip_rev == 1 && !(_p_dev)->chip_metal) #endif u8 chip_metal; #define CHIP_METAL_MASK 0xff #define CHIP_METAL_SHIFT 0 u8 chip_bond_id; #define CHIP_BOND_ID_MASK 0xff #define CHIP_BOND_ID_SHIFT 0 u8 num_engines; u8 num_ports; u8 num_ports_in_engine; u8 num_funcs_in_port; u8 path_id; unsigned long mf_bits; #ifndef __EXTRACT__LINUX__THROW__ enum ecore_mf_mode mf_mode; #define IS_MF_DEFAULT(_p_hwfn) (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_DEFAULT) #define IS_MF_SI(_p_hwfn) (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_NPAR) #define IS_MF_SD(_p_hwfn) (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_OVLAN) #endif int pcie_width; int pcie_speed; /* Add MF related configuration */ u8 mcp_rev; u8 boot_mode; /* WoL related configurations */ u8 wol_config; u8 wol_mac[ETH_ALEN]; u32 int_mode; enum ecore_coalescing_mode int_coalescing_mode; u16 rx_coalesce_usecs; u16 tx_coalesce_usecs; /* Start Bar offset of first hwfn */ void OSAL_IOMEM *regview; void OSAL_IOMEM *doorbells; u64 db_phys_addr; unsigned long db_size; /* PCI */ u8 cache_shift; /* Init */ const struct iro *iro_arr; #define IRO (p_hwfn->p_dev->iro_arr) /* HW functions */ u8 num_hwfns; struct ecore_hwfn hwfns[MAX_HWFNS_PER_DEVICE]; #define ECORE_LEADING_HWFN(dev) (&dev->hwfns[0]) #define ECORE_IS_CMT(dev) ((dev)->num_hwfns > 1) /* Engine affinity */ u8 l2_affin_hint; u8 fir_affin; u8 iwarp_affin; /* Macro for getting the engine-affinitized hwfn for FCoE/iSCSI/RoCE */ #define ECORE_FIR_AFFIN_HWFN(dev) (&dev->hwfns[dev->fir_affin]) /* Macro for getting the engine-affinitized hwfn for iWARP */ #define ECORE_IWARP_AFFIN_HWFN(dev) (&dev->hwfns[dev->iwarp_affin]) /* Generic macro for getting the engine-affinitized hwfn */ #define ECORE_AFFIN_HWFN(dev) \ (ECORE_IS_IWARP_PERSONALITY(ECORE_LEADING_HWFN(dev)) ? \ ECORE_IWARP_AFFIN_HWFN(dev) : \ ECORE_FIR_AFFIN_HWFN(dev)) /* Macro for getting the index (0/1) of the engine-affinitized hwfn */ #define ECORE_AFFIN_HWFN_IDX(dev) \ (IS_LEAD_HWFN(ECORE_AFFIN_HWFN(dev)) ? 0 : 1) /* SRIOV */ struct ecore_hw_sriov_info *p_iov_info; #define IS_ECORE_SRIOV(p_dev) (!!(p_dev)->p_iov_info) struct ecore_tunnel_info tunnel; bool b_is_vf; bool b_dont_override_vf_msix; u32 drv_type; u32 rdma_max_sge; u32 rdma_max_inline; u32 rdma_max_srq_sge; u8 ilt_page_size; struct ecore_eth_stats *reset_stats; struct ecore_fw_data *fw_data; u32 mcp_nvm_resp; /* Recovery */ bool recov_in_prog; /* Indicates whether should prevent attentions from being reasserted */ bool attn_clr_en; /* Indicates whether allowing the MFW to collect a crash dump */ bool allow_mdump; /* Indicates if the reg_fifo is checked after any register access */ bool chk_reg_fifo; #ifndef ASIC_ONLY bool b_is_emul_full; #endif /* LLH info */ u8 ppfid_bitmap; struct ecore_llh_info *p_llh_info; }; #define NUM_OF_VFS(dev) (ECORE_IS_BB(dev) ? MAX_NUM_VFS_BB \ : MAX_NUM_VFS_K2) #define NUM_OF_L2_QUEUES(dev) (ECORE_IS_BB(dev) ? MAX_NUM_L2_QUEUES_BB \ : MAX_NUM_L2_QUEUES_K2) #define NUM_OF_PORTS(dev) (ECORE_IS_BB(dev) ? MAX_NUM_PORTS_BB \ : MAX_NUM_PORTS_K2) #define NUM_OF_SBS(dev) (ECORE_IS_BB(dev) ? MAX_SB_PER_PATH_BB \ : MAX_SB_PER_PATH_K2) #define NUM_OF_ENG_PFS(dev) (ECORE_IS_BB(dev) ? MAX_NUM_PFS_BB \ : MAX_NUM_PFS_K2) #ifndef LINUX_REMOVE #define CRC8_TABLE_SIZE 256 #endif /** * @brief ecore_concrete_to_sw_fid - get the sw function id from * the concrete value. * * @param concrete_fid * * @return OSAL_INLINE u8 */ static OSAL_INLINE u8 ecore_concrete_to_sw_fid(u32 concrete_fid) { u8 vfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID); u8 pfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID); u8 vf_valid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID); u8 sw_fid; if (vf_valid) sw_fid = vfid + MAX_NUM_PFS; else sw_fid = pfid; return sw_fid; } #define PKT_LB_TC 9 #define MAX_NUM_VOQS_E4 20 int ecore_configure_vport_wfq(struct ecore_dev *p_dev, u16 vp_id, u32 rate); void ecore_configure_vp_wfq_on_link_change(struct ecore_dev *p_dev, struct ecore_ptt *p_ptt, u32 min_pf_rate); int ecore_configure_pf_max_bandwidth(struct ecore_dev *p_dev, u8 max_bw); int ecore_configure_pf_min_bandwidth(struct ecore_dev *p_dev, u8 min_bw); void ecore_clean_wfq_db(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); int ecore_device_num_engines(struct ecore_dev *p_dev); int ecore_device_num_ports(struct ecore_dev *p_dev); void ecore_set_fw_mac_addr(__le16 *fw_msb, __le16 *fw_mid, __le16 *fw_lsb, u8 *mac); /* Flags for indication of required queues */ #define PQ_FLAGS_RLS (1 << 0) #define PQ_FLAGS_MCOS (1 << 1) #define PQ_FLAGS_LB (1 << 2) #define PQ_FLAGS_OOO (1 << 3) #define PQ_FLAGS_ACK (1 << 4) #define PQ_FLAGS_OFLD (1 << 5) #define PQ_FLAGS_VFS (1 << 6) #define PQ_FLAGS_LLT (1 << 7) /* physical queue index for cm context intialization */ u16 ecore_get_cm_pq_idx(struct ecore_hwfn *p_hwfn, u32 pq_flags); u16 ecore_get_cm_pq_idx_mcos(struct ecore_hwfn *p_hwfn, u8 tc); u16 ecore_get_cm_pq_idx_vf(struct ecore_hwfn *p_hwfn, u16 vf); u16 ecore_get_cm_pq_idx_rl(struct ecore_hwfn *p_hwfn, u8 qpid); const char *ecore_hw_get_resc_name(enum ecore_resources res_id); /* doorbell recovery mechanism */ void ecore_db_recovery_dp(struct ecore_hwfn *p_hwfn); void ecore_db_recovery_execute(struct ecore_hwfn *p_hwfn, enum ecore_db_rec_exec db_exec); /* amount of resources used in qm init */ u8 ecore_init_qm_get_num_tcs(struct ecore_hwfn *p_hwfn); u16 ecore_init_qm_get_num_vfs(struct ecore_hwfn *p_hwfn); u16 ecore_init_qm_get_num_pf_rls(struct ecore_hwfn *p_hwfn); u16 ecore_init_qm_get_num_vports(struct ecore_hwfn *p_hwfn); u16 ecore_init_qm_get_num_pqs(struct ecore_hwfn *p_hwfn); #define MFW_PORT(_p_hwfn) ((_p_hwfn)->abs_pf_id % \ ecore_device_num_ports((_p_hwfn)->p_dev)) /* The PFID<->PPFID calculation is based on the relative index of a PF on its * port. In BB there is a bug in the LLH in which the PPFID is actually engine * based, and thus it equals the PFID. */ #define ECORE_PFID_BY_PPFID(_p_hwfn, abs_ppfid) \ (ECORE_IS_BB((_p_hwfn)->p_dev) ? \ (abs_ppfid) : \ (abs_ppfid) * (_p_hwfn)->p_dev->num_ports_in_engine + \ MFW_PORT(_p_hwfn)) #define ECORE_PPFID_BY_PFID(_p_hwfn) \ (ECORE_IS_BB((_p_hwfn)->p_dev) ? \ (_p_hwfn)->rel_pf_id : \ (_p_hwfn)->rel_pf_id / (_p_hwfn)->p_dev->num_ports_in_engine) enum _ecore_status_t ecore_all_ppfids_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 addr, u32 val); /* Utility functions for dumping the content of the NIG LLH filters */ enum _ecore_status_t ecore_llh_dump_ppfid(struct ecore_dev *p_dev, u8 ppfid); enum _ecore_status_t ecore_llh_dump_all(struct ecore_dev *p_dev); #endif /* __ECORE_H */ diff --git a/sys/dev/rtsx/rtsx.c b/sys/dev/rtsx/rtsx.c index cb85685c875c..cae35243d137 100644 --- a/sys/dev/rtsx/rtsx.c +++ b/sys/dev/rtsx/rtsx.c @@ -1,3893 +1,3893 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2006 Uwe Stuehler * Copyright (c) 2012 Stefan Sperling * Copyright (c) 2020 Henri Hennebert * Copyright (c) 2020 Gary Jennejohn * Copyright (c) 2020 Jesper Schmitz Mouridsen * All rights reserved. * * Patch from: * - Lutz Bichler * * Base on OpenBSD /sys/dev/pci/rtsx_pci.c & /dev/ic/rtsx.c * on Linux /drivers/mmc/host/rtsx_pci_sdmmc.c, * /include/linux/rtsx_pci.h & * /drivers/misc/cardreader/rtsx_pcr.c * on NetBSD /sys/dev/ic/rtsx.c * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include /* For FreeBSD 11 */ #include /* For FreeBSD 11 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "opt_mmccam.h" #ifdef MMCCAM #include #include #include #include #include #endif /* MMCCAM */ #include "rtsxreg.h" /* The softc holds our per-instance data. */ struct rtsx_softc { struct mtx rtsx_mtx; /* device mutex */ device_t rtsx_dev; /* device */ uint16_t rtsx_flags; /* device flags */ uint16_t rtsx_device_id; /* device ID */ device_t rtsx_mmc_dev; /* device of mmc bus */ uint32_t rtsx_intr_enabled; /* enabled interrupts */ uint32_t rtsx_intr_status; /* soft interrupt status */ int rtsx_irq_res_id; /* bus IRQ resource id */ struct resource *rtsx_irq_res; /* bus IRQ resource */ void *rtsx_irq_cookie; /* bus IRQ resource cookie */ struct callout rtsx_timeout_callout; /* callout for timeout */ int rtsx_timeout; /* interrupt timeout value */ void (*rtsx_intr_trans_ok)(struct rtsx_softc *sc); /* function to call if transfer succeed */ void (*rtsx_intr_trans_ko)(struct rtsx_softc *sc); /* function to call if transfer fail */ struct timeout_task rtsx_card_insert_task; /* card insert delayed task */ struct task rtsx_card_remove_task; /* card remove task */ int rtsx_res_id; /* bus memory resource id */ struct resource *rtsx_res; /* bus memory resource */ int rtsx_res_type; /* bus memory resource type */ bus_space_tag_t rtsx_btag; /* host register set tag */ bus_space_handle_t rtsx_bhandle; /* host register set handle */ bus_dma_tag_t rtsx_cmd_dma_tag; /* DMA tag for command transfer */ bus_dmamap_t rtsx_cmd_dmamap; /* DMA map for command transfer */ void *rtsx_cmd_dmamem; /* DMA mem for command transfer */ bus_addr_t rtsx_cmd_buffer; /* device visible address of the DMA segment */ int rtsx_cmd_index; /* index in rtsx_cmd_buffer */ bus_dma_tag_t rtsx_data_dma_tag; /* DMA tag for data transfer */ bus_dmamap_t rtsx_data_dmamap; /* DMA map for data transfer */ void *rtsx_data_dmamem; /* DMA mem for data transfer */ bus_addr_t rtsx_data_buffer; /* device visible address of the DMA segment */ #ifdef MMCCAM struct cam_devq *rtsx_devq; /* CAM queue of requests */ struct cam_sim *rtsx_sim; /* descriptor of our SCSI Interface Modules (SIM) */ struct mtx rtsx_sim_mtx; /* SIM mutex */ union ccb *rtsx_ccb; /* CAM control block */ struct mmc_request rtsx_cam_req; /* CAM MMC request */ #endif /* MMCCAM */ struct mmc_request *rtsx_req; /* MMC request */ struct mmc_host rtsx_host; /* host parameters */ int rtsx_pcie_cap; /* PCIe capability offset */ int8_t rtsx_bus_busy; /* bus busy status */ int8_t rtsx_ios_bus_width; /* current host.ios.bus_width */ int32_t rtsx_ios_clock; /* current host.ios.clock */ int8_t rtsx_ios_power_mode; /* current host.ios.power mode */ int8_t rtsx_ios_timing; /* current host.ios.timing */ int8_t rtsx_ios_vccq; /* current host.ios.vccq */ uint8_t rtsx_read_only; /* card read only status */ uint8_t rtsx_inversion; /* inversion of card detection and read only status */ uint8_t rtsx_force_timing; /* force bus_timing_uhs_sdr50 */ uint8_t rtsx_debug; /* print debugging */ #ifdef MMCCAM uint8_t rtsx_cam_status; /* CAM status - 1 if card in use */ #endif /* MMCCAM */ uint64_t rtsx_read_count; /* count of read operations */ uint64_t rtsx_write_count; /* count of write operations */ bool rtsx_discovery_mode; /* are we in discovery mode? */ bool rtsx_tuning_mode; /* are we tuning */ bool rtsx_double_clk; /* double clock freqency */ bool rtsx_vpclk; /* voltage at Pulse-width Modulation(PWM) clock? */ uint8_t rtsx_ssc_depth; /* Spread spectrum clocking depth */ uint8_t rtsx_card_drive_sel; /* value for RTSX_CARD_DRIVE_SEL */ uint8_t rtsx_sd30_drive_sel_3v3;/* value for RTSX_SD30_DRIVE_SEL */ }; /* rtsx_flags values */ #define RTSX_F_DEFAULT 0x0000 #define RTSX_F_CARD_PRESENT 0x0001 #define RTSX_F_SDIO_SUPPORT 0x0002 #define RTSX_F_VERSION_A 0x0004 #define RTSX_F_VERSION_B 0x0008 #define RTSX_F_VERSION_C 0x0010 #define RTSX_F_VERSION_D 0x0020 #define RTSX_F_8411B_QFN48 0x0040 #define RTSX_F_REVERSE_SOCKET 0x0080 #define RTSX_REALTEK 0x10ec #define RTSX_RTS5209 0x5209 #define RTSX_RTS5227 0x5227 #define RTSX_RTS5229 0x5229 #define RTSX_RTS522A 0x522a #define RTSX_RTS525A 0x525a #define RTSX_RTS5249 0x5249 #define RTSX_RTL8402 0x5286 #define RTSX_RTL8411 0x5289 #define RTSX_RTL8411B 0x5287 #define RTSX_VERSION "2.0c" static const struct rtsx_device { uint16_t vendor_id; uint16_t device_id; const char *desc; } rtsx_devices[] = { { RTSX_REALTEK, RTSX_RTS5209, RTSX_VERSION " Realtek RTS5209 PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTS5227, RTSX_VERSION " Realtek RTS5227 PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTS5229, RTSX_VERSION " Realtek RTS5229 PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTS522A, RTSX_VERSION " Realtek RTS522A PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTS525A, RTSX_VERSION " Realtek RTS525A PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTS5249, RTSX_VERSION " Realtek RTS5249 PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTL8402, RTSX_VERSION " Realtek RTL8402 PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTL8411, RTSX_VERSION " Realtek RTL8411 PCI MMC/SD Card Reader"}, { RTSX_REALTEK, RTSX_RTL8411B, RTSX_VERSION " Realtek RTL8411B PCI MMC/SD Card Reader"}, { 0, 0, NULL} }; static int rtsx_dma_alloc(struct rtsx_softc *sc); static void rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error); static void rtsx_dma_free(struct rtsx_softc *sc); static void rtsx_intr(void *arg); static void rtsx_handle_card_present(struct rtsx_softc *sc); static void rtsx_card_task(void *arg, int pending __unused); static bool rtsx_is_card_present(struct rtsx_softc *sc); static int rtsx_init(struct rtsx_softc *sc); static int rtsx_map_sd_drive(int index); static int rtsx_rts5227_fill_driving(struct rtsx_softc *sc); static int rtsx_rts5249_fill_driving(struct rtsx_softc *sc); static int rtsx_read(struct rtsx_softc *, uint16_t, uint8_t *); static int rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val); static int rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val); static int rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val); static int rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val); static int rtsx_bus_power_off(struct rtsx_softc *sc); static int rtsx_bus_power_on(struct rtsx_softc *sc); static int rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width); static int rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing); static int rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq); static int rtsx_stop_sd_clock(struct rtsx_softc *sc); static int rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu); static void rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point); static void rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point); static void rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map); static int rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point); static int rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc); static void rtsx_sd_wait_data_idle(struct rtsx_softc *sc); static uint8_t rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map); static int rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit); #if 0 /* For led */ static int rtsx_led_enable(struct rtsx_softc *sc); static int rtsx_led_disable(struct rtsx_softc *sc); #endif /* For led */ static uint8_t rtsx_response_type(uint16_t mmc_rsp); static void rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg, uint8_t mask, uint8_t data); static void rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt); static void rtsx_send_cmd(struct rtsx_softc *sc); static void rtsx_ret_resp(struct rtsx_softc *sc); static void rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_stop_cmd(struct rtsx_softc *sc); static void rtsx_clear_error(struct rtsx_softc *sc); static void rtsx_req_done(struct rtsx_softc *sc); static int rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd); static int rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_ask_ppbuf_part1(struct rtsx_softc *sc); static void rtsx_get_ppbuf_part1(struct rtsx_softc *sc); static void rtsx_get_ppbuf_part2(struct rtsx_softc *sc); static void rtsx_put_ppbuf_part1(struct rtsx_softc *sc); static void rtsx_put_ppbuf_part2(struct rtsx_softc *sc); static void rtsx_write_ppbuf(struct rtsx_softc *sc); static int rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd); static void rtsx_xfer_begin(struct rtsx_softc *sc); static void rtsx_xfer_start(struct rtsx_softc *sc); static void rtsx_xfer_finish(struct rtsx_softc *sc); static void rtsx_timeout(void *arg); #ifdef MMCCAM static void rtsx_cam_action(struct cam_sim *sim, union ccb *ccb); static void rtsx_cam_poll(struct cam_sim *sim); static void rtsx_cam_set_tran_settings(struct rtsx_softc *sc, union ccb *ccb); static void rtsx_cam_request(struct rtsx_softc *sc, union ccb *ccb); #endif /* MMCCAM */ static int rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result); static int rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value); static int rtsx_mmcbr_update_ios(device_t bus, device_t child __unused); static int rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused); static int rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400 __unused); static int rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused); static int rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req); static int rtsx_mmcbr_get_ro(device_t bus, device_t child __unused); static int rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused); static int rtsx_mmcbr_release_host(device_t bus, device_t child __unused); static int rtsx_probe(device_t dev); static int rtsx_attach(device_t dev); static int rtsx_detach(device_t dev); static int rtsx_shutdown(device_t dev); static int rtsx_suspend(device_t dev); static int rtsx_resume(device_t dev); #define RTSX_LOCK_INIT(_sc) mtx_init(&(_sc)->rtsx_mtx, \ device_get_nameunit(sc->rtsx_dev), "rtsx", MTX_DEF) #define RTSX_LOCK(_sc) mtx_lock(&(_sc)->rtsx_mtx) #define RTSX_UNLOCK(_sc) mtx_unlock(&(_sc)->rtsx_mtx) #define RTSX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rtsx_mtx) #define RTSX_SDCLK_OFF 0 #define RTSX_SDCLK_250KHZ 250000 #define RTSX_SDCLK_400KHZ 400000 #define RTSX_SDCLK_25MHZ 25000000 #define RTSX_SDCLK_50MHZ 50000000 #define RTSX_SDCLK_100MHZ 100000000 #define RTSX_SDCLK_208MHZ 208000000 #define RTSX_MIN_DIV_N 80 #define RTSX_MAX_DIV_N 208 #define RTSX_MAX_DATA_BLKLEN 512 #define RTSX_DMA_ALIGN 4 #define RTSX_HOSTCMD_MAX 256 #define RTSX_DMA_CMD_BIFSIZE (sizeof(uint32_t) * RTSX_HOSTCMD_MAX) #define RTSX_DMA_DATA_BUFSIZE MAXPHYS #define ISSET(t, f) ((t) & (f)) #define READ4(sc, reg) \ (bus_space_read_4((sc)->rtsx_btag, (sc)->rtsx_bhandle, (reg))) #define WRITE4(sc, reg, val) \ (bus_space_write_4((sc)->rtsx_btag, (sc)->rtsx_bhandle, (reg), (val))) #define RTSX_READ(sc, reg, val) \ do { \ int err = rtsx_read((sc), (reg), (val)); \ if (err) \ return (err); \ } while (0) #define RTSX_WRITE(sc, reg, val) \ do { \ int err = rtsx_write((sc), (reg), 0xff, (val)); \ if (err) \ return (err); \ } while (0) #define RTSX_CLR(sc, reg, bits) \ do { \ int err = rtsx_write((sc), (reg), (bits), 0); \ if (err) \ return (err); \ } while (0) #define RTSX_SET(sc, reg, bits) \ do { \ int err = rtsx_write((sc), (reg), (bits), 0xff);\ if (err) \ return (err); \ } while (0) #define RTSX_BITOP(sc, reg, mask, bits) \ do { \ int err = rtsx_write((sc), (reg), (mask), (bits)); \ if (err) \ return (err); \ } while (0) /* * We use two DMA buffers: a command buffer and a data buffer. * * The command buffer contains a command queue for the host controller, * which describes SD/MMC commands to run, and other parameters. The chip * runs the command queue when a special bit in the RTSX_HCBAR register is * set and signals completion with the RTSX_TRANS_OK_INT interrupt. * Each command is encoded as a 4 byte sequence containing command number * (read, write, or check a host controller register), a register address, * and a data bit-mask and value. * SD/MMC commands which do not transfer any data from/to the card only use * the command buffer. * * The data buffer is used for transfer longer than 512. Data transfer is * controlled via the RTSX_HDBAR register and completion is signalled by * the RTSX_TRANS_OK_INT interrupt. * * The chip is unable to perform DMA above 4GB. */ /* * Main commands in the usual seqence used: * * CMD0 Go idle state * CMD8 Send interface condition * CMD55 Application Command for next ACMD * ACMD41 Send Operation Conditions Register (OCR: voltage profile of the card) * CMD2 Send Card Identification (CID) Register * CMD3 Send relative address * CMD9 Send Card Specific Data (CSD) * CMD13 Send status (32 bits - bit 25: card password protected) * CMD7 Select card (before Get card SCR) * ACMD51 Send SCR (SD CARD Configuration Register - [51:48]: Bus widths supported) * CMD6 SD switch function * ACMD13 Send SD status (512 bits) * ACMD42 Set/Clear card detect * ACMD6 Set bus width * CMD19 Send tuning block * CMD12 Stop transmission * * CMD17 Read single block (<=512) * CMD18 Read multiple blocks (>512) * CMD24 Write single block (<=512) * CMD25 Write multiple blocks (>512) * * CMD52 IO R/W direct * CMD5 Send Operation Conditions */ static int rtsx_dma_alloc(struct rtsx_softc *sc) { int error = 0; error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev), /* inherit from parent */ RTSX_DMA_ALIGN, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ RTSX_DMA_CMD_BIFSIZE, 1, /* maxsize, nsegments */ RTSX_DMA_CMD_BIFSIZE, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->rtsx_cmd_dma_tag); if (error) { device_printf(sc->rtsx_dev, "Can't create cmd parent DMA tag\n"); return (error); } error = bus_dmamem_alloc(sc->rtsx_cmd_dma_tag, /* DMA tag */ &sc->rtsx_cmd_dmamem, /* will hold the KVA pointer */ BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, /* flags */ &sc->rtsx_cmd_dmamap); /* DMA map */ if (error) { device_printf(sc->rtsx_dev, "Can't create DMA map for command transfer\n"); goto destroy_cmd_dma_tag; } error = bus_dmamap_load(sc->rtsx_cmd_dma_tag, /* DMA tag */ sc->rtsx_cmd_dmamap, /* DMA map */ sc->rtsx_cmd_dmamem, /* KVA pointer to be mapped */ RTSX_DMA_CMD_BIFSIZE, /* size of buffer */ rtsx_dmamap_cb, /* callback */ &sc->rtsx_cmd_buffer, /* first arg of callback */ 0); /* flags */ if (error || sc->rtsx_cmd_buffer == 0) { device_printf(sc->rtsx_dev, "Can't load DMA memory for command transfer\n"); error = (error) ? error : EFAULT; goto destroy_cmd_dmamem_alloc; } error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev), /* inherit from parent */ RTSX_DMA_DATA_BUFSIZE, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ RTSX_DMA_DATA_BUFSIZE, 1, /* maxsize, nsegments */ RTSX_DMA_DATA_BUFSIZE, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->rtsx_data_dma_tag); if (error) { device_printf(sc->rtsx_dev, "Can't create data parent DMA tag\n"); goto destroy_cmd_dmamap_load; } error = bus_dmamem_alloc(sc->rtsx_data_dma_tag, /* DMA tag */ &sc->rtsx_data_dmamem, /* will hold the KVA pointer */ BUS_DMA_WAITOK | BUS_DMA_ZERO, /* flags */ &sc->rtsx_data_dmamap); /* DMA map */ if (error) { device_printf(sc->rtsx_dev, "Can't create DMA map for data transfer\n"); goto destroy_data_dma_tag; } error = bus_dmamap_load(sc->rtsx_data_dma_tag, /* DMA tag */ sc->rtsx_data_dmamap, /* DMA map */ sc->rtsx_data_dmamem, /* KVA pointer to be mapped */ RTSX_DMA_DATA_BUFSIZE, /* size of buffer */ rtsx_dmamap_cb, /* callback */ &sc->rtsx_data_buffer, /* first arg of callback */ 0); /* flags */ if (error || sc->rtsx_data_buffer == 0) { device_printf(sc->rtsx_dev, "Can't load DMA memory for data transfer\n"); error = (error) ? error : EFAULT; goto destroy_data_dmamem_alloc; } return (error); destroy_data_dmamem_alloc: bus_dmamem_free(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamem, sc->rtsx_data_dmamap); destroy_data_dma_tag: bus_dma_tag_destroy(sc->rtsx_data_dma_tag); destroy_cmd_dmamap_load: bus_dmamap_unload(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap); destroy_cmd_dmamem_alloc: bus_dmamem_free(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamem, sc->rtsx_cmd_dmamap); destroy_cmd_dma_tag: bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag); return (error); } static void rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error) { printf("rtsx_dmamap_cb: error %d\n", error); return; } *(bus_addr_t *)arg = segs[0].ds_addr; } static void rtsx_dma_free(struct rtsx_softc *sc) { if (sc->rtsx_cmd_dma_tag != NULL) { if (sc->rtsx_cmd_dmamap != NULL) bus_dmamap_unload(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap); if (sc->rtsx_cmd_dmamem != NULL) bus_dmamem_free(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamem, sc->rtsx_cmd_dmamap); sc->rtsx_cmd_dmamap = NULL; sc->rtsx_cmd_dmamem = NULL; sc->rtsx_cmd_buffer = 0; bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag); sc->rtsx_cmd_dma_tag = NULL; } if (sc->rtsx_data_dma_tag != NULL) { if (sc->rtsx_data_dmamap != NULL) bus_dmamap_unload(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap); if (sc->rtsx_data_dmamem != NULL) bus_dmamem_free(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamem, sc->rtsx_data_dmamap); sc->rtsx_data_dmamap = NULL; sc->rtsx_data_dmamem = NULL; sc->rtsx_data_buffer = 0; bus_dma_tag_destroy(sc->rtsx_data_dma_tag); sc->rtsx_data_dma_tag = NULL; } } static void rtsx_intr(void *arg) { struct rtsx_softc *sc = arg; uint32_t enabled; uint32_t status; RTSX_LOCK(sc); enabled = sc->rtsx_intr_enabled; status = READ4(sc, RTSX_BIPR); /* read Bus Interrupt Pending Register */ sc->rtsx_intr_status = status; if (bootverbose) device_printf(sc->rtsx_dev, "Interrupt handler - enabled: 0x%08x, status: 0x%08x\n", enabled, status); /* Ack interrupts. */ WRITE4(sc, RTSX_BIPR, status); if (((enabled & status) == 0) || status == 0xffffffff) { device_printf(sc->rtsx_dev, "Spurious interrupt - enabled: 0x%08x, status: 0x%08x\n", enabled, status); RTSX_UNLOCK(sc); return; } /* Detect write protect. */ if (status & RTSX_SD_WRITE_PROTECT) sc->rtsx_read_only = 1; else sc->rtsx_read_only = 0; /* Start task to handle SD card status change (from dwmmc.c). */ if (status & RTSX_SD_INT) { device_printf(sc->rtsx_dev, "Interrupt card inserted/removed\n"); rtsx_handle_card_present(sc); } if (sc->rtsx_req == NULL) { RTSX_UNLOCK(sc); return; } if (status & RTSX_TRANS_OK_INT) { sc->rtsx_req->cmd->error = MMC_ERR_NONE; if (sc->rtsx_intr_trans_ok != NULL) sc->rtsx_intr_trans_ok(sc); } else if (status & RTSX_TRANS_FAIL_INT) { uint8_t stat1; sc->rtsx_req->cmd->error = MMC_ERR_FAILED; if (rtsx_read(sc, RTSX_SD_STAT1, &stat1) == 0 && (stat1 & RTSX_SD_CRC_ERR)) { device_printf(sc->rtsx_dev, "CRC error\n"); sc->rtsx_req->cmd->error = MMC_ERR_BADCRC; } if (!sc->rtsx_tuning_mode) device_printf(sc->rtsx_dev, "Transfer fail - status: 0x%08x\n", status); rtsx_stop_cmd(sc); if (sc->rtsx_intr_trans_ko != NULL) sc->rtsx_intr_trans_ko(sc); } RTSX_UNLOCK(sc); } /* * Function called from the IRQ handler (from dwmmc.c). */ static void rtsx_handle_card_present(struct rtsx_softc *sc) { bool was_present; bool is_present; #ifdef MMCCAM was_present = sc->rtsx_cam_status; #else was_present = sc->rtsx_mmc_dev != NULL; #endif /* MMCCAM */ is_present = rtsx_is_card_present(sc); if (is_present) device_printf(sc->rtsx_dev, "Card present\n"); else device_printf(sc->rtsx_dev, "Card absent\n"); if (!was_present && is_present) { /* * The delay is to debounce the card insert * (sometimes the card detect pin stabilizes * before the other pins have made good contact). */ taskqueue_enqueue_timeout(taskqueue_swi_giant, &sc->rtsx_card_insert_task, -hz); } else if (was_present && !is_present) { taskqueue_enqueue(taskqueue_swi_giant, &sc->rtsx_card_remove_task); } } /* - * This funtion is called at startup. + * This function is called at startup. */ static void rtsx_card_task(void *arg, int pending __unused) { struct rtsx_softc *sc = arg; RTSX_LOCK(sc); if (rtsx_is_card_present(sc)) { sc->rtsx_flags |= RTSX_F_CARD_PRESENT; /* Card is present, attach if necessary. */ #ifdef MMCCAM if (sc->rtsx_cam_status == 0) { union ccb *ccb; uint32_t pathid; #else if (sc->rtsx_mmc_dev == NULL) { #endif /* MMCCAM */ if (bootverbose) device_printf(sc->rtsx_dev, "Card inserted\n"); sc->rtsx_read_count = sc->rtsx_write_count = 0; #ifdef MMCCAM sc->rtsx_cam_status = 1; pathid = cam_sim_path(sc->rtsx_sim); ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { device_printf(sc->rtsx_dev, "Unable to alloc CCB for rescan\n"); RTSX_UNLOCK(sc); return; } /* * We create a rescan request for BUS:0:0, since the card * will be at lun 0. */ if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) { device_printf(sc->rtsx_dev, "Unable to create path for rescan\n"); RTSX_UNLOCK(sc); xpt_free_ccb(ccb); return; } RTSX_UNLOCK(sc); xpt_rescan(ccb); #else sc->rtsx_mmc_dev = device_add_child(sc->rtsx_dev, "mmc", -1); RTSX_UNLOCK(sc); if (sc->rtsx_mmc_dev == NULL) { device_printf(sc->rtsx_dev, "Adding MMC bus failed\n"); } else { device_set_ivars(sc->rtsx_mmc_dev, sc); device_probe_and_attach(sc->rtsx_mmc_dev); } #endif /* MMCCAM */ } else RTSX_UNLOCK(sc); } else { sc->rtsx_flags &= ~RTSX_F_CARD_PRESENT; /* Card isn't present, detach if necessary. */ #ifdef MMCCAM if (sc->rtsx_cam_status != 0) { union ccb *ccb; uint32_t pathid; #else if (sc->rtsx_mmc_dev != NULL) { #endif /* MMCCAM */ if (bootverbose) device_printf(sc->rtsx_dev, "Card removed\n"); if (sc->rtsx_debug) device_printf(sc->rtsx_dev, "Read count: %" PRIu64 ", write count: %" PRIu64 "\n", sc->rtsx_read_count, sc->rtsx_write_count); #ifdef MMCCAM sc->rtsx_cam_status = 0; pathid = cam_sim_path(sc->rtsx_sim); ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { device_printf(sc->rtsx_dev, "Unable to alloc CCB for rescan\n"); RTSX_UNLOCK(sc); return; } /* * We create a rescan request for BUS:0:0, since the card * will be at lun 0. */ if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) { device_printf(sc->rtsx_dev, "Unable to create path for rescan\n"); RTSX_UNLOCK(sc); xpt_free_ccb(ccb); return; } RTSX_UNLOCK(sc); xpt_rescan(ccb); #else RTSX_UNLOCK(sc); if (device_delete_child(sc->rtsx_dev, sc->rtsx_mmc_dev)) device_printf(sc->rtsx_dev, "Detaching MMC bus failed\n"); sc->rtsx_mmc_dev = NULL; #endif /* MMCCAM */ } else RTSX_UNLOCK(sc); } } static bool rtsx_is_card_present(struct rtsx_softc *sc) { uint32_t status; status = READ4(sc, RTSX_BIPR); if (sc->rtsx_inversion == 0) return (status & RTSX_SD_EXIST); else return !(status & RTSX_SD_EXIST); } static int rtsx_init(struct rtsx_softc *sc) { bool rtsx_init_debug = false; uint8_t version; uint8_t val; int error; sc->rtsx_host.host_ocr = RTSX_SUPPORTED_VOLTAGE; sc->rtsx_host.f_min = RTSX_SDCLK_250KHZ; sc->rtsx_host.f_max = RTSX_SDCLK_208MHZ; sc->rtsx_host.caps = MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED | MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25; sc->rtsx_host.caps |= MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104; if (sc->rtsx_device_id == RTSX_RTS5209) sc->rtsx_host.caps |= MMC_CAP_8_BIT_DATA; pci_find_cap(sc->rtsx_dev, PCIY_EXPRESS, &(sc->rtsx_pcie_cap)); /* * Check IC version. */ switch (sc->rtsx_device_id) { case RTSX_RTS5229: /* Read IC version from dummy register. */ RTSX_READ(sc, RTSX_DUMMY_REG, &version); if ((version & 0x0F) == RTSX_IC_VERSION_C) sc->rtsx_flags |= RTSX_F_VERSION_C; break; case RTSX_RTS522A: /* Read IC version from dummy register. */ RTSX_READ(sc, RTSX_DUMMY_REG, &version); if ((version & 0x0F) == RTSX_IC_VERSION_A) sc->rtsx_flags |= RTSX_F_VERSION_A; break; case RTSX_RTS525A: /* Read IC version from dummy register. */ RTSX_READ(sc, RTSX_DUMMY_REG, &version); if ((version & 0x0F) == RTSX_IC_VERSION_A) sc->rtsx_flags |= RTSX_F_VERSION_A; break; case RTSX_RTL8411B: RTSX_READ(sc, RTSX_RTL8411B_PACKAGE, &version); if (version & RTSX_RTL8411B_QFN48) sc->rtsx_flags |= RTSX_F_8411B_QFN48; break; } /* * Fetch vendor settings. */ /* * Normally OEMs will set vendor setting to the config space * of Realtek card reader in BIOS stage. This statement reads * the setting and configure the internal registers according * to it, to improve card reader's compatibility condition. */ sc->rtsx_card_drive_sel = RTSX_CARD_DRIVE_DEFAULT; switch (sc->rtsx_device_id) { uint32_t reg; uint32_t reg1; uint8_t reg3; case RTSX_RTS5209: sc->rtsx_card_drive_sel = RTSX_RTS5209_CARD_DRIVE_DEFAULT; sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); if (!(reg & 0x80)) { sc->rtsx_card_drive_sel = (reg >> 8) & 0x3F; sc->rtsx_sd30_drive_sel_3v3 = reg & 0x07; } else { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || rtsx_init_debug) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS5227: case RTSX_RTS522A: sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (!(reg & 0x1000000)) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03; if (reg & 0x4000) sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET; } else { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || rtsx_init_debug) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3, (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false"); break; case RTSX_RTS5229: sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (!(reg & 0x1000000)) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive((reg >> 5) & 0x03); } else { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || rtsx_init_debug) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS525A: case RTSX_RTS5249: sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if ((reg & 0x1000000)) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4); sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03; if (reg & 0x4000) sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET; } else { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || rtsx_init_debug) device_printf(sc->rtsx_dev, "card_drive_sel = 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3, (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false"); break; case RTSX_RTL8402: case RTSX_RTL8411: sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT; sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg1 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (reg1 & 0x1000000) { sc->rtsx_card_drive_sel &= 0x3F; sc->rtsx_card_drive_sel |= ((reg1 >> 25) & 0x01) << 6; reg3 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG3, 1); sc->rtsx_sd30_drive_sel_3v3 = (reg3 >> 5) & 0x07; } else { device_printf(sc->rtsx_dev, "pci_read_config() error - reg1: 0x%08x\n", reg1); } if (bootverbose || rtsx_init_debug) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTL8411B: sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT; sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D; reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4); if (!(reg & 0x1000000)) { sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive(reg & 0x03); } else { device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg); } if (bootverbose || rtsx_init_debug) device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n", sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3); break; } if (bootverbose || rtsx_init_debug) device_printf(sc->rtsx_dev, "rtsx_init() rtsx_flags: 0x%04x\n", sc->rtsx_flags); /* Enable interrupts. */ sc->rtsx_intr_enabled = RTSX_TRANS_OK_INT_EN | RTSX_TRANS_FAIL_INT_EN | RTSX_SD_INT_EN | RTSX_MS_INT_EN; WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled); /* Power on SSC clock. */ RTSX_CLR(sc, RTSX_FPDCTL, RTSX_SSC_POWER_DOWN); /* Wait SSC power stable. */ DELAY(200); /* Disable ASPM */ val = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, 1); pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, val & 0xfc, 1); /* * Optimize phy. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: /* Some magic numbers from Linux driver. */ if ((error = rtsx_write_phy(sc, 0x00, 0xB966))) return (error); break; case RTSX_RTS5227: RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); /* Optimize RX sensitivity. */ if ((error = rtsx_write_phy(sc, 0x00, 0xBA42))) return (error); break; case RTSX_RTS5229: /* Optimize RX sensitivity. */ if ((error = rtsx_write_phy(sc, 0x00, 0xBA42))) return (error); break; case RTSX_RTS522A: RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); if (sc->rtsx_flags & RTSX_F_VERSION_A) { if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2, RTSX_PHY_RCR2_INIT_27S))) return (error); } if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1, RTSX_PHY_RCR1_INIT_27S))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD0, RTSX_PHY_FLD0_INIT_27S))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3, RTSX_PHY_FLD3_INIT_27S))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4, RTSX_PHY_FLD4_INIT_27S))) return (error); break; case RTSX_RTS525A: if ((error = rtsx_write_phy(sc, RTSX__PHY_FLD0, RTSX__PHY_FLD0_CLK_REQ_20C | RTSX__PHY_FLD0_RX_IDLE_EN | RTSX__PHY_FLD0_BIT_ERR_RSTN | RTSX__PHY_FLD0_BER_COUNT | RTSX__PHY_FLD0_BER_TIMER | RTSX__PHY_FLD0_CHECK_EN))) return (error); if ((error = rtsx_write_phy(sc, RTSX__PHY_ANA03, RTSX__PHY_ANA03_TIMER_MAX | RTSX__PHY_ANA03_OOBS_DEB_EN | RTSX__PHY_CMU_DEBUG_EN))) return (error); if (sc->rtsx_flags & RTSX_F_VERSION_A) if ((error = rtsx_write_phy(sc, RTSX__PHY_REV0, RTSX__PHY_REV0_FILTER_OUT | RTSX__PHY_REV0_CDR_BYPASS_PFD | RTSX__PHY_REV0_CDR_RX_IDLE_BYPASS))) return (error); break; case RTSX_RTS5249: RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); if ((error = rtsx_write_phy(sc, RTSX_PHY_REV, RTSX_PHY_REV_RESV | RTSX_PHY_REV_RXIDLE_LATCHED | RTSX_PHY_REV_P1_EN | RTSX_PHY_REV_RXIDLE_EN | RTSX_PHY_REV_CLKREQ_TX_EN | RTSX_PHY_REV_RX_PWST | RTSX_PHY_REV_CLKREQ_DT_1_0 | RTSX_PHY_REV_STOP_CLKRD | RTSX_PHY_REV_STOP_CLKWR))) return (error); DELAY(10); if ((error = rtsx_write_phy(sc, RTSX_PHY_BPCR, RTSX_PHY_BPCR_IBRXSEL | RTSX_PHY_BPCR_IBTXSEL | RTSX_PHY_BPCR_IB_FILTER | RTSX_PHY_BPCR_CMIRROR_EN))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_PCR, RTSX_PHY_PCR_FORCE_CODE | RTSX_PHY_PCR_OOBS_CALI_50 | RTSX_PHY_PCR_OOBS_VCM_08 | RTSX_PHY_PCR_OOBS_SEN_90 | RTSX_PHY_PCR_RSSI_EN | RTSX_PHY_PCR_RX10K))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2, RTSX_PHY_RCR2_EMPHASE_EN | RTSX_PHY_RCR2_NADJR | RTSX_PHY_RCR2_CDR_SR_2 | RTSX_PHY_RCR2_FREQSEL_12 | RTSX_PHY_RCR2_CDR_SC_12P | RTSX_PHY_RCR2_CALIB_LATE))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4, RTSX_PHY_FLD4_FLDEN_SEL | RTSX_PHY_FLD4_REQ_REF | RTSX_PHY_FLD4_RXAMP_OFF | RTSX_PHY_FLD4_REQ_ADDA | RTSX_PHY_FLD4_BER_COUNT | RTSX_PHY_FLD4_BER_TIMER | RTSX_PHY_FLD4_BER_CHK_EN))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_RDR, RTSX_PHY_RDR_RXDSEL_1_9 | RTSX_PHY_SSC_AUTO_PWD))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1, RTSX_PHY_RCR1_ADP_TIME_4 | RTSX_PHY_RCR1_VCO_COARSE))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3, RTSX_PHY_FLD3_TIMER_4 | RTSX_PHY_FLD3_TIMER_6 | RTSX_PHY_FLD3_RXDELINK))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE, RTSX_PHY_TUNE_TUNEREF_1_0 | RTSX_PHY_TUNE_VBGSEL_1252 | RTSX_PHY_TUNE_SDBUS_33 | RTSX_PHY_TUNE_TUNED18 | RTSX_PHY_TUNE_TUNED12 | RTSX_PHY_TUNE_TUNEA12))) return (error); break; } /* Set mcu_cnt to 7 to ensure data can be sampled properly. */ RTSX_BITOP(sc, RTSX_CLK_DIV, 0x07, 0x07); /* Disable sleep mode. */ RTSX_CLR(sc, RTSX_HOST_SLEEP_STATE, RTSX_HOST_ENTER_S1 | RTSX_HOST_ENTER_S3); /* Disable card clock. */ RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL); /* Reset delink mode. */ RTSX_CLR(sc, RTSX_CHANGE_LINK_STATE, RTSX_FORCE_RST_CORE_EN | RTSX_NON_STICKY_RST_N_DBG); /* Card driving select. */ RTSX_WRITE(sc, RTSX_CARD_DRIVE_SEL, sc->rtsx_card_drive_sel); /* Enable SSC clock. */ RTSX_WRITE(sc, RTSX_SSC_CTL1, RTSX_SSC_8X_EN | RTSX_SSC_SEL_4M); RTSX_WRITE(sc, RTSX_SSC_CTL2, 0x12); /* Disable cd_pwr_save. */ RTSX_BITOP(sc, RTSX_CHANGE_LINK_STATE, 0x16, RTSX_MAC_PHY_RST_N_DBG); /* Clear Link Ready Interrupt. */ RTSX_BITOP(sc, RTSX_IRQSTAT0, RTSX_LINK_READY_INT, RTSX_LINK_READY_INT); /* Enlarge the estimation window of PERST# glitch * to reduce the chance of invalid card interrupt. */ RTSX_WRITE(sc, RTSX_PERST_GLITCH_WIDTH, 0x80); /* Set RC oscillator to 400K. */ RTSX_CLR(sc, RTSX_RCCTL, RTSX_RCCTL_F_2M); /* Enable interrupt write-clear (default is read-clear). */ RTSX_CLR(sc, RTSX_NFTS_TX_CTRL, RTSX_INT_READ_CLR); if (sc->rtsx_device_id == RTSX_RTS525A) RTSX_BITOP(sc, RTSX_PM_CLK_FORCE_CTL, 1, 1); /* OC power down. */ RTSX_BITOP(sc, RTSX_FPDCTL, RTSX_SD_OC_POWER_DOWN, RTSX_SD_OC_POWER_DOWN); /* Enable clk_request_n to enable clock power management */ pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL + 1, 1, 1); /* Enter L1 when host tx idle */ pci_write_config(sc->rtsx_dev, 0x70F, 0x5B, 1); /* * Specific extra init. */ switch (sc->rtsx_device_id) { uint16_t cap; case RTSX_RTS5209: /* Turn off LED. */ RTSX_WRITE(sc, RTSX_CARD_GPIO, 0x03); /* Reset ASPM state to default value. */ RTSX_CLR(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK); /* Force CLKREQ# PIN to drive 0 to request clock. */ RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08); /* Configure GPIO as output. */ RTSX_WRITE(sc, RTSX_CARD_GPIO_DIR, 0x03); /* Configure driving. */ RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS5227: /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure LTR. */ cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2); if (cap & PCIEM_CTL2_LTR_ENABLE) RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3); /* Configure OBFF. */ RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE); /* Configure driving. */ if ((error = rtsx_rts5227_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88); RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN); /*!!! Added for reboot after Windows. */ RTSX_BITOP(sc, RTSX_PM_CTRL3, RTSX_PM_WAKE_EN, RTSX_PM_WAKE_EN); break; case RTSX_RTS5229: /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ /* With this reset: dd if=/dev/random of=/dev/mmcsd0 encounter a timeout. */ //!!! RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Force CLKREQ# PIN to drive 0 to request clock. */ RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08); /* Switch LDO3318 source from DV33 to card_3v3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure driving. */ RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); break; case RTSX_RTS522A: /* Add specific init from RTS5227. */ /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure LTR. */ cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2); if (cap & PCIEM_CTL2_LTR_ENABLE) RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3); /* Configure OBFF. */ RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE); /* Configure driving. */ if ((error = rtsx_rts5227_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88); RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3, 0x10); /* specific for RTS522A. */ RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL, RTSX_FUNC_FORCE_UPME_XMT_DBG, RTSX_FUNC_FORCE_UPME_XMT_DBG); RTSX_BITOP(sc, RTSX_PCLK_CTL, 0x04, 0x04); RTSX_BITOP(sc, RTSX_PM_EVENT_DEBUG, RTSX_PME_DEBUG_0, RTSX_PME_DEBUG_0); RTSX_WRITE(sc, RTSX_PM_CLK_FORCE_CTL, 0x11); break; case RTSX_RTS525A: /* Add specific init from RTS5249. */ /* Rest L1SUB Config. */ RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff); /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure driving. */ if ((error = rtsx_rts5249_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80); /* Specifc for RTS525A. */ RTSX_BITOP(sc, RTSX_PCLK_CTL, RTSX_PCLK_MODE_SEL, RTSX_PCLK_MODE_SEL); if (sc->rtsx_flags & RTSX_F_VERSION_A) { RTSX_WRITE(sc, RTSX_L1SUB_CONFIG2, RTSX_L1SUB_AUTO_CFG); RTSX_BITOP(sc, RTSX_RREF_CFG, RTSX_RREF_VBGSEL_MASK, RTSX_RREF_VBGSEL_1V25); RTSX_BITOP(sc, RTSX_LDO_VIO_CFG, RTSX_LDO_VIO_TUNE_MASK, RTSX_LDO_VIO_1V7); RTSX_BITOP(sc, RTSX_LDO_DV12S_CFG, RTSX_LDO_D12_TUNE_MASK, RTSX_LDO_D12_TUNE_DF); RTSX_BITOP(sc, RTSX_LDO_AV12S_CFG, RTSX_LDO_AV12S_TUNE_MASK, RTSX_LDO_AV12S_TUNE_DF); RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0, RTSX_LDO_VCC_LMTVTH_MASK, RTSX_LDO_VCC_LMTVTH_2A); RTSX_BITOP(sc, RTSX_OOBS_CONFIG, RTSX_OOBS_AUTOK_DIS | RTSX_OOBS_VAL_MASK, 0x89); } break; case RTSX_RTS5249: /* Rest L1SUB Config. */ RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff); /* Configure GPIO as output. */ RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON); /* Reset ASPM state to default value. */ RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM); /* Switch LDO3318 source from DV33 to 3V3. */ RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33); RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3); /* Set default OLT blink period. */ RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD); /* Configure driving. */ if ((error = rtsx_rts5249_fill_driving(sc))) return (error); /* Configure force_clock_req. */ if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0); else RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80); break; case RTSX_RTL8402: case RTSX_RTL8411: RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE, RTSX_CD_ENABLE); break; case RTSX_RTL8411B: if (sc->rtsx_flags & RTSX_F_8411B_QFN48) RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5); RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3); /* Enable SD interrupt. */ RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE, RTSX_CD_ENABLE); /* Clear hw_pfm_en to disable hardware PFM mode. */ RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL, 0x06, 0x00); break; } /*!!! Added for reboot after Windows. */ rtsx_bus_power_off(sc); rtsx_set_sd_timing(sc, bus_timing_normal); rtsx_set_sd_clock(sc, 0); /*!!! Added for reboot after Windows. */ return (0); } static int rtsx_map_sd_drive(int index) { uint8_t sd_drive[4] = { 0x01, /* Type D */ 0x02, /* Type C */ 0x05, /* Type A */ 0x03 /* Type B */ }; return (sd_drive[index]); } /* For voltage 3v3. */ static int rtsx_rts5227_fill_driving(struct rtsx_softc *sc) { u_char driving_3v3[4][3] = { {0x13, 0x13, 0x13}, {0x96, 0x96, 0x96}, {0x7F, 0x7F, 0x7F}, {0x96, 0x96, 0x96}, }; RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]); RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]); RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]); return (0); } /* For voltage 3v3. */ static int rtsx_rts5249_fill_driving(struct rtsx_softc *sc) { u_char driving_3v3[4][3] = { {0x11, 0x11, 0x18}, {0x55, 0x55, 0x5C}, {0xFF, 0xFF, 0xFF}, {0x96, 0x96, 0x96}, }; RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]); RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]); RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]); return (0); } static int rtsx_read(struct rtsx_softc *sc, uint16_t addr, uint8_t *val) { int tries = 1024; uint32_t reg; WRITE4(sc, RTSX_HAIMR, RTSX_HAIMR_BUSY | (uint32_t)((addr & 0x3FFF) << 16)); while (tries--) { reg = READ4(sc, RTSX_HAIMR); if (!(reg & RTSX_HAIMR_BUSY)) break; } *val = (reg & 0xff); return ((tries == 0) ? ETIMEDOUT : 0); } static int rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val) { int tries = 1024; uint8_t data0, data1, data2, data3, rwctl; RTSX_WRITE(sc, RTSX_CFGADDR0, addr); RTSX_WRITE(sc, RTSX_CFGADDR1, addr >> 8); RTSX_WRITE(sc, RTSX_CFGRWCTL, RTSX_CFG_BUSY | (func & 0x03 << 4)); while (tries--) { RTSX_READ(sc, RTSX_CFGRWCTL, &rwctl); if (!(rwctl & RTSX_CFG_BUSY)) break; } if (tries == 0) return (ETIMEDOUT); RTSX_READ(sc, RTSX_CFGDATA0, &data0); RTSX_READ(sc, RTSX_CFGDATA1, &data1); RTSX_READ(sc, RTSX_CFGDATA2, &data2); RTSX_READ(sc, RTSX_CFGDATA3, &data3); *val = (data3 << 24) | (data2 << 16) | (data1 << 8) | data0; return (0); } static int rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val) { int tries = 1024; uint32_t reg; WRITE4(sc, RTSX_HAIMR, RTSX_HAIMR_BUSY | RTSX_HAIMR_WRITE | (uint32_t)(((addr & 0x3FFF) << 16) | (mask << 8) | val)); while (tries--) { reg = READ4(sc, RTSX_HAIMR); if (!(reg & RTSX_HAIMR_BUSY)) { if (val != (reg & 0xff)) return (EIO); return (0); } } return (ETIMEDOUT); } static int rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val) { int tries = 100000; uint8_t data0, data1, rwctl; RTSX_WRITE(sc, RTSX_PHY_ADDR, addr); RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_READ); while (tries--) { RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl); if (!(rwctl & RTSX_PHY_BUSY)) break; } if (tries == 0) return (ETIMEDOUT); RTSX_READ(sc, RTSX_PHY_DATA0, &data0); RTSX_READ(sc, RTSX_PHY_DATA1, &data1); *val = data1 << 8 | data0; return (0); } static int rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val) { int tries = 100000; uint8_t rwctl; RTSX_WRITE(sc, RTSX_PHY_DATA0, val); RTSX_WRITE(sc, RTSX_PHY_DATA1, val >> 8); RTSX_WRITE(sc, RTSX_PHY_ADDR, addr); RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_WRITE); while (tries--) { RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl); if (!(rwctl & RTSX_PHY_BUSY)) break; } return ((tries == 0) ? ETIMEDOUT : 0); } /* * Notice that the meaning of RTSX_PWR_GATE_CTRL changes between RTS5209 and * RTS5229. In RTS5209 it is a mask of disabled power gates, while in RTS5229 * it is a mask of *enabled* gates. */ static int rtsx_bus_power_off(struct rtsx_softc *sc) { if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_bus_power_off()\n"); /* Disable SD clock. */ RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN); /* Disable SD output. */ RTSX_CLR(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN); /* Turn off power. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK, RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA); RTSX_SET(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_OFF); break; case RTSX_RTS5227: case RTSX_RTS5229: case RTSX_RTS522A: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK, RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA); RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK); break; case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_OFF); RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB, RTSX_BPP_LDO_SUSPEND); break; default: RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK); RTSX_SET(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_OFF); RTSX_CLR(sc, RTSX_CARD_PWR_CTL, RTSX_PMOS_STRG_800mA); break; } /* Disable pull control. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); break; case RTSX_RTS5227: case RTSX_RTS522A: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); break; case RTSX_RTS5229: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); if (sc->rtsx_flags & RTSX_F_VERSION_C) RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3_TYPE_C); else RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); break; case RTSX_RTS525A: case RTSX_RTS5249: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x55); break; case RTSX_RTL8402: case RTSX_RTL8411: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0x95); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x05); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04); break; case RTSX_RTL8411B: if (sc->rtsx_flags & RTSX_F_8411B_QFN48) { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15); } else { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd5); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15); } break; } return (0); } static int rtsx_bus_power_on(struct rtsx_softc *sc) { if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_bus_power_on()\n"); /* Select SD card. */ RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL); RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD); /* Enable SD clock. */ RTSX_BITOP(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN, RTSX_SD_CLK_EN); /* Enable pull control. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); break; case RTSX_RTS5227: case RTSX_RTS522A: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); break; case RTSX_RTS5229: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); if (sc->rtsx_flags & RTSX_F_VERSION_C) RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3_TYPE_C); else RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); break; case RTSX_RTS525A: case RTSX_RTS5249: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0xaa); break; case RTSX_RTL8402: case RTSX_RTL8411: RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xa9); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x09); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04); break; case RTSX_RTL8411B: if (sc->rtsx_flags & RTSX_F_8411B_QFN48) { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf9); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x19); } else { RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd9); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55); RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15); } break; } /* * To avoid a current peak, enable card power in two phases * with a delay in between. */ switch (sc->rtsx_device_id) { case RTSX_RTS5209: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC2); DELAY(200); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_ON); break; case RTSX_RTS5227: case RTSX_RTS522A: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1); DELAY(200); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2); RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN, RTSX_SD_OUTPUT_EN); RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_MS_OUTPUT_EN, RTSX_MS_OUTPUT_EN); break; case RTSX_RTS5229: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1); DELAY(200); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2); break; case RTSX_RTS525A: RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_VCC_TUNE_MASK, RTSX_LDO_VCC_3V3); case RTSX_RTS5249: /* Partial power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1); DELAY(200); /* Full power. */ RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON); RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2); break; case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_5_PERCENT_ON); RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB, RTSX_BPP_LDO_SUSPEND); DELAY(150); RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_10_PERCENT_ON); DELAY(150); RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_15_PERCENT_ON); DELAY(150); RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK, RTSX_BPP_POWER_ON); RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB, RTSX_BPP_LDO_ON); break; } /* Enable SD card output. */ RTSX_WRITE(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN); DELAY(200); return (0); } /* * Set but width. */ static int rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width) { uint32_t bus_width; switch (width) { case bus_width_1: bus_width = RTSX_BUS_WIDTH_1; break; case bus_width_4: bus_width = RTSX_BUS_WIDTH_4; break; case bus_width_8: bus_width = RTSX_BUS_WIDTH_8; break; default: return (MMC_ERR_INVALID); } RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_BUS_WIDTH_MASK, bus_width); if (bootverbose || sc->rtsx_debug) { char *busw[] = { "1 bit", "4 bits", "8 bits" }; device_printf(sc->rtsx_dev, "Setting bus width to %s\n", busw[bus_width]); } return (0); } static int rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing) { if (timing == bus_timing_hs && sc->rtsx_force_timing) { timing = bus_timing_uhs_sdr50; sc->rtsx_ios_timing = timing; } if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_set_sd_timing(%u)\n", timing); switch (timing) { case bus_timing_uhs_sdr50: case bus_timing_uhs_sdr104: sc->rtsx_double_clk = false; sc->rtsx_vpclk = true; RTSX_BITOP(sc, RTSX_SD_CFG1, 0x0c | RTSX_SD_ASYNC_FIFO_NOT_RST, RTSX_SD30_MODE | RTSX_SD_ASYNC_FIFO_NOT_RST); RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE, RTSX_CRC_VAR_CLK0 | RTSX_SD30_FIX_CLK | RTSX_SAMPLE_VAR_CLK1); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); break; case bus_timing_hs: RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE); RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE, RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); RTSX_BITOP(sc, RTSX_SD_PUSH_POINT_CTL, RTSX_SD20_TX_SEL_MASK, RTSX_SD20_TX_14_AHEAD); RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL, RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_14_DELAY); break; default: RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE); RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE, RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_SD_PUSH_POINT_CTL, RTSX_SD20_TX_NEG_EDGE); RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL, RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_POS_EDGE); break; } return (0); } /* * Set or change SDCLK frequency or disable the SD clock. * Return zero on success. */ static int rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq) { uint8_t clk; uint8_t clk_divider, n, div, mcu; int error = 0; if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_set_sd_clock(%u)\n", freq); if (freq == RTSX_SDCLK_OFF) { error = rtsx_stop_sd_clock(sc); return error; } sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_500K; sc->rtsx_discovery_mode = (freq <= 1000000) ? true : false; if (sc->rtsx_discovery_mode) { /* We use 250k(around) here, in discovery stage. */ clk_divider = RTSX_CLK_DIVIDE_128; freq = 30000000; } else { clk_divider = RTSX_CLK_DIVIDE_0; } RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, clk_divider); freq /= 1000000; if (sc->rtsx_discovery_mode || !sc->rtsx_double_clk) clk = freq; else clk = freq * 2; switch (sc->rtsx_device_id) { case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: n = clk * 4 / 5 - 2; break; default: n = clk - 2; break; } if ((clk <= 2) || (n > RTSX_MAX_DIV_N)) return (MMC_ERR_INVALID); mcu = 125 / clk + 3; if (mcu > 15) mcu = 15; /* Make sure that the SSC clock div_n is not less than RTSX_MIN_DIV_N. */ div = RTSX_CLK_DIV_1; while ((n < RTSX_MIN_DIV_N) && (div < RTSX_CLK_DIV_8)) { switch (sc->rtsx_device_id) { case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: n = (((n + 2) * 5 / 4) * 2) * 4 / 5 - 2; break; default: n = (n + 2) * 2 - 2; break; } div++; } if (sc->rtsx_double_clk && sc->rtsx_ssc_depth > 1) sc->rtsx_ssc_depth -= 1; if (div > RTSX_CLK_DIV_1) { if (sc->rtsx_ssc_depth > (div - 1)) sc->rtsx_ssc_depth -= (div - 1); else sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_4M; } /* Enable SD clock. */ error = rtsx_switch_sd_clock(sc, clk, n, div, mcu); return (error); } static int rtsx_stop_sd_clock(struct rtsx_softc *sc) { RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL); RTSX_SET(sc, RTSX_SD_BUS_STAT, RTSX_SD_CLK_FORCE_STOP); return (0); } static int rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu) { if (bootverbose || sc->rtsx_debug) { device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - discovery-mode is %s, ssc_depth: %d\n", (sc->rtsx_discovery_mode) ? "true" : "false", sc->rtsx_ssc_depth); device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - clk: %d, n: %d, div: %d, mcu: %d\n", clk, n, div, mcu); } RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ); RTSX_WRITE(sc, RTSX_CLK_DIV, (div << 4) | mcu); RTSX_CLR(sc, RTSX_SSC_CTL1, RTSX_RSTB); RTSX_BITOP(sc, RTSX_SSC_CTL2, RTSX_SSC_DEPTH_MASK, sc->rtsx_ssc_depth); RTSX_WRITE(sc, RTSX_SSC_DIV_N_0, n); RTSX_BITOP(sc, RTSX_SSC_CTL1, RTSX_RSTB, RTSX_RSTB); if (sc->rtsx_vpclk) { RTSX_CLR(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET); RTSX_BITOP(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET); } /* Wait SSC clock stable. */ DELAY(200); RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ); return (0); } static void rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point) { if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_sd_change_tx_phase() - sample_point: %d\n", sample_point); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK); rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_SELECT_MASK, sample_point); rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, 0); rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0); rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0); } static void rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point) { if (bootverbose || sc->rtsx_debug == 2) device_printf(sc->rtsx_dev, "rtsx_sd_change_rx_phase() - sample_point: %d\n", sample_point); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK); rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_SELECT_MASK, sample_point); rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, 0); rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET); rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0); rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0); } static void rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map) { uint32_t raw_phase_map = 0; int i; int error; for (i = 0; i < RTSX_RX_PHASE_MAX; i++) { error = rtsx_sd_tuning_rx_cmd(sc, (uint8_t)i); if (error == 0) raw_phase_map |= 1 << i; } if (phase_map != NULL) *phase_map = raw_phase_map; } static int rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point) { struct mmc_request req = {}; struct mmc_command cmd = {}; int error = 0; cmd.opcode = MMC_SEND_TUNING_BLOCK; cmd.arg = 0; req.cmd = &cmd; RTSX_LOCK(sc); sc->rtsx_req = &req; rtsx_sd_change_rx_phase(sc, sample_point); rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN, RTSX_SD_RSP_80CLK_TIMEOUT_EN); rtsx_init_cmd(sc, &cmd); rtsx_set_cmd_data_len(sc, 1, 0x40); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_AUTO_TUNING | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Set interrupt post processing */ sc->rtsx_intr_trans_ok = rtsx_sd_tuning_rx_cmd_wakeup; sc->rtsx_intr_trans_ko = rtsx_sd_tuning_rx_cmd_wakeup; /* Run the command queue. */ rtsx_send_cmd(sc); error = rtsx_sd_tuning_rx_cmd_wait(sc, &cmd); if (error) { if (bootverbose || sc->rtsx_debug == 2) device_printf(sc->rtsx_dev, "rtsx_sd_tuning_rx_cmd() - error: %d\n", error); rtsx_sd_wait_data_idle(sc); rtsx_clear_error(sc); } rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN, 0); sc->rtsx_req = NULL; RTSX_UNLOCK(sc); return (error); } static int rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd) { int status; int mask = RTSX_TRANS_OK_INT | RTSX_TRANS_FAIL_INT; status = sc->rtsx_intr_status & mask; while (status == 0) { if (msleep(&sc->rtsx_intr_status, &sc->rtsx_mtx, 0, "rtsxintr", sc->rtsx_timeout) == EWOULDBLOCK) { cmd->error = MMC_ERR_TIMEOUT; return (MMC_ERR_TIMEOUT); } status = sc->rtsx_intr_status & mask; } return (cmd->error); } static void rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc) { wakeup(&sc->rtsx_intr_status); } static void rtsx_sd_wait_data_idle(struct rtsx_softc *sc) { int i; uint8_t val; for (i = 0; i < 100; i++) { rtsx_read(sc, RTSX_SD_DATA_STATE, &val); if (val & RTSX_SD_DATA_IDLE) return; DELAY(100); } } static uint8_t rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map) { int start = 0, len = 0; int start_final = 0, len_final = 0; uint8_t final_phase = 0xff; while (start < RTSX_RX_PHASE_MAX) { len = rtsx_sd_get_rx_phase_len(phase_map, start); if (len_final < len) { start_final = start; len_final = len; } start += len ? len : 1; } final_phase = (start_final + len_final / 2) % RTSX_RX_PHASE_MAX; if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_sd_search_final_rx_phase() - phase_map: %x, start_final: %d, len_final: %d, final_phase: %d\n", phase_map, start_final, len_final, final_phase); return final_phase; } static int rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit) { int i; for (i = 0; i < RTSX_RX_PHASE_MAX; i++) { if ((phase_map & (1 << (start_bit + i) % RTSX_RX_PHASE_MAX)) == 0) return i; } return RTSX_RX_PHASE_MAX; } #if 0 /* For led */ static int rtsx_led_enable(struct rtsx_softc *sc) { switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_CLR(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF); RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED); break; case RTSX_RTL8411B: RTSX_CLR(sc, RTSX_GPIO_CTL, 0x01); RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED); break; default: RTSX_SET(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON); RTSX_SET(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK); break; } return (0); } static int rtsx_led_disable(struct rtsx_softc *sc) { switch (sc->rtsx_device_id) { case RTSX_RTS5209: RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN); RTSX_WRITE(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF); break; case RTSX_RTL8411B: RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN); RTSX_SET(sc, RTSX_GPIO_CTL, 0x01); break; default: RTSX_CLR(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK); RTSX_CLR(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON); break; } return (0); } #endif /* For led */ static uint8_t rtsx_response_type(uint16_t mmc_rsp) { int i; struct rsp_type { uint16_t mmc_rsp; uint8_t rtsx_rsp; } rsp_types[] = { { MMC_RSP_NONE, RTSX_SD_RSP_TYPE_R0 }, { MMC_RSP_R1, RTSX_SD_RSP_TYPE_R1 }, { MMC_RSP_R1B, RTSX_SD_RSP_TYPE_R1B }, { MMC_RSP_R2, RTSX_SD_RSP_TYPE_R2 }, { MMC_RSP_R3, RTSX_SD_RSP_TYPE_R3 }, { MMC_RSP_R4, RTSX_SD_RSP_TYPE_R4 }, { MMC_RSP_R5, RTSX_SD_RSP_TYPE_R5 }, { MMC_RSP_R6, RTSX_SD_RSP_TYPE_R6 }, { MMC_RSP_R7, RTSX_SD_RSP_TYPE_R7 } }; for (i = 0; i < nitems(rsp_types); i++) { if (mmc_rsp == rsp_types[i].mmc_rsp) return (rsp_types[i].rtsx_rsp); } return (0); } /* * Init command buffer with SD command index and argument. */ static void rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd) { sc->rtsx_cmd_index = 0; rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD0, 0xff, RTSX_SD_CMD_START | cmd->opcode); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD1, 0xff, cmd->arg >> 24); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD2, 0xff, cmd->arg >> 16); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD3, 0xff, cmd->arg >> 8); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD4, 0xff, cmd->arg); } /* * Append a properly encoded host command to the host command buffer. */ static void rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg, uint8_t mask, uint8_t data) { KASSERT(sc->rtsx_cmd_index < RTSX_HOSTCMD_MAX, ("rtsx: Too many host commands (%d)\n", sc->rtsx_cmd_index)); uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem); cmd_buffer[sc->rtsx_cmd_index++] = htole32((uint32_t)(cmd & 0x3) << 30) | ((uint32_t)(reg & 0x3fff) << 16) | ((uint32_t)(mask) << 8) | ((uint32_t)data); } /* * Queue commands to configure data transfer size. */ static void rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt) { rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_L, 0xff, block_cnt & 0xff); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_H, 0xff, block_cnt >> 8); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_L, 0xff, byte_cnt & 0xff); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_H, 0xff, byte_cnt >> 8); } /* * Run the command queue. */ static void rtsx_send_cmd(struct rtsx_softc *sc) { if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_send_cmd()\n"); sc->rtsx_intr_status = 0; /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREWRITE); /* Tell the chip where the command buffer is and run the commands. */ WRITE4(sc, RTSX_HCBAR, (uint32_t)sc->rtsx_cmd_buffer); WRITE4(sc, RTSX_HCBCTLR, ((sc->rtsx_cmd_index * 4) & 0x00ffffff) | RTSX_START_CMD | RTSX_HW_AUTO_RSP); } /* * Stop previous command. */ static void rtsx_stop_cmd(struct rtsx_softc *sc) { /* Stop command transfer. */ WRITE4(sc, RTSX_HCBCTLR, RTSX_STOP_CMD); /* Stop DMA transfer. */ WRITE4(sc, RTSX_HDBCTLR, RTSX_STOP_DMA); rtsx_write(sc, RTSX_DMACTL, RTSX_DMA_RST, RTSX_DMA_RST); rtsx_write(sc, RTSX_RBCTL, RTSX_RB_FLUSH, RTSX_RB_FLUSH); } /* * Clear error. */ static void rtsx_clear_error(struct rtsx_softc *sc) { /* Clear error. */ rtsx_write(sc, RTSX_CARD_STOP, RTSX_SD_STOP | RTSX_SD_CLR_ERR, RTSX_SD_STOP | RTSX_SD_CLR_ERR); } /* * Signal end of request to mmc/mmcsd. */ static void rtsx_req_done(struct rtsx_softc *sc) { #ifdef MMCCAM union ccb *ccb; #endif /* MMCCAM */ struct mmc_request *req; req = sc->rtsx_req; if (req->cmd->error == MMC_ERR_NONE) { if (req->cmd->opcode == MMC_READ_SINGLE_BLOCK || req->cmd->opcode == MMC_READ_MULTIPLE_BLOCK) sc->rtsx_read_count++; else if (req->cmd->opcode == MMC_WRITE_BLOCK || req->cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK) sc->rtsx_write_count++; } else { rtsx_clear_error(sc); } callout_stop(&sc->rtsx_timeout_callout); sc->rtsx_req = NULL; #ifdef MMCCAM ccb = sc->rtsx_ccb; sc->rtsx_ccb = NULL; ccb->ccb_h.status = (req->cmd->error == 0 ? CAM_REQ_CMP : CAM_REQ_CMP_ERR); xpt_done(ccb); #else req->done(req); #endif /* MMCCAM */ } /* * Send request. */ static int rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd) { uint8_t rsp_type; uint16_t reg; if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_send_req() - CMD%d\n", cmd->opcode); /* Convert response type. */ rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK); if (rsp_type == 0) { device_printf(sc->rtsx_dev, "Unknown rsp_type: 0x%lx\n", (cmd->flags & MMC_RSP_MASK)); cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } rtsx_init_cmd(sc, cmd); /* Queue command to set response type. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, rsp_type); /* Use the ping-pong buffer (cmd buffer) for commands which do not transfer data. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE, 0x01, RTSX_PINGPONG_BUFFER); /* Queue commands to perform SD transfer. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_CMD_RSP | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE, RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE); /* If needed queue commands to read back card status response. */ if (rsp_type == RTSX_SD_RSP_TYPE_R2) { /* Read data from ping-pong buffer. */ for (reg = RTSX_PPBUF_BASE2; reg < RTSX_PPBUF_BASE2 + 16; reg++) rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0); } else if (rsp_type != RTSX_SD_RSP_TYPE_R0) { /* Read data from SD_CMDx registers. */ for (reg = RTSX_SD_CMD0; reg <= RTSX_SD_CMD4; reg++) rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0); } rtsx_push_cmd(sc, RTSX_READ_REG_CMD, RTSX_SD_STAT1, 0, 0); /* Set transfer OK function. */ if (sc->rtsx_intr_trans_ok == NULL) sc->rtsx_intr_trans_ok = rtsx_ret_resp; /* Run the command queue. */ rtsx_send_cmd(sc); return (0); } /* * Return response of previous command (case cmd->data == NULL) and complete resquest. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_ret_resp(struct rtsx_softc *sc) { struct mmc_command *cmd; cmd = sc->rtsx_req->cmd; rtsx_set_resp(sc, cmd); rtsx_req_done(sc); } /* * Set response of previous command. */ static void rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd) { uint8_t rsp_type; rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK); /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE); /* Copy card response into mmc response buffer. */ if (ISSET(cmd->flags, MMC_RSP_PRESENT)) { uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem); if (bootverbose) { device_printf(sc->rtsx_dev, "cmd_buffer: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", cmd_buffer[0], cmd_buffer[1], cmd_buffer[2], cmd_buffer[3], cmd_buffer[4]); } if (rsp_type == RTSX_SD_RSP_TYPE_R2) { /* First byte is CHECK_REG_CMD return value, skip it. */ unsigned char *ptr = (unsigned char *)cmd_buffer + 1; int i; /* * The controller offloads the last byte {CRC-7, end bit 1} * of response type R2. Assign dummy CRC, 0, and end bit to this * byte (ptr[16], goes into the LSB of resp[3] later). */ ptr[16] = 0x01; /* The second byte is the status of response, skip it. */ for (i = 0; i < 4; i++) cmd->resp[i] = be32dec(ptr + 1 + i * 4); } else { /* * First byte is CHECK_REG_CMD return value, second * one is the command op code -- we skip those. */ cmd->resp[0] = ((be32toh(cmd_buffer[0]) & 0x0000ffff) << 16) | ((be32toh(cmd_buffer[1]) & 0xffff0000) >> 16); } if (bootverbose) device_printf(sc->rtsx_dev, "cmd->resp: 0x%08x 0x%08x 0x%08x 0x%08x\n", cmd->resp[0], cmd->resp[1], cmd->resp[2], cmd->resp[3]); } } /* * Use the ping-pong buffer (cmd buffer) for transfer <= 512 bytes. */ static int rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd) { int read; if (cmd->data == NULL || cmd->data->len == 0) { cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ? RTSX_MAX_DATA_BLKLEN : cmd->data->len; read = ISSET(cmd->data->flags, MMC_DATA_READ); if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_xfer_short() - %s xfer: %ld bytes with block size %ld\n", read ? "Read" : "Write", (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len); if (cmd->data->len > 512) { device_printf(sc->rtsx_dev, "rtsx_xfer_short() - length too large: %ld > 512\n", (unsigned long)cmd->data->len); cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } if (read) { if (sc->rtsx_discovery_mode) rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_0); rtsx_init_cmd(sc, cmd); /* Queue commands to configure data transfer size. */ rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len); /* From Linux: rtsx_pci_sdmmc.c sd_read_data(). */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6); /* Use the ping-pong buffer (cmd buffer). */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE, 0x01, RTSX_PINGPONG_BUFFER); /* Queue commands to perform SD transfer. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_NORMAL_READ | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_ask_ppbuf_part1; /* Run the command queue. */ rtsx_send_cmd(sc); } else { /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part1; /* Run the command queue. */ rtsx_send_req(sc, cmd); } return (0); } /* * Use the ping-pong buffer (cmd buffer) for the transfer - first part <= 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_ask_ppbuf_part1(struct rtsx_softc *sc) { struct mmc_command *cmd; uint16_t reg = RTSX_PPBUF_BASE2; int len; int i; cmd = sc->rtsx_req->cmd; len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len; sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0); } /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part1; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Get the data from the ping-pong buffer (cmd buffer) - first part <= 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_get_ppbuf_part1(struct rtsx_softc *sc) { struct mmc_command *cmd; uint8_t *ptr; int len; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len; /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE); memcpy(ptr, sc->rtsx_cmd_dmamem, len); len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? cmd->data->len - RTSX_HOSTCMD_MAX : 0; /* Use the ping-pong buffer (cmd buffer) for the transfer - second part > 256 bytes. */ if (len > 0) { uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX; int i; sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0); } /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part2; /* Run the command queue. */ rtsx_send_cmd(sc); } else { if (bootverbose && cmd->opcode == ACMD_SEND_SCR) { uint8_t *ptr = cmd->data->data; device_printf(sc->rtsx_dev, "SCR: 0x%02x%02x%02x%02x%02x%02x%02x%02x\n", ptr[0], ptr[1], ptr[2], ptr[3], ptr[4], ptr[5], ptr[6], ptr[7]); } if (sc->rtsx_discovery_mode) rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128); rtsx_req_done(sc); } } /* * Get the data from the ping-pong buffer (cmd buffer) - second part > 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_get_ppbuf_part2(struct rtsx_softc *sc) { struct mmc_command *cmd; uint8_t *ptr; int len; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; ptr += RTSX_HOSTCMD_MAX; len = cmd->data->len - RTSX_HOSTCMD_MAX; /* Sync command DMA buffer. */ bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE); memcpy(ptr, sc->rtsx_cmd_dmamem, len); if (sc->rtsx_discovery_mode) rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128); rtsx_req_done(sc); } /* * Use the ping-pong buffer (cmd buffer) for transfer - first part <= 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_put_ppbuf_part1(struct rtsx_softc *sc) { struct mmc_command *cmd; uint16_t reg = RTSX_PPBUF_BASE2; uint8_t *ptr; int len; int i; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len; rtsx_set_resp(sc, cmd); sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr); ptr++; } /* Set transfer OK function. */ if (cmd->data->len > RTSX_HOSTCMD_MAX) sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part2; else sc->rtsx_intr_trans_ok = rtsx_write_ppbuf; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Use the ping-pong buffer (cmd buffer) for transfer - second part > 256 bytes. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_put_ppbuf_part2(struct rtsx_softc *sc) { struct mmc_command *cmd; uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX; uint8_t *ptr; int len; int i; cmd = sc->rtsx_req->cmd; ptr = cmd->data->data; ptr += RTSX_HOSTCMD_MAX; len = cmd->data->len - RTSX_HOSTCMD_MAX; sc->rtsx_cmd_index = 0; for (i = 0; i < len; i++) { rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr); ptr++; } /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_write_ppbuf; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Write the data previously given via the ping-pong buffer on the card. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_write_ppbuf(struct rtsx_softc *sc) { struct mmc_command *cmd; cmd = sc->rtsx_req->cmd; sc->rtsx_cmd_index = 0; /* Queue commands to configure data transfer size. */ rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len); /* From Linux: rtsx_pci_sdmmc.c sd_write_data(). */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_0); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, RTSX_TM_AUTO_WRITE3 | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_req_done; /* Run the command queue. */ rtsx_send_cmd(sc); } /* * Use the data buffer for transfer > 512 bytes. */ static int rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd) { int read = ISSET(cmd->data->flags, MMC_DATA_READ); cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ? RTSX_MAX_DATA_BLKLEN : cmd->data->len; if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_xfer() - %s xfer: %ld bytes with block size %ld\n", read ? "Read" : "Write", (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len); if (cmd->data->len > RTSX_DMA_DATA_BUFSIZE) { device_printf(sc->rtsx_dev, "rtsx_xfer() length too large: %ld > %d\n", (unsigned long)cmd->data->len, RTSX_DMA_DATA_BUFSIZE); cmd->error = MMC_ERR_INVALID; return (MMC_ERR_INVALID); } if (!read) { /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_xfer_begin; /* Run the command queue. */ rtsx_send_req(sc, cmd); } else { rtsx_xfer_start(sc); } return (0); } /* * Get request response and start dma data transfer (write command). * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_xfer_begin(struct rtsx_softc *sc) { struct mmc_command *cmd; cmd = sc->rtsx_req->cmd; if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_xfer_begin() - CMD%d\n", cmd->opcode); rtsx_set_resp(sc, cmd); rtsx_xfer_start(sc); } /* * Start dma data transfer. */ static void rtsx_xfer_start(struct rtsx_softc *sc) { struct mmc_command *cmd; int read; uint8_t cfg2; int dma_dir; int tmode; cmd = sc->rtsx_req->cmd; read = ISSET(cmd->data->flags, MMC_DATA_READ); /* Configure DMA transfer mode parameters. */ if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK) cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_6; else cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_0; if (read) { dma_dir = RTSX_DMA_DIR_FROM_CARD; /* * Use transfer mode AUTO_READ1, which assume we not * already send the read command and don't need to send * CMD 12 manually after read. */ tmode = RTSX_TM_AUTO_READ1; cfg2 |= RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC7; rtsx_init_cmd(sc, cmd); } else { dma_dir = RTSX_DMA_DIR_TO_CARD; /* * Use transfer mode AUTO_WRITE3, wich assumes we've already * sent the write command and gotten the response, and will * send CMD 12 manually after writing. */ tmode = RTSX_TM_AUTO_WRITE3; cfg2 |= RTSX_SD_NO_CALCULATE_CRC7 | RTSX_SD_NO_CHECK_CRC7; sc->rtsx_cmd_index = 0; } /* Queue commands to configure data transfer size. */ rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len); /* Configure DMA controller. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_IRQSTAT0, RTSX_DMA_DONE_INT, RTSX_DMA_DONE_INT); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC3, 0xff, cmd->data->len >> 24); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC2, 0xff, cmd->data->len >> 16); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC1, 0xff, cmd->data->len >> 8); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC0, 0xff, cmd->data->len); rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMACTL, RTSX_DMA_EN | RTSX_DMA_DIR | RTSX_DMA_PACK_SIZE_MASK, RTSX_DMA_EN | dma_dir | RTSX_DMA_512); /* Use the DMA ring buffer for commands which transfer data. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE, 0x01, RTSX_RING_BUFFER); /* Queue command to set response type. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, cfg2); /* Queue commands to perform SD transfer. */ rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff, tmode | RTSX_SD_TRANSFER_START); rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER, RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END); /* Run the command queue. */ rtsx_send_cmd(sc); if (!read) memcpy(sc->rtsx_data_dmamem, cmd->data->data, cmd->data->len); /* Sync data DMA buffer. */ bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREREAD); bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREWRITE); /* Set transfer OK function. */ sc->rtsx_intr_trans_ok = rtsx_xfer_finish; /* Tell the chip where the data buffer is and run the transfer. */ WRITE4(sc, RTSX_HDBAR, sc->rtsx_data_buffer); WRITE4(sc, RTSX_HDBCTLR, RTSX_TRIG_DMA | (read ? RTSX_DMA_READ : 0) | (cmd->data->len & 0x00ffffff)); } /* * Finish dma data transfer. * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok. */ static void rtsx_xfer_finish(struct rtsx_softc *sc) { struct mmc_command *cmd; int read; cmd = sc->rtsx_req->cmd; if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_xfer_finish() - CMD%d\n", cmd->opcode); read = ISSET(cmd->data->flags, MMC_DATA_READ); /* Sync data DMA buffer. */ bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTWRITE); if (read) { memcpy(cmd->data->data, sc->rtsx_data_dmamem, cmd->data->len); rtsx_req_done(sc); } else { /* Send CMD12 after AUTO_WRITE3 (see mmcsd_rw() in mmcsd.c) */ /* and complete request. */ sc->rtsx_intr_trans_ok = NULL; rtsx_send_req(sc, sc->rtsx_req->stop); } } /* * Manage request timeout. */ static void rtsx_timeout(void *arg) { struct rtsx_softc *sc; sc = (struct rtsx_softc *)arg; if (sc->rtsx_req != NULL) { device_printf(sc->rtsx_dev, "Controller timeout for CMD%u\n", sc->rtsx_req->cmd->opcode); sc->rtsx_req->cmd->error = MMC_ERR_TIMEOUT; rtsx_stop_cmd(sc); rtsx_req_done(sc); } else { device_printf(sc->rtsx_dev, "Controller timeout!\n"); } } #ifdef MMCCAM static void rtsx_cam_action(struct cam_sim *sim, union ccb *ccb) { struct rtsx_softc *sc; sc = cam_sim_softc(sim); if (sc == NULL) { ccb->ccb_h.status = CAM_SEL_TIMEOUT; xpt_done(ccb); return; } switch (ccb->ccb_h.func_code) { case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; cpi->version_num = 1; /* SIM driver version number - now all drivers use 1 */ cpi->hba_inquiry = 0; /* bitmask of features supported by the controller */ cpi->target_sprt = 0; /* flags for target mode support */ cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN; cpi->hba_eng_cnt = 0; /* HBA engine count - always set to 0 */ cpi->max_target = 0; /* maximal supported target ID */ cpi->max_lun = 0; /* maximal supported LUN ID */ cpi->initiator_id = 1; /* the SCSI ID of the controller itself */ cpi->maxio = RTSX_DMA_DATA_BUFSIZE; /* maximum io size */ strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); /* vendor ID of the SIM */ strncpy(cpi->hba_vid, "Realtek", HBA_IDLEN); /* vendor ID of the HBA */ strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); /* device name for SIM */ cpi->unit_number = cam_sim_unit(sim); /* controller unit number */ cpi->bus_id = cam_sim_bus(sim); /* bus number */ cpi->protocol = PROTO_MMCSD; cpi->protocol_version = SCSI_REV_0; cpi->transport = XPORT_MMCSD; cpi->transport_version = 1; cpi->ccb_h.status = CAM_REQ_CMP; break; } case XPT_GET_TRAN_SETTINGS: { struct ccb_trans_settings *cts = &ccb->cts; if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_action() - got XPT_GET_TRAN_SETTINGS\n"); cts->protocol = PROTO_MMCSD; cts->protocol_version = 1; cts->transport = XPORT_MMCSD; cts->transport_version = 1; cts->xport_specific.valid = 0; cts->proto_specific.mmc.host_ocr = sc->rtsx_host.host_ocr; cts->proto_specific.mmc.host_f_min = sc->rtsx_host.f_min; cts->proto_specific.mmc.host_f_max = sc->rtsx_host.f_max; cts->proto_specific.mmc.host_caps = sc->rtsx_host.caps; #if __FreeBSD__ > 12 cts->proto_specific.mmc.host_max_data = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE; #endif memcpy(&cts->proto_specific.mmc.ios, &sc->rtsx_host.ios, sizeof(struct mmc_ios)); ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_SET_TRAN_SETTINGS: if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_action() - got XPT_SET_TRAN_SETTINGS\n"); /* Apply settings and set ccb->ccb_h.status accordingly. */ rtsx_cam_set_tran_settings(sc, ccb); break; case XPT_RESET_BUS: if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "got XPT_RESET_BUS, ACK it...\n"); ccb->ccb_h.status = CAM_REQ_CMP; break; case XPT_MMC_IO: /* * Here is the HW-dependent part of sending * the command to the underlying h/w. * At some point in the future an interrupt comes * and the request will be marked as completed. */ ccb->ccb_h.status = CAM_REQ_INPROG; rtsx_cam_request(sc, ccb); return; default: ccb->ccb_h.status = CAM_REQ_INVALID; break; } xpt_done(ccb); return; } static void rtsx_cam_poll(struct cam_sim *sim) { return; } /* * Apply settings and set ccb->ccb_h.status accordingly. */ static void rtsx_cam_set_tran_settings(struct rtsx_softc *sc, union ccb *ccb) { struct mmc_ios *ios; struct mmc_ios *new_ios; struct ccb_trans_settings_mmc *cts; ios = &sc->rtsx_host.ios; cts = &ccb->cts.proto_specific.mmc; new_ios = &cts->ios; /* Update only requested fields */ if (cts->ios_valid & MMC_CLK) { ios->clock = new_ios->clock; sc->rtsx_ios_clock = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - clock: %u\n", ios->clock); } if (cts->ios_valid & MMC_VDD) { ios->vdd = new_ios->vdd; if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - vdd: %d\n", ios->vdd); } if (cts->ios_valid & MMC_CS) { ios->chip_select = new_ios->chip_select; if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - chip_select: %d\n", ios->chip_select); } if (cts->ios_valid & MMC_BW) { ios->bus_width = new_ios->bus_width; sc->rtsx_ios_bus_width = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - bus width: %d\n", ios->bus_width); } if (cts->ios_valid & MMC_PM) { ios->power_mode = new_ios->power_mode; sc->rtsx_ios_power_mode = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - power mode: %d\n", ios->power_mode); } if (cts->ios_valid & MMC_BT) { ios->timing = new_ios->timing; sc->rtsx_ios_timing = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - timing: %d\n", ios->timing); } if (cts->ios_valid & MMC_BM) { ios->bus_mode = new_ios->bus_mode; if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - bus mode: %d\n", ios->bus_mode); } #if __FreeBSD__ > 12 if (cts->ios_valid & MMC_VCCQ) { ios->vccq = new_ios->vccq; sc->rtsx_ios_vccq = -1; /* To be updated by rtsx_mmcbr_update_ios(). */ if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - vccq: %d\n", ios->vccq); } #endif if (rtsx_mmcbr_update_ios(sc->rtsx_dev, NULL) == 0) ccb->ccb_h.status = CAM_REQ_CMP; else ccb->ccb_h.status = CAM_REQ_CMP_ERR; return; } /* * Build a request and run it. */ static void rtsx_cam_request(struct rtsx_softc *sc, union ccb *ccb) { RTSX_LOCK(sc); if (sc->rtsx_ccb != NULL) { RTSX_UNLOCK(sc); ccb->ccb_h.status = CAM_BUSY; /* i.e. CAM_REQ_CMP | CAM_REQ_CMP_ERR */ return; } sc->rtsx_ccb = ccb; sc->rtsx_cam_req.cmd = &ccb->mmcio.cmd; sc->rtsx_cam_req.stop = &ccb->mmcio.stop; RTSX_UNLOCK(sc); rtsx_mmcbr_request(sc->rtsx_dev, NULL, &sc->rtsx_cam_req); return; } #endif /* MMCCAM */ static int rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct rtsx_softc *sc; sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: /* ivar 0 - 1 = opendrain, 2 = pushpull */ *result = sc->rtsx_host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: /* ivar 1 - 0 = 1b 2 = 4b, 3 = 8b */ *result = sc->rtsx_host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: /* ivar 2 - O = dontcare, 1 = cs_high, 2 = cs_low */ *result = sc->rtsx_host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: /* ivar 3 - clock in Hz */ *result = sc->rtsx_host.ios.clock; break; case MMCBR_IVAR_F_MIN: /* ivar 4 */ *result = sc->rtsx_host.f_min; break; case MMCBR_IVAR_F_MAX: /* ivar 5 */ *result = sc->rtsx_host.f_max; break; case MMCBR_IVAR_HOST_OCR: /* ivar 6 - host operation conditions register */ *result = sc->rtsx_host.host_ocr; break; case MMCBR_IVAR_MODE: /* ivar 7 - 0 = mode_mmc, 1 = mode_sd */ *result = sc->rtsx_host.mode; break; case MMCBR_IVAR_OCR: /* ivar 8 - operation conditions register */ *result = sc->rtsx_host.ocr; break; case MMCBR_IVAR_POWER_MODE: /* ivar 9 - 0 = off, 1 = up, 2 = on */ *result = sc->rtsx_host.ios.power_mode; break; case MMCBR_IVAR_VDD: /* ivar 11 - voltage power pin */ *result = sc->rtsx_host.ios.vdd; break; case MMCBR_IVAR_VCCQ: /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */ *result = sc->rtsx_host.ios.vccq; break; case MMCBR_IVAR_CAPS: /* ivar 13 */ *result = sc->rtsx_host.caps; break; case MMCBR_IVAR_TIMING: /* ivar 14 - 0 = normal, 1 = timing_hs, ... */ *result = sc->rtsx_host.ios.timing; break; case MMCBR_IVAR_MAX_DATA: /* ivar 15 */ *result = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE; break; case MMCBR_IVAR_RETUNE_REQ: /* ivar 10 */ case MMCBR_IVAR_MAX_BUSY_TIMEOUT: /* ivar 16 */ default: return (EINVAL); } if (bootverbose) device_printf(bus, "Read ivar #%d, value %#x / #%d\n", which, *(int *)result, *(int *)result); return (0); } static int rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct rtsx_softc *sc; if (bootverbose) device_printf(bus, "Write ivar #%d, value %#x / #%d\n", which, (int)value, (int)value); sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: /* ivar 0 - 1 = opendrain, 2 = pushpull */ sc->rtsx_host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: /* ivar 1 - 0 = 1b 2 = 4b, 3 = 8b */ sc->rtsx_host.ios.bus_width = value; sc->rtsx_ios_bus_width = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; case MMCBR_IVAR_CHIP_SELECT: /* ivar 2 - O = dontcare, 1 = cs_high, 2 = cs_low */ sc->rtsx_host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: /* ivar 3 - clock in Hz */ sc->rtsx_host.ios.clock = value; sc->rtsx_ios_clock = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; case MMCBR_IVAR_MODE: /* ivar 7 - 0 = mode_mmc, 1 = mode_sd */ sc->rtsx_host.mode = value; break; case MMCBR_IVAR_OCR: /* ivar 8 - operation conditions register */ sc->rtsx_host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: /* ivar 9 - 0 = off, 1 = up, 2 = on */ sc->rtsx_host.ios.power_mode = value; sc->rtsx_ios_power_mode = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; case MMCBR_IVAR_VDD: /* ivar 11 - voltage power pin */ sc->rtsx_host.ios.vdd = value; break; case MMCBR_IVAR_VCCQ: /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */ sc->rtsx_host.ios.vccq = value; sc->rtsx_ios_vccq = value; /* rtsx_mmcbr_switch_vccq() will be called by mmc.c (MMCCAM undef). */ break; case MMCBR_IVAR_TIMING: /* ivar 14 - 0 = normal, 1 = timing_hs, ... */ sc->rtsx_host.ios.timing = value; sc->rtsx_ios_timing = -1; /* To be updated on next rtsx_mmcbr_update_ios(). */ break; /* These are read-only. */ case MMCBR_IVAR_F_MIN: /* ivar 4 */ case MMCBR_IVAR_F_MAX: /* ivar 5 */ case MMCBR_IVAR_HOST_OCR: /* ivar 6 - host operation conditions register */ case MMCBR_IVAR_RETUNE_REQ: /* ivar 10 */ case MMCBR_IVAR_CAPS: /* ivar 13 */ case MMCBR_IVAR_MAX_DATA: /* ivar 15 */ case MMCBR_IVAR_MAX_BUSY_TIMEOUT: /* ivar 16 */ default: return (EINVAL); } return (0); } static int rtsx_mmcbr_update_ios(device_t bus, device_t child__unused) { struct rtsx_softc *sc; struct mmc_ios *ios; int error; sc = device_get_softc(bus); ios = &sc->rtsx_host.ios; if (bootverbose) device_printf(bus, "rtsx_mmcbr_update_ios()\n"); /* if MMCBR_IVAR_BUS_WIDTH updated. */ if (sc->rtsx_ios_bus_width < 0) { sc->rtsx_ios_bus_width = ios->bus_width; if ((error = rtsx_set_bus_width(sc, ios->bus_width))) return (error); } /* if MMCBR_IVAR_POWER_MODE updated. */ if (sc->rtsx_ios_power_mode < 0) { sc->rtsx_ios_power_mode = ios->power_mode; switch (ios->power_mode) { case power_off: if ((error = rtsx_bus_power_off(sc))) return (error); break; case power_up: if ((error = rtsx_bus_power_on(sc))) return (error); break; case power_on: if ((error = rtsx_bus_power_on(sc))) return (error); break; } } sc->rtsx_double_clk = true; sc->rtsx_vpclk = false; /* if MMCBR_IVAR_TIMING updated. */ if (sc->rtsx_ios_timing < 0) { sc->rtsx_ios_timing = ios->timing; if ((error = rtsx_set_sd_timing(sc, ios->timing))) return (error); } /* if MMCBR_IVAR_CLOCK updated, must be after rtsx_set_sd_timing() */ if (sc->rtsx_ios_clock < 0) { sc->rtsx_ios_clock = ios->clock; if ((error = rtsx_set_sd_clock(sc, ios->clock))) return (error); } /* if MMCCAM and vccq updated */ if (sc->rtsx_ios_vccq < 0) { sc->rtsx_ios_vccq = ios->vccq; if ((error = rtsx_mmcbr_switch_vccq(sc->rtsx_dev, NULL))) return (error); } return (0); } /* * Set output stage logic power voltage. */ static int rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused) { struct rtsx_softc *sc; int vccq = 0; int error; sc = device_get_softc(bus); switch (sc->rtsx_host.ios.vccq) { case vccq_120: vccq = 120; break; case vccq_180: vccq = 180; break; case vccq_330: vccq = 330; break; }; /* It seems it is always vccq_330. */ if (vccq == 330) { switch (sc->rtsx_device_id) { uint16_t val; case RTSX_RTS5227: if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4))) return (error); if ((error = rtsx_rts5227_fill_driving(sc))) return (error); break; case RTSX_RTS5209: case RTSX_RTS5229: RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3); if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4))) return (error); break; case RTSX_RTS522A: if ((error = rtsx_write_phy(sc, 0x08, 0x57E4))) return (error); if ((error = rtsx_rts5227_fill_driving(sc))) return (error); break; case RTSX_RTS525A: RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_LDO_D3318_MASK, RTSX_LDO_D3318_33V); RTSX_BITOP(sc, RTSX_SD_PAD_CTL, RTSX_SD_IO_USING_1V8, 0); if ((error = rtsx_rts5249_fill_driving(sc))) return (error); break; case RTSX_RTS5249: if ((error = rtsx_read_phy(sc, RTSX_PHY_TUNE, &val))) return (error); if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE, (val & RTSX_PHY_TUNE_VOLTAGE_MASK) | RTSX_PHY_TUNE_VOLTAGE_3V3))) return (error); if ((error = rtsx_rts5249_fill_driving(sc))) return (error); break; case RTSX_RTL8402: RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3); RTSX_BITOP(sc, RTSX_LDO_CTL, (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_MASK, (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_3V3); break; case RTSX_RTL8411: case RTSX_RTL8411B: RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3); RTSX_BITOP(sc, RTSX_LDO_CTL, (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_MASK, (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_3V3); break; } DELAY(300); } if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_mmcbr_switch_vccq(%d)\n", vccq); return (0); } /* * Tune card if bus_timing_uhs_sdr50. */ static int rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400) { struct rtsx_softc *sc; uint32_t raw_phase_map[RTSX_RX_TUNING_CNT] = {0}; uint32_t phase_map; uint8_t final_phase; int i; sc = device_get_softc(bus); if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - hs400 is %s\n", (hs400) ? "true" : "false"); if (sc->rtsx_ios_timing != bus_timing_uhs_sdr50) return (0); sc->rtsx_tuning_mode = true; switch (sc->rtsx_device_id) { case RTSX_RTS5209: case RTSX_RTS5227: rtsx_sd_change_tx_phase(sc, 27); break; case RTSX_RTS522A: rtsx_sd_change_tx_phase(sc, 20); break; case RTSX_RTS5229: rtsx_sd_change_tx_phase(sc, 27); break; case RTSX_RTS525A: case RTSX_RTS5249: rtsx_sd_change_tx_phase(sc, 29); break; case RTSX_RTL8402: case RTSX_RTL8411: case RTSX_RTL8411B: rtsx_sd_change_tx_phase(sc, 7); break; } /* trying rx tuning for bus_timing_uhs_sdr50. */ for (i = 0; i < RTSX_RX_TUNING_CNT; i++) { rtsx_sd_tuning_rx_phase(sc, &(raw_phase_map[i])); if (raw_phase_map[i] == 0) break; } phase_map = 0xffffffff; for (i = 0; i < RTSX_RX_TUNING_CNT; i++) { if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX raw_phase_map[%d]: 0x%08x\n", i, raw_phase_map[i]); phase_map &= raw_phase_map[i]; } if (bootverbose || sc->rtsx_debug) device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX phase_map: 0x%08x\n", phase_map); if (phase_map) { final_phase = rtsx_sd_search_final_rx_phase(sc, phase_map); if (final_phase != 0xff) { if (sc->rtsx_debug == 1) { sc->rtsx_debug = 2; rtsx_sd_change_rx_phase(sc, final_phase); sc->rtsx_debug = 1; } else { rtsx_sd_change_rx_phase(sc, final_phase); } } } sc->rtsx_tuning_mode = false; return (0); } static int rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused) { struct rtsx_softc *sc; sc = device_get_softc(bus); if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_mmcbr_retune()\n"); return (0); } static int rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req) { struct rtsx_softc *sc; struct mmc_command *cmd; int error; sc = device_get_softc(bus); RTSX_LOCK(sc); if (sc->rtsx_req != NULL) { RTSX_UNLOCK(sc); return (EBUSY); } sc->rtsx_req = req; cmd = req->cmd; cmd->error = error = MMC_ERR_NONE; sc->rtsx_intr_status = 0; sc->rtsx_intr_trans_ok = NULL; sc->rtsx_intr_trans_ko = rtsx_req_done; if (bootverbose) device_printf(sc->rtsx_dev, "rtsx_mmcbr_request(CMD%u arg %#x, flags %#x, dlen %u, dflags %#x)\n", cmd->opcode, cmd->arg, cmd->flags, cmd->data != NULL ? (unsigned int)cmd->data->len : 0, cmd->data != NULL ? cmd->data->flags : 0); /* Check if card present. */ if (!ISSET(sc->rtsx_flags, RTSX_F_CARD_PRESENT)) { cmd->error = error = MMC_ERR_FAILED; goto end; } /* Refuse SDIO probe if the chip doesn't support SDIO. */ if (cmd->opcode == IO_SEND_OP_COND && !ISSET(sc->rtsx_flags, RTSX_F_SDIO_SUPPORT)) { cmd->error = error = MMC_ERR_INVALID; goto end; } /* Return MMC_ERR_TIMEOUT for SD_IO_RW_DIRECT and IO_SEND_OP_COND. */ if (cmd->opcode == SD_IO_RW_DIRECT || cmd->opcode == IO_SEND_OP_COND) { cmd->error = error = MMC_ERR_TIMEOUT; goto end; } /* Select SD card. */ RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL); RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD); if (cmd->data == NULL) { DELAY(200); error = rtsx_send_req(sc, cmd); } else if (cmd->data->len <= 512) { error = rtsx_xfer_short(sc, cmd); } else { error = rtsx_xfer(sc, cmd); } end: if (error == MMC_ERR_NONE) { callout_reset(&sc->rtsx_timeout_callout, sc->rtsx_timeout * hz, rtsx_timeout, sc); } else { rtsx_req_done(sc); } RTSX_UNLOCK(sc); return (error); } static int rtsx_mmcbr_get_ro(device_t bus, device_t child __unused) { struct rtsx_softc *sc; sc = device_get_softc(bus); if (sc->rtsx_inversion == 0) return (sc->rtsx_read_only); else return !(sc->rtsx_read_only); } static int rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused) { struct rtsx_softc *sc; if (bootverbose) device_printf(bus, "rtsx_mmcbr_acquire_host()\n"); sc = device_get_softc(bus); RTSX_LOCK(sc); while (sc->rtsx_bus_busy) msleep(&sc->rtsx_bus_busy, &sc->rtsx_mtx, 0, "rtsxah", 0); sc->rtsx_bus_busy++; RTSX_UNLOCK(sc); return (0); } static int rtsx_mmcbr_release_host(device_t bus, device_t child __unused) { struct rtsx_softc *sc; if (bootverbose) device_printf(bus, "rtsx_mmcbr_release_host()\n"); sc = device_get_softc(bus); RTSX_LOCK(sc); sc->rtsx_bus_busy--; wakeup(&sc->rtsx_bus_busy); RTSX_UNLOCK(sc); return (0); } /* * * PCI Support Functions * */ /* * Compare the device ID (chip) of this device against the IDs that this driver * supports. If there is a match, set the description and return success. */ static int rtsx_probe(device_t dev) { struct rtsx_softc *sc; uint16_t vendor_id; uint16_t device_id; int i; int result; vendor_id = pci_get_vendor(dev); device_id = pci_get_device(dev); result = ENXIO; for (i = 0; rtsx_devices[i].vendor_id != 0; i++) { if (rtsx_devices[i].vendor_id == vendor_id && rtsx_devices[i].device_id == device_id) { device_set_desc(dev, rtsx_devices[i].desc); sc = device_get_softc(dev); sc->rtsx_device_id = device_id; result = BUS_PROBE_DEFAULT; break; } } return (result); } /* * Attach function is only called if the probe is successful. */ static int rtsx_attach(device_t dev) { struct rtsx_softc *sc = device_get_softc(dev); struct sysctl_ctx_list *ctx; struct sysctl_oid_list *tree; int msi_count = 1; uint32_t sdio_cfg; int error; if (bootverbose) device_printf(dev, "Attach - Vendor ID: 0x%x - Device ID: 0x%x\n", pci_get_vendor(dev), pci_get_device(dev)); sc->rtsx_dev = dev; sc->rtsx_req = NULL; sc->rtsx_timeout = 10; sc->rtsx_read_only = 0; sc->rtsx_force_timing = 0; sc->rtsx_debug = 0; sc->rtsx_read_count = 0; sc->rtsx_write_count = 0; RTSX_LOCK_INIT(sc); ctx = device_get_sysctl_ctx(dev); tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); SYSCTL_ADD_INT(ctx, tree, OID_AUTO, "req_timeout", CTLFLAG_RW, &sc->rtsx_timeout, 0, "Request timeout in seconds"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "read_only", CTLFLAG_RD, &sc->rtsx_read_only, 0, "Card is write protected"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "inversion", CTLFLAG_RWTUN, &sc->rtsx_inversion, 0, "Inversion of card detection and read only status"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "force_timing", CTLFLAG_RW, &sc->rtsx_force_timing, 0, "Force bus_timing_uhs_sdr50"); SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "debug", CTLFLAG_RW, &sc->rtsx_debug, 0, "Debugging flag"); SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "read_count", CTLFLAG_RD, &sc->rtsx_read_count, 0, "Count of read operations"); SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "write_count", CTLFLAG_RD, &sc->rtsx_write_count, 0, "Count of write operations"); /* Allocate IRQ. */ sc->rtsx_irq_res_id = 0; if (pci_alloc_msi(dev, &msi_count) == 0) sc->rtsx_irq_res_id = 1; sc->rtsx_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->rtsx_irq_res_id, RF_ACTIVE | (sc->rtsx_irq_res_id != 0 ? 0 : RF_SHAREABLE)); if (sc->rtsx_irq_res == NULL) { device_printf(dev, "Can't allocate IRQ resources for %d\n", sc->rtsx_irq_res_id); pci_release_msi(dev); return (ENXIO); } callout_init_mtx(&sc->rtsx_timeout_callout, &sc->rtsx_mtx, 0); /* Allocate memory resource. */ if (sc->rtsx_device_id == RTSX_RTS525A) sc->rtsx_res_id = PCIR_BAR(1); else sc->rtsx_res_id = PCIR_BAR(0); sc->rtsx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->rtsx_res_id, RF_ACTIVE); if (sc->rtsx_res == NULL) { device_printf(dev, "Can't allocate memory resource for %d\n", sc->rtsx_res_id); goto destroy_rtsx_irq_res; } if (bootverbose) device_printf(dev, "rtsx_irq_res_id: %d, rtsx_res_id: %d\n", sc->rtsx_irq_res_id, sc->rtsx_res_id); sc->rtsx_btag = rman_get_bustag(sc->rtsx_res); sc->rtsx_bhandle = rman_get_bushandle(sc->rtsx_res); /* Activate the interrupt. */ error = bus_setup_intr(dev, sc->rtsx_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, rtsx_intr, sc, &sc->rtsx_irq_cookie); if (error) { device_printf(dev, "Can't set up irq [0x%x]!\n", error); goto destroy_rtsx_res; } pci_enable_busmaster(dev); if (rtsx_read_cfg(sc, 0, RTSX_SDIOCFG_REG, &sdio_cfg) == 0) { if ((sdio_cfg & RTSX_SDIOCFG_SDIO_ONLY) || (sdio_cfg & RTSX_SDIOCFG_HAVE_SDIO)) sc->rtsx_flags |= RTSX_F_SDIO_SUPPORT; } /* Allocate two DMA buffers: a command buffer and a data buffer. */ error = rtsx_dma_alloc(sc); if (error) { goto destroy_rtsx_irq; } /* From dwmmc.c. */ TIMEOUT_TASK_INIT(taskqueue_swi_giant, &sc->rtsx_card_insert_task, 0, rtsx_card_task, sc); TASK_INIT(&sc->rtsx_card_remove_task, 0, rtsx_card_task, sc); #ifdef MMCCAM sc->rtsx_ccb = NULL; sc->rtsx_cam_status = 0; SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "cam_status", CTLFLAG_RD, &sc->rtsx_cam_status, 0, "driver cam card present"); if ((sc->rtsx_devq = cam_simq_alloc(1)) == NULL) { device_printf(dev, "Error during CAM queue allocation\n"); goto destroy_rtsx_irq; } mtx_init(&sc->rtsx_sim_mtx, "rtsxsim", NULL, MTX_DEF); sc->rtsx_sim = cam_sim_alloc(rtsx_cam_action, rtsx_cam_poll, "rtsx", sc, device_get_unit(dev), &sc->rtsx_sim_mtx, 1, 1, sc->rtsx_devq); if (sc->rtsx_sim == NULL) { device_printf(dev, "Can't allocate CAM SIM\n"); goto destroy_rtsx_irq; } mtx_lock(&sc->rtsx_sim_mtx); if (xpt_bus_register(sc->rtsx_sim, dev, 0) != 0) { device_printf(dev, "Can't register SCSI pass-through bus\n"); mtx_unlock(&sc->rtsx_sim_mtx); goto destroy_rtsx_irq; } mtx_unlock(&sc->rtsx_sim_mtx); #endif /* MMCCAM */ /* Initialize device. */ if (rtsx_init(sc)) { device_printf(dev, "Error during rtsx_init()\n"); goto destroy_rtsx_irq; } /* * Schedule a card detection as we won't get an interrupt * if the card is inserted when we attach */ DELAY(500); if (rtsx_is_card_present(sc)) device_printf(sc->rtsx_dev, "Card present\n"); else device_printf(sc->rtsx_dev, "Card absent\n"); rtsx_card_task(sc, 0); if (bootverbose) device_printf(dev, "Device attached\n"); return (0); destroy_rtsx_irq: bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie); destroy_rtsx_res: bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_res_id, sc->rtsx_res); destroy_rtsx_irq_res: callout_drain(&sc->rtsx_timeout_callout); bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id, sc->rtsx_irq_res); pci_release_msi(dev); RTSX_LOCK_DESTROY(sc); #ifdef MMCCAM if (sc->rtsx_sim != NULL) { mtx_lock(&sc->rtsx_sim_mtx); xpt_bus_deregister(cam_sim_path(sc->rtsx_sim)); cam_sim_free(sc->rtsx_sim, FALSE); mtx_unlock(&sc->rtsx_sim_mtx); } if (sc->rtsx_devq != NULL) { mtx_destroy(&sc->rtsx_sim_mtx); cam_simq_free(sc->rtsx_devq); } #endif /* MMCCAM */ return (ENXIO); } static int rtsx_detach(device_t dev) { struct rtsx_softc *sc = device_get_softc(dev); int error; if (bootverbose) device_printf(dev, "Detach - Vendor ID: 0x%x - Device ID: 0x%x\n", pci_get_vendor(dev), pci_get_device(dev)); /* Disable interrupts. */ sc->rtsx_intr_enabled = 0; WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled); /* Stop device. */ error = device_delete_children(sc->rtsx_dev); sc->rtsx_mmc_dev = NULL; if (error) return (error); taskqueue_drain_timeout(taskqueue_swi_giant, &sc->rtsx_card_insert_task); taskqueue_drain(taskqueue_swi_giant, &sc->rtsx_card_remove_task); /* Teardown the state in our softc created in our attach routine. */ rtsx_dma_free(sc); if (sc->rtsx_res != NULL) bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_res_id, sc->rtsx_res); if (sc->rtsx_irq_cookie != NULL) bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie); if (sc->rtsx_irq_res != NULL) { callout_drain(&sc->rtsx_timeout_callout); bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id, sc->rtsx_irq_res); pci_release_msi(dev); } RTSX_LOCK_DESTROY(sc); #ifdef MMCCAM if (sc->rtsx_sim != NULL) { mtx_lock(&sc->rtsx_sim_mtx); xpt_bus_deregister(cam_sim_path(sc->rtsx_sim)); cam_sim_free(sc->rtsx_sim, FALSE); mtx_unlock(&sc->rtsx_sim_mtx); } if (sc->rtsx_devq != NULL) { mtx_destroy(&sc->rtsx_sim_mtx); cam_simq_free(sc->rtsx_devq); } #endif /* MMCCAM */ return (0); } static int rtsx_shutdown(device_t dev) { if (bootverbose) device_printf(dev, "Shutdown\n"); rtsx_detach(dev); return (0); } /* * Device suspend routine. */ static int rtsx_suspend(device_t dev) { struct rtsx_softc *sc = device_get_softc(dev); device_printf(dev, "Suspend\n"); #ifdef MMCCAM if (sc->rtsx_ccb != NULL) { device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n", sc->rtsx_ccb->mmcio.cmd.opcode, sc->rtsx_intr_status); } #else if (sc->rtsx_req != NULL) { device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n", sc->rtsx_req->cmd->opcode, sc->rtsx_intr_status); } #endif /* MMCCAM */ bus_generic_suspend(dev); return (0); } /* * Device resume routine. */ static int rtsx_resume(device_t dev) { device_printf(dev, "Resume\n"); bus_generic_resume(dev); return (0); } static device_method_t rtsx_methods[] = { /* Device interface */ DEVMETHOD(device_probe, rtsx_probe), DEVMETHOD(device_attach, rtsx_attach), DEVMETHOD(device_detach, rtsx_detach), DEVMETHOD(device_shutdown, rtsx_shutdown), DEVMETHOD(device_suspend, rtsx_suspend), DEVMETHOD(device_resume, rtsx_resume), /* Bus interface */ DEVMETHOD(bus_read_ivar, rtsx_read_ivar), DEVMETHOD(bus_write_ivar, rtsx_write_ivar), /* MMC bridge interface */ DEVMETHOD(mmcbr_update_ios, rtsx_mmcbr_update_ios), DEVMETHOD(mmcbr_switch_vccq, rtsx_mmcbr_switch_vccq), DEVMETHOD(mmcbr_tune, rtsx_mmcbr_tune), DEVMETHOD(mmcbr_retune, rtsx_mmcbr_retune), DEVMETHOD(mmcbr_request, rtsx_mmcbr_request), DEVMETHOD(mmcbr_get_ro, rtsx_mmcbr_get_ro), DEVMETHOD(mmcbr_acquire_host, rtsx_mmcbr_acquire_host), DEVMETHOD(mmcbr_release_host, rtsx_mmcbr_release_host), DEVMETHOD_END }; static devclass_t rtsx_devclass; DEFINE_CLASS_0(rtsx, rtsx_driver, rtsx_methods, sizeof(struct rtsx_softc)); DRIVER_MODULE(rtsx, pci, rtsx_driver, rtsx_devclass, NULL, NULL); #ifndef MMCCAM MMC_DECLARE_BRIDGE(rtsx); #endif /* MMCCAM */ diff --git a/sys/dev/sfxge/sfxge.c b/sys/dev/sfxge/sfxge.c index 866f940a7b29..62daadf35ae5 100644 --- a/sys/dev/sfxge/sfxge.c +++ b/sys/dev/sfxge/sfxge.c @@ -1,1211 +1,1211 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010-2016 Solarflare Communications Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "opt_rss.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef RSS #include #endif #include "common/efx.h" #include "sfxge.h" #include "sfxge_rx.h" #include "sfxge_ioc.h" #include "sfxge_version.h" #define SFXGE_CAP (IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM | \ IFCAP_RXCSUM | IFCAP_TXCSUM | \ IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6 | \ IFCAP_TSO4 | IFCAP_TSO6 | \ IFCAP_JUMBO_MTU | \ IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWSTATS) #define SFXGE_CAP_ENABLE SFXGE_CAP #define SFXGE_CAP_FIXED (IFCAP_VLAN_MTU | \ IFCAP_JUMBO_MTU | IFCAP_LINKSTATE | IFCAP_HWSTATS) MALLOC_DEFINE(M_SFXGE, "sfxge", "Solarflare 10GigE driver"); SYSCTL_NODE(_hw, OID_AUTO, sfxge, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "SFXGE driver parameters"); #define SFXGE_PARAM_RX_RING SFXGE_PARAM(rx_ring) static int sfxge_rx_ring_entries = SFXGE_NDESCS; TUNABLE_INT(SFXGE_PARAM_RX_RING, &sfxge_rx_ring_entries); SYSCTL_INT(_hw_sfxge, OID_AUTO, rx_ring, CTLFLAG_RDTUN, &sfxge_rx_ring_entries, 0, "Maximum number of descriptors in a receive ring"); #define SFXGE_PARAM_TX_RING SFXGE_PARAM(tx_ring) static int sfxge_tx_ring_entries = SFXGE_NDESCS; TUNABLE_INT(SFXGE_PARAM_TX_RING, &sfxge_tx_ring_entries); SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_ring, CTLFLAG_RDTUN, &sfxge_tx_ring_entries, 0, "Maximum number of descriptors in a transmit ring"); #define SFXGE_PARAM_RESTART_ATTEMPTS SFXGE_PARAM(restart_attempts) static int sfxge_restart_attempts = 3; TUNABLE_INT(SFXGE_PARAM_RESTART_ATTEMPTS, &sfxge_restart_attempts); SYSCTL_INT(_hw_sfxge, OID_AUTO, restart_attempts, CTLFLAG_RDTUN, &sfxge_restart_attempts, 0, "Maximum number of attempts to bring interface up after reset"); #if EFSYS_OPT_MCDI_LOGGING #define SFXGE_PARAM_MCDI_LOGGING SFXGE_PARAM(mcdi_logging) static int sfxge_mcdi_logging = 0; TUNABLE_INT(SFXGE_PARAM_MCDI_LOGGING, &sfxge_mcdi_logging); #endif static void sfxge_reset(void *arg, int npending); static int sfxge_estimate_rsrc_limits(struct sfxge_softc *sc) { efx_drv_limits_t limits; int rc; unsigned int evq_max; uint32_t evq_allocated; uint32_t rxq_allocated; uint32_t txq_allocated; /* * Limit the number of event queues to: * - number of CPUs * - hardwire maximum RSS channels * - administratively specified maximum RSS channels */ #ifdef RSS /* * Avoid extra limitations so that the number of queues * may be configured at administrator's will */ evq_max = MIN(MAX(rss_getnumbuckets(), 1), EFX_MAXRSS); #else evq_max = MIN(mp_ncpus, EFX_MAXRSS); #endif if (sc->max_rss_channels > 0) evq_max = MIN(evq_max, sc->max_rss_channels); memset(&limits, 0, sizeof(limits)); limits.edl_min_evq_count = 1; limits.edl_max_evq_count = evq_max; limits.edl_min_txq_count = SFXGE_EVQ0_N_TXQ(sc); limits.edl_max_txq_count = evq_max + SFXGE_EVQ0_N_TXQ(sc) - 1; limits.edl_min_rxq_count = 1; limits.edl_max_rxq_count = evq_max; efx_nic_set_drv_limits(sc->enp, &limits); if ((rc = efx_nic_init(sc->enp)) != 0) return (rc); rc = efx_nic_get_vi_pool(sc->enp, &evq_allocated, &rxq_allocated, &txq_allocated); if (rc != 0) { efx_nic_fini(sc->enp); return (rc); } KASSERT(txq_allocated >= SFXGE_EVQ0_N_TXQ(sc), ("txq_allocated < %u", SFXGE_EVQ0_N_TXQ(sc))); sc->evq_max = MIN(evq_allocated, evq_max); sc->evq_max = MIN(rxq_allocated, sc->evq_max); sc->evq_max = MIN(txq_allocated - (SFXGE_EVQ0_N_TXQ(sc) - 1), sc->evq_max); KASSERT(sc->evq_max <= evq_max, ("allocated more than maximum requested")); #ifdef RSS if (sc->evq_max < rss_getnumbuckets()) device_printf(sc->dev, "The number of allocated queues (%u) " "is less than the number of RSS buckets (%u); " "performance degradation might be observed", sc->evq_max, rss_getnumbuckets()); #endif /* * NIC is kept initialized in the case of success to be able to * initialize port to find out media types. */ return (0); } static int sfxge_set_drv_limits(struct sfxge_softc *sc) { efx_drv_limits_t limits; memset(&limits, 0, sizeof(limits)); /* Limits are strict since take into account initial estimation */ limits.edl_min_evq_count = limits.edl_max_evq_count = sc->intr.n_alloc; limits.edl_min_txq_count = limits.edl_max_txq_count = sc->intr.n_alloc + SFXGE_EVQ0_N_TXQ(sc) - 1; limits.edl_min_rxq_count = limits.edl_max_rxq_count = sc->intr.n_alloc; return (efx_nic_set_drv_limits(sc->enp, &limits)); } static int sfxge_start(struct sfxge_softc *sc) { int rc; SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc); if (sc->init_state == SFXGE_STARTED) return (0); if (sc->init_state != SFXGE_REGISTERED) { rc = EINVAL; goto fail; } /* Set required resource limits */ if ((rc = sfxge_set_drv_limits(sc)) != 0) goto fail; if ((rc = efx_nic_init(sc->enp)) != 0) goto fail; /* Start processing interrupts. */ if ((rc = sfxge_intr_start(sc)) != 0) goto fail2; /* Start processing events. */ if ((rc = sfxge_ev_start(sc)) != 0) goto fail3; /* Fire up the port. */ if ((rc = sfxge_port_start(sc)) != 0) goto fail4; /* Start the receiver side. */ if ((rc = sfxge_rx_start(sc)) != 0) goto fail5; /* Start the transmitter side. */ if ((rc = sfxge_tx_start(sc)) != 0) goto fail6; sc->init_state = SFXGE_STARTED; /* Tell the stack we're running. */ sc->ifnet->if_drv_flags |= IFF_DRV_RUNNING; sc->ifnet->if_drv_flags &= ~IFF_DRV_OACTIVE; return (0); fail6: sfxge_rx_stop(sc); fail5: sfxge_port_stop(sc); fail4: sfxge_ev_stop(sc); fail3: sfxge_intr_stop(sc); fail2: efx_nic_fini(sc->enp); fail: device_printf(sc->dev, "sfxge_start: %d\n", rc); return (rc); } static void sfxge_if_init(void *arg) { struct sfxge_softc *sc; sc = (struct sfxge_softc *)arg; SFXGE_ADAPTER_LOCK(sc); (void)sfxge_start(sc); SFXGE_ADAPTER_UNLOCK(sc); } static void sfxge_stop(struct sfxge_softc *sc) { SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc); if (sc->init_state != SFXGE_STARTED) return; sc->init_state = SFXGE_REGISTERED; /* Stop the transmitter. */ sfxge_tx_stop(sc); /* Stop the receiver. */ sfxge_rx_stop(sc); /* Stop the port. */ sfxge_port_stop(sc); /* Stop processing events. */ sfxge_ev_stop(sc); /* Stop processing interrupts. */ sfxge_intr_stop(sc); efx_nic_fini(sc->enp); sc->ifnet->if_drv_flags &= ~IFF_DRV_RUNNING; } static int sfxge_vpd_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ioc) { efx_vpd_value_t value; int rc = 0; switch (ioc->u.vpd.op) { case SFXGE_VPD_OP_GET_KEYWORD: value.evv_tag = ioc->u.vpd.tag; value.evv_keyword = ioc->u.vpd.keyword; rc = efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value); if (rc != 0) break; ioc->u.vpd.len = MIN(ioc->u.vpd.len, value.evv_length); if (ioc->u.vpd.payload != 0) { rc = copyout(value.evv_value, ioc->u.vpd.payload, ioc->u.vpd.len); } break; case SFXGE_VPD_OP_SET_KEYWORD: if (ioc->u.vpd.len > sizeof(value.evv_value)) return (EINVAL); value.evv_tag = ioc->u.vpd.tag; value.evv_keyword = ioc->u.vpd.keyword; value.evv_length = ioc->u.vpd.len; rc = copyin(ioc->u.vpd.payload, value.evv_value, value.evv_length); if (rc != 0) break; rc = efx_vpd_set(sc->enp, sc->vpd_data, sc->vpd_size, &value); if (rc != 0) break; rc = efx_vpd_verify(sc->enp, sc->vpd_data, sc->vpd_size); if (rc != 0) break; rc = efx_vpd_write(sc->enp, sc->vpd_data, sc->vpd_size); break; default: rc = EOPNOTSUPP; break; } return (rc); } static int sfxge_private_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ioc) { switch (ioc->op) { case SFXGE_MCDI_IOC: return (sfxge_mcdi_ioctl(sc, ioc)); case SFXGE_NVRAM_IOC: return (sfxge_nvram_ioctl(sc, ioc)); case SFXGE_VPD_IOC: return (sfxge_vpd_ioctl(sc, ioc)); default: return (EOPNOTSUPP); } } static int sfxge_if_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) { struct sfxge_softc *sc; struct ifreq *ifr; sfxge_ioc_t ioc; int error; ifr = (struct ifreq *)data; sc = ifp->if_softc; error = 0; switch (command) { case SIOCSIFFLAGS: SFXGE_ADAPTER_LOCK(sc); if (ifp->if_flags & IFF_UP) { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if ((ifp->if_flags ^ sc->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { sfxge_mac_filter_set(sc); } } else sfxge_start(sc); } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) sfxge_stop(sc); sc->if_flags = ifp->if_flags; SFXGE_ADAPTER_UNLOCK(sc); break; case SIOCSIFMTU: if (ifr->ifr_mtu == ifp->if_mtu) { /* Nothing to do */ error = 0; } else if (ifr->ifr_mtu > SFXGE_MAX_MTU) { error = EINVAL; } else if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { ifp->if_mtu = ifr->ifr_mtu; error = 0; } else { /* Restart required */ SFXGE_ADAPTER_LOCK(sc); sfxge_stop(sc); ifp->if_mtu = ifr->ifr_mtu; error = sfxge_start(sc); SFXGE_ADAPTER_UNLOCK(sc); if (error != 0) { ifp->if_flags &= ~IFF_UP; ifp->if_drv_flags &= ~IFF_DRV_RUNNING; if_down(ifp); } } break; case SIOCADDMULTI: case SIOCDELMULTI: if (ifp->if_drv_flags & IFF_DRV_RUNNING) sfxge_mac_filter_set(sc); break; case SIOCSIFCAP: { int reqcap = ifr->ifr_reqcap; int capchg_mask; SFXGE_ADAPTER_LOCK(sc); /* Capabilities to be changed in accordance with request */ capchg_mask = ifp->if_capenable ^ reqcap; /* * The networking core already rejects attempts to * enable capabilities we don't have. We still have * to reject attempts to disable capabilities that we * can't (yet) disable. */ KASSERT((reqcap & ~ifp->if_capabilities) == 0, ("Unsupported capabilities 0x%x requested 0x%x vs " "supported 0x%x", reqcap & ~ifp->if_capabilities, reqcap , ifp->if_capabilities)); if (capchg_mask & SFXGE_CAP_FIXED) { error = EINVAL; SFXGE_ADAPTER_UNLOCK(sc); break; } /* Check request before any changes */ if ((capchg_mask & IFCAP_TSO4) && (reqcap & (IFCAP_TSO4 | IFCAP_TXCSUM)) == IFCAP_TSO4) { error = EAGAIN; SFXGE_ADAPTER_UNLOCK(sc); if_printf(ifp, "enable txcsum before tso4\n"); break; } if ((capchg_mask & IFCAP_TSO6) && (reqcap & (IFCAP_TSO6 | IFCAP_TXCSUM_IPV6)) == IFCAP_TSO6) { error = EAGAIN; SFXGE_ADAPTER_UNLOCK(sc); if_printf(ifp, "enable txcsum6 before tso6\n"); break; } if (reqcap & IFCAP_TXCSUM) { ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); } else { ifp->if_hwassist &= ~(CSUM_IP | CSUM_TCP | CSUM_UDP); if (reqcap & IFCAP_TSO4) { reqcap &= ~IFCAP_TSO4; if_printf(ifp, "tso4 disabled due to -txcsum\n"); } } if (reqcap & IFCAP_TXCSUM_IPV6) { ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); } else { ifp->if_hwassist &= ~(CSUM_TCP_IPV6 | CSUM_UDP_IPV6); if (reqcap & IFCAP_TSO6) { reqcap &= ~IFCAP_TSO6; if_printf(ifp, "tso6 disabled due to -txcsum6\n"); } } /* * The kernel takes both IFCAP_TSOx and CSUM_TSO into * account before using TSO. So, we do not touch * checksum flags when IFCAP_TSOx is modified. * Note that CSUM_TSO is (CSUM_IP_TSO|CSUM_IP6_TSO), * but both bits are set in IPv4 and IPv6 mbufs. */ ifp->if_capenable = reqcap; SFXGE_ADAPTER_UNLOCK(sc); break; } case SIOCSIFMEDIA: case SIOCGIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &sc->media, command); break; #ifdef SIOCGI2C case SIOCGI2C: { struct ifi2creq i2c; error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); if (error != 0) break; if (i2c.len > sizeof(i2c.data)) { error = EINVAL; break; } SFXGE_ADAPTER_LOCK(sc); error = efx_phy_module_get_info(sc->enp, i2c.dev_addr, i2c.offset, i2c.len, &i2c.data[0]); SFXGE_ADAPTER_UNLOCK(sc); if (error == 0) error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); break; } #endif case SIOCGPRIVATE_0: error = priv_check(curthread, PRIV_DRIVER); if (error != 0) break; error = copyin(ifr_data_get_ptr(ifr), &ioc, sizeof(ioc)); if (error != 0) return (error); error = sfxge_private_ioctl(sc, &ioc); if (error == 0) { error = copyout(&ioc, ifr_data_get_ptr(ifr), sizeof(ioc)); } break; default: error = ether_ioctl(ifp, command, data); } return (error); } static void sfxge_ifnet_fini(struct ifnet *ifp) { struct sfxge_softc *sc = ifp->if_softc; SFXGE_ADAPTER_LOCK(sc); sfxge_stop(sc); SFXGE_ADAPTER_UNLOCK(sc); ifmedia_removeall(&sc->media); ether_ifdetach(ifp); if_free(ifp); } static int sfxge_ifnet_init(struct ifnet *ifp, struct sfxge_softc *sc) { const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp); device_t dev; int rc; dev = sc->dev; sc->ifnet = ifp; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_init = sfxge_if_init; ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = sfxge_if_ioctl; ifp->if_capabilities = SFXGE_CAP; ifp->if_capenable = SFXGE_CAP_ENABLE; ifp->if_hw_tsomax = SFXGE_TSO_MAX_SIZE; ifp->if_hw_tsomaxsegcount = SFXGE_TX_MAPPING_MAX_SEG; ifp->if_hw_tsomaxsegsize = PAGE_SIZE; #ifdef SFXGE_LRO ifp->if_capabilities |= IFCAP_LRO; ifp->if_capenable |= IFCAP_LRO; #endif if (encp->enc_hw_tx_insert_vlan_enabled) { ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; ifp->if_capenable |= IFCAP_VLAN_HWTAGGING; } ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | CSUM_TCP_IPV6 | CSUM_UDP_IPV6; ether_ifattach(ifp, encp->enc_mac_addr); ifp->if_transmit = sfxge_if_transmit; ifp->if_qflush = sfxge_if_qflush; ifp->if_get_counter = sfxge_get_counter; DBGPRINT(sc->dev, "ifmedia_init"); if ((rc = sfxge_port_ifmedia_init(sc)) != 0) goto fail; return (0); fail: ether_ifdetach(sc->ifnet); return (rc); } void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp) { KASSERT(sc->buffer_table_next + n <= efx_nic_cfg_get(sc->enp)->enc_buftbl_limit, ("buffer table full")); *idp = sc->buffer_table_next; sc->buffer_table_next += n; } static int sfxge_bar_init(struct sfxge_softc *sc) { efsys_bar_t *esbp = &sc->bar; esbp->esb_rid = PCIR_BAR(sc->mem_bar); if ((esbp->esb_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &esbp->esb_rid, RF_ACTIVE)) == NULL) { device_printf(sc->dev, "Cannot allocate BAR region %d\n", sc->mem_bar); return (ENXIO); } esbp->esb_tag = rman_get_bustag(esbp->esb_res); esbp->esb_handle = rman_get_bushandle(esbp->esb_res); SFXGE_BAR_LOCK_INIT(esbp, device_get_nameunit(sc->dev)); return (0); } static void sfxge_bar_fini(struct sfxge_softc *sc) { efsys_bar_t *esbp = &sc->bar; bus_release_resource(sc->dev, SYS_RES_MEMORY, esbp->esb_rid, esbp->esb_res); SFXGE_BAR_LOCK_DESTROY(esbp); } static int sfxge_create(struct sfxge_softc *sc) { device_t dev; efx_nic_t *enp; int error; char rss_param_name[sizeof(SFXGE_PARAM(%d.max_rss_channels))]; #if EFSYS_OPT_MCDI_LOGGING char mcdi_log_param_name[sizeof(SFXGE_PARAM(%d.mcdi_logging))]; #endif dev = sc->dev; SFXGE_ADAPTER_LOCK_INIT(sc, device_get_nameunit(sc->dev)); sc->max_rss_channels = 0; snprintf(rss_param_name, sizeof(rss_param_name), SFXGE_PARAM(%d.max_rss_channels), (int)device_get_unit(dev)); TUNABLE_INT_FETCH(rss_param_name, &sc->max_rss_channels); #if EFSYS_OPT_MCDI_LOGGING sc->mcdi_logging = sfxge_mcdi_logging; snprintf(mcdi_log_param_name, sizeof(mcdi_log_param_name), SFXGE_PARAM(%d.mcdi_logging), (int)device_get_unit(dev)); TUNABLE_INT_FETCH(mcdi_log_param_name, &sc->mcdi_logging); #endif sc->stats_node = SYSCTL_ADD_NODE(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics"); if (sc->stats_node == NULL) { error = ENOMEM; goto fail; } TASK_INIT(&sc->task_reset, 0, sfxge_reset, sc); (void) pci_enable_busmaster(dev); /* Initialize DMA mappings. */ DBGPRINT(sc->dev, "dma_init..."); if ((error = sfxge_dma_init(sc)) != 0) goto fail; error = efx_family(pci_get_vendor(dev), pci_get_device(dev), &sc->family, &sc->mem_bar); KASSERT(error == 0, ("Family should be filtered by sfxge_probe()")); /* Map the device registers. */ DBGPRINT(sc->dev, "bar_init..."); if ((error = sfxge_bar_init(sc)) != 0) goto fail; DBGPRINT(sc->dev, "nic_create..."); /* Create the common code nic object. */ SFXGE_EFSYS_LOCK_INIT(&sc->enp_lock, device_get_nameunit(sc->dev), "nic"); if ((error = efx_nic_create(sc->family, (efsys_identifier_t *)sc, &sc->bar, &sc->enp_lock, &enp)) != 0) goto fail3; sc->enp = enp; /* Initialize MCDI to talk to the microcontroller. */ DBGPRINT(sc->dev, "mcdi_init..."); if ((error = sfxge_mcdi_init(sc)) != 0) goto fail4; /* Probe the NIC and build the configuration data area. */ DBGPRINT(sc->dev, "nic_probe..."); if ((error = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE)) != 0) goto fail5; if (!ISP2(sfxge_rx_ring_entries) || (sfxge_rx_ring_entries < EFX_RXQ_MINNDESCS) || (sfxge_rx_ring_entries > EFX_RXQ_MAXNDESCS)) { log(LOG_ERR, "%s=%d must be power of 2 from %u to %u", SFXGE_PARAM_RX_RING, sfxge_rx_ring_entries, EFX_RXQ_MINNDESCS, EFX_RXQ_MAXNDESCS); error = EINVAL; goto fail_rx_ring_entries; } sc->rxq_entries = sfxge_rx_ring_entries; if (efx_nic_cfg_get(enp)->enc_features & EFX_FEATURE_TXQ_CKSUM_OP_DESC) sc->txq_dynamic_cksum_toggle_supported = B_TRUE; else sc->txq_dynamic_cksum_toggle_supported = B_FALSE; if (!ISP2(sfxge_tx_ring_entries) || (sfxge_tx_ring_entries < EFX_TXQ_MINNDESCS) || (sfxge_tx_ring_entries > efx_nic_cfg_get(enp)->enc_txq_max_ndescs)) { log(LOG_ERR, "%s=%d must be power of 2 from %u to %u", SFXGE_PARAM_TX_RING, sfxge_tx_ring_entries, EFX_TXQ_MINNDESCS, efx_nic_cfg_get(enp)->enc_txq_max_ndescs); error = EINVAL; goto fail_tx_ring_entries; } sc->txq_entries = sfxge_tx_ring_entries; SYSCTL_ADD_STRING(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "version", CTLFLAG_RD, SFXGE_VERSION_STRING, 0, "Driver version"); SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "phy_type", CTLFLAG_RD, NULL, efx_nic_cfg_get(enp)->enc_phy_type, "PHY type"); /* Initialize the NVRAM. */ DBGPRINT(sc->dev, "nvram_init..."); if ((error = efx_nvram_init(enp)) != 0) goto fail6; /* Initialize the VPD. */ DBGPRINT(sc->dev, "vpd_init..."); if ((error = efx_vpd_init(enp)) != 0) goto fail7; efx_mcdi_new_epoch(enp); /* Reset the NIC. */ DBGPRINT(sc->dev, "nic_reset..."); if ((error = efx_nic_reset(enp)) != 0) goto fail8; /* Initialize buffer table allocation. */ sc->buffer_table_next = 0; /* * Guarantee minimum and estimate maximum number of event queues * to take it into account when MSI-X interrupts are allocated. * It initializes NIC and keeps it initialized on success. */ if ((error = sfxge_estimate_rsrc_limits(sc)) != 0) goto fail8; /* Set up interrupts. */ DBGPRINT(sc->dev, "intr_init..."); if ((error = sfxge_intr_init(sc)) != 0) goto fail9; /* Initialize event processing state. */ DBGPRINT(sc->dev, "ev_init..."); if ((error = sfxge_ev_init(sc)) != 0) goto fail11; /* Initialize port state. */ DBGPRINT(sc->dev, "port_init..."); if ((error = sfxge_port_init(sc)) != 0) goto fail12; /* Initialize receive state. */ DBGPRINT(sc->dev, "rx_init..."); if ((error = sfxge_rx_init(sc)) != 0) goto fail13; /* Initialize transmit state. */ DBGPRINT(sc->dev, "tx_init..."); if ((error = sfxge_tx_init(sc)) != 0) goto fail14; sc->init_state = SFXGE_INITIALIZED; DBGPRINT(sc->dev, "success"); return (0); fail14: sfxge_rx_fini(sc); fail13: sfxge_port_fini(sc); fail12: sfxge_ev_fini(sc); fail11: sfxge_intr_fini(sc); fail9: efx_nic_fini(sc->enp); fail8: efx_vpd_fini(enp); fail7: efx_nvram_fini(enp); fail6: fail_tx_ring_entries: fail_rx_ring_entries: efx_nic_unprobe(enp); fail5: sfxge_mcdi_fini(sc); fail4: sc->enp = NULL; efx_nic_destroy(enp); SFXGE_EFSYS_LOCK_DESTROY(&sc->enp_lock); fail3: sfxge_bar_fini(sc); (void) pci_disable_busmaster(sc->dev); fail: DBGPRINT(sc->dev, "failed %d", error); sc->dev = NULL; SFXGE_ADAPTER_LOCK_DESTROY(sc); return (error); } static void sfxge_destroy(struct sfxge_softc *sc) { efx_nic_t *enp; /* Clean up transmit state. */ sfxge_tx_fini(sc); /* Clean up receive state. */ sfxge_rx_fini(sc); /* Clean up port state. */ sfxge_port_fini(sc); /* Clean up event processing state. */ sfxge_ev_fini(sc); /* Clean up interrupts. */ sfxge_intr_fini(sc); /* Tear down common code subsystems. */ efx_nic_reset(sc->enp); efx_vpd_fini(sc->enp); efx_nvram_fini(sc->enp); efx_nic_unprobe(sc->enp); /* Tear down MCDI. */ sfxge_mcdi_fini(sc); /* Destroy common code context. */ enp = sc->enp; sc->enp = NULL; efx_nic_destroy(enp); /* Free DMA memory. */ sfxge_dma_fini(sc); /* Free mapped BARs. */ sfxge_bar_fini(sc); (void) pci_disable_busmaster(sc->dev); taskqueue_drain(taskqueue_thread, &sc->task_reset); /* Destroy the softc lock. */ SFXGE_ADAPTER_LOCK_DESTROY(sc); } static int sfxge_vpd_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; efx_vpd_value_t value; int rc; value.evv_tag = arg2 >> 16; value.evv_keyword = arg2 & 0xffff; if ((rc = efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value)) != 0) return (rc); return (SYSCTL_OUT(req, value.evv_value, value.evv_length)); } static void sfxge_vpd_try_add(struct sfxge_softc *sc, struct sysctl_oid_list *list, efx_vpd_tag_t tag, const char *keyword) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); efx_vpd_value_t value; /* Check whether VPD tag/keyword is present */ value.evv_tag = tag; value.evv_keyword = EFX_VPD_KEYWORD(keyword[0], keyword[1]); if (efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value) != 0) return; SYSCTL_ADD_PROC(ctx, list, OID_AUTO, keyword, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, tag << 16 | EFX_VPD_KEYWORD(keyword[0], keyword[1]), sfxge_vpd_handler, "A", ""); } static int sfxge_vpd_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid *vpd_node; struct sysctl_oid_list *vpd_list; char keyword[3]; efx_vpd_value_t value; int rc; if ((rc = efx_vpd_size(sc->enp, &sc->vpd_size)) != 0) { /* - * Unpriviledged functions deny VPD access. + * Unprivileged functions deny VPD access. * Simply skip VPD in this case. */ if (rc == EACCES) goto done; goto fail; } sc->vpd_data = malloc(sc->vpd_size, M_SFXGE, M_WAITOK); if ((rc = efx_vpd_read(sc->enp, sc->vpd_data, sc->vpd_size)) != 0) goto fail2; /* Copy ID (product name) into device description, and log it. */ value.evv_tag = EFX_VPD_ID; if (efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value) == 0) { value.evv_value[value.evv_length] = 0; device_set_desc_copy(sc->dev, value.evv_value); device_printf(sc->dev, "%s\n", value.evv_value); } vpd_node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "vpd", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Vital Product Data"); vpd_list = SYSCTL_CHILDREN(vpd_node); /* Add sysctls for all expected and any vendor-defined keywords. */ sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "PN"); sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "EC"); sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "SN"); keyword[0] = 'V'; keyword[2] = 0; for (keyword[1] = '0'; keyword[1] <= '9'; keyword[1]++) sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, keyword); for (keyword[1] = 'A'; keyword[1] <= 'Z'; keyword[1]++) sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, keyword); done: return (0); fail2: free(sc->vpd_data, M_SFXGE); fail: return (rc); } static void sfxge_vpd_fini(struct sfxge_softc *sc) { free(sc->vpd_data, M_SFXGE); } static void sfxge_reset(void *arg, int npending) { struct sfxge_softc *sc; int rc; unsigned attempt; (void)npending; sc = (struct sfxge_softc *)arg; SFXGE_ADAPTER_LOCK(sc); if (sc->init_state != SFXGE_STARTED) goto done; sfxge_stop(sc); efx_nic_reset(sc->enp); for (attempt = 0; attempt < sfxge_restart_attempts; ++attempt) { if ((rc = sfxge_start(sc)) == 0) goto done; device_printf(sc->dev, "start on reset failed (%d)\n", rc); DELAY(100000); } device_printf(sc->dev, "reset failed; interface is now stopped\n"); done: SFXGE_ADAPTER_UNLOCK(sc); } void sfxge_schedule_reset(struct sfxge_softc *sc) { taskqueue_enqueue(taskqueue_thread, &sc->task_reset); } static int sfxge_attach(device_t dev) { struct sfxge_softc *sc; struct ifnet *ifp; int error; sc = device_get_softc(dev); sc->dev = dev; /* Allocate ifnet. */ ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "Couldn't allocate ifnet\n"); error = ENOMEM; goto fail; } sc->ifnet = ifp; /* Initialize hardware. */ DBGPRINT(sc->dev, "create nic"); if ((error = sfxge_create(sc)) != 0) goto fail2; /* Create the ifnet for the port. */ DBGPRINT(sc->dev, "init ifnet"); if ((error = sfxge_ifnet_init(ifp, sc)) != 0) goto fail3; DBGPRINT(sc->dev, "init vpd"); if ((error = sfxge_vpd_init(sc)) != 0) goto fail4; /* * NIC is initialized inside sfxge_create() and kept inialized * to be able to initialize port to discover media types in * sfxge_ifnet_init(). */ efx_nic_fini(sc->enp); sc->init_state = SFXGE_REGISTERED; DBGPRINT(sc->dev, "success"); return (0); fail4: sfxge_ifnet_fini(ifp); fail3: efx_nic_fini(sc->enp); sfxge_destroy(sc); fail2: if_free(sc->ifnet); fail: DBGPRINT(sc->dev, "failed %d", error); return (error); } static int sfxge_detach(device_t dev) { struct sfxge_softc *sc; sc = device_get_softc(dev); sfxge_vpd_fini(sc); /* Destroy the ifnet. */ sfxge_ifnet_fini(sc->ifnet); /* Tear down hardware. */ sfxge_destroy(sc); return (0); } static int sfxge_probe(device_t dev) { uint16_t pci_vendor_id; uint16_t pci_device_id; efx_family_t family; unsigned int mem_bar; int rc; pci_vendor_id = pci_get_vendor(dev); pci_device_id = pci_get_device(dev); DBGPRINT(dev, "PCI ID %04x:%04x", pci_vendor_id, pci_device_id); rc = efx_family(pci_vendor_id, pci_device_id, &family, &mem_bar); if (rc != 0) { DBGPRINT(dev, "efx_family fail %d", rc); return (ENXIO); } if (family == EFX_FAMILY_SIENA) { device_set_desc(dev, "Solarflare SFC9000 family"); return (0); } if (family == EFX_FAMILY_HUNTINGTON) { device_set_desc(dev, "Solarflare SFC9100 family"); return (0); } if (family == EFX_FAMILY_MEDFORD) { device_set_desc(dev, "Solarflare SFC9200 family"); return (0); } if (family == EFX_FAMILY_MEDFORD2) { device_set_desc(dev, "Solarflare SFC9250 family"); return (0); } DBGPRINT(dev, "impossible controller family %d", family); return (ENXIO); } static device_method_t sfxge_methods[] = { DEVMETHOD(device_probe, sfxge_probe), DEVMETHOD(device_attach, sfxge_attach), DEVMETHOD(device_detach, sfxge_detach), DEVMETHOD_END }; static devclass_t sfxge_devclass; static driver_t sfxge_driver = { "sfxge", sfxge_methods, sizeof(struct sfxge_softc) }; DRIVER_MODULE(sfxge, pci, sfxge_driver, sfxge_devclass, 0, 0); diff --git a/sys/geom/journal/g_journal.c b/sys/geom/journal/g_journal.c index 7c8c60183bff..97516ed293e7 100644 --- a/sys/geom/journal/g_journal.c +++ b/sys/geom/journal/g_journal.c @@ -1,3024 +1,3024 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005-2006 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef GJ_MEMDEBUG #include #include #endif #include #include #include #include #include FEATURE(geom_journal, "GEOM journaling support"); /* * On-disk journal format: * * JH - Journal header * RH - Record header * * %%%%%% ****** +------+ +------+ ****** +------+ %%%%%% * % JH % * RH * | Data | | Data | ... * RH * | Data | ... % JH % ... * %%%%%% ****** +------+ +------+ ****** +------+ %%%%%% * */ CTASSERT(sizeof(struct g_journal_header) <= 512); CTASSERT(sizeof(struct g_journal_record_header) <= 512); static MALLOC_DEFINE(M_JOURNAL, "journal_data", "GEOM_JOURNAL Data"); static struct mtx g_journal_cache_mtx; MTX_SYSINIT(g_journal_cache, &g_journal_cache_mtx, "cache usage", MTX_DEF); const struct g_journal_desc *g_journal_filesystems[] = { &g_journal_ufs, NULL }; SYSCTL_DECL(_kern_geom); int g_journal_debug = 0; static u_int g_journal_switch_time = 10; static u_int g_journal_force_switch = 70; static u_int g_journal_parallel_flushes = 16; static u_int g_journal_parallel_copies = 16; static u_int g_journal_accept_immediately = 64; static u_int g_journal_record_entries = GJ_RECORD_HEADER_NENTRIES; static u_int g_journal_do_optimize = 1; static SYSCTL_NODE(_kern_geom, OID_AUTO, journal, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "GEOM_JOURNAL stuff"); SYSCTL_INT(_kern_geom_journal, OID_AUTO, debug, CTLFLAG_RWTUN, &g_journal_debug, 0, "Debug level"); SYSCTL_UINT(_kern_geom_journal, OID_AUTO, switch_time, CTLFLAG_RW, &g_journal_switch_time, 0, "Switch journals every N seconds"); SYSCTL_UINT(_kern_geom_journal, OID_AUTO, force_switch, CTLFLAG_RW, &g_journal_force_switch, 0, "Force switch when journal is N% full"); SYSCTL_UINT(_kern_geom_journal, OID_AUTO, parallel_flushes, CTLFLAG_RW, &g_journal_parallel_flushes, 0, "Number of flush I/O requests to send in parallel"); SYSCTL_UINT(_kern_geom_journal, OID_AUTO, accept_immediately, CTLFLAG_RW, &g_journal_accept_immediately, 0, "Number of I/O requests accepted immediately"); SYSCTL_UINT(_kern_geom_journal, OID_AUTO, parallel_copies, CTLFLAG_RW, &g_journal_parallel_copies, 0, "Number of copy I/O requests to send in parallel"); static int g_journal_record_entries_sysctl(SYSCTL_HANDLER_ARGS) { u_int entries; int error; entries = g_journal_record_entries; error = sysctl_handle_int(oidp, &entries, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (entries < 1 || entries > GJ_RECORD_HEADER_NENTRIES) return (EINVAL); g_journal_record_entries = entries; return (0); } SYSCTL_PROC(_kern_geom_journal, OID_AUTO, record_entries, CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, g_journal_record_entries_sysctl, "I", "Maximum number of entires in one journal record"); SYSCTL_UINT(_kern_geom_journal, OID_AUTO, optimize, CTLFLAG_RW, &g_journal_do_optimize, 0, "Try to combine bios on flush and copy"); static u_long g_journal_cache_used = 0; static u_long g_journal_cache_limit = 64 * 1024 * 1024; static u_int g_journal_cache_divisor = 2; static u_int g_journal_cache_switch = 90; static u_int g_journal_cache_misses = 0; static u_int g_journal_cache_alloc_failures = 0; static u_long g_journal_cache_low = 0; static SYSCTL_NODE(_kern_geom_journal, OID_AUTO, cache, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "GEOM_JOURNAL cache"); SYSCTL_ULONG(_kern_geom_journal_cache, OID_AUTO, used, CTLFLAG_RD, &g_journal_cache_used, 0, "Number of allocated bytes"); static int g_journal_cache_limit_sysctl(SYSCTL_HANDLER_ARGS) { u_long limit; int error; limit = g_journal_cache_limit; error = sysctl_handle_long(oidp, &limit, 0, req); if (error != 0 || req->newptr == NULL) return (error); g_journal_cache_limit = limit; g_journal_cache_low = (limit / 100) * g_journal_cache_switch; return (0); } SYSCTL_PROC(_kern_geom_journal_cache, OID_AUTO, limit, CTLTYPE_ULONG | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, NULL, 0, g_journal_cache_limit_sysctl, "I", "Maximum number of allocated bytes"); SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, divisor, CTLFLAG_RDTUN, &g_journal_cache_divisor, 0, "(kmem_size / kern.geom.journal.cache.divisor) == cache size"); static int g_journal_cache_switch_sysctl(SYSCTL_HANDLER_ARGS) { u_int cswitch; int error; cswitch = g_journal_cache_switch; error = sysctl_handle_int(oidp, &cswitch, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (cswitch > 100) return (EINVAL); g_journal_cache_switch = cswitch; g_journal_cache_low = (g_journal_cache_limit / 100) * cswitch; return (0); } SYSCTL_PROC(_kern_geom_journal_cache, OID_AUTO, switch, CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, g_journal_cache_switch_sysctl, "I", "Force switch when we hit this percent of cache use"); SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, misses, CTLFLAG_RW, &g_journal_cache_misses, 0, "Number of cache misses"); SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, alloc_failures, CTLFLAG_RW, &g_journal_cache_alloc_failures, 0, "Memory allocation failures"); static u_long g_journal_stats_bytes_skipped = 0; static u_long g_journal_stats_combined_ios = 0; static u_long g_journal_stats_switches = 0; static u_long g_journal_stats_wait_for_copy = 0; static u_long g_journal_stats_journal_full = 0; static u_long g_journal_stats_low_mem = 0; static SYSCTL_NODE(_kern_geom_journal, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "GEOM_JOURNAL statistics"); SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, skipped_bytes, CTLFLAG_RW, &g_journal_stats_bytes_skipped, 0, "Number of skipped bytes"); SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, combined_ios, CTLFLAG_RW, &g_journal_stats_combined_ios, 0, "Number of combined I/O requests"); SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, switches, CTLFLAG_RW, &g_journal_stats_switches, 0, "Number of journal switches"); SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, wait_for_copy, CTLFLAG_RW, &g_journal_stats_wait_for_copy, 0, "Wait for journal copy on switch"); SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, journal_full, CTLFLAG_RW, &g_journal_stats_journal_full, 0, "Number of times journal was almost full."); SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, low_mem, CTLFLAG_RW, &g_journal_stats_low_mem, 0, "Number of times low_mem hook was called."); static g_taste_t g_journal_taste; static g_ctl_req_t g_journal_config; static g_dumpconf_t g_journal_dumpconf; static g_init_t g_journal_init; static g_fini_t g_journal_fini; struct g_class g_journal_class = { .name = G_JOURNAL_CLASS_NAME, .version = G_VERSION, .taste = g_journal_taste, .ctlreq = g_journal_config, .dumpconf = g_journal_dumpconf, .init = g_journal_init, .fini = g_journal_fini }; static int g_journal_destroy(struct g_journal_softc *sc); static void g_journal_metadata_update(struct g_journal_softc *sc); static void g_journal_start_switcher(struct g_class *mp); static void g_journal_stop_switcher(void); static void g_journal_switch_wait(struct g_journal_softc *sc); #define GJ_SWITCHER_WORKING 0 #define GJ_SWITCHER_DIE 1 #define GJ_SWITCHER_DIED 2 static struct proc *g_journal_switcher_proc = NULL; static int g_journal_switcher_state = GJ_SWITCHER_WORKING; static int g_journal_switcher_wokenup = 0; static int g_journal_sync_requested = 0; #ifdef GJ_MEMDEBUG struct meminfo { size_t mi_size; struct stack mi_stack; }; #endif /* - * We use our own malloc/realloc/free funtions, so we can collect statistics + * We use our own malloc/realloc/free functions, so we can collect statistics * and force journal switch when we're running out of cache. */ static void * gj_malloc(size_t size, int flags) { void *p; #ifdef GJ_MEMDEBUG struct meminfo *mi; #endif mtx_lock(&g_journal_cache_mtx); if (g_journal_cache_limit > 0 && !g_journal_switcher_wokenup && g_journal_cache_used + size > g_journal_cache_low) { GJ_DEBUG(1, "No cache, waking up the switcher."); g_journal_switcher_wokenup = 1; wakeup(&g_journal_switcher_state); } if ((flags & M_NOWAIT) && g_journal_cache_limit > 0 && g_journal_cache_used + size > g_journal_cache_limit) { mtx_unlock(&g_journal_cache_mtx); g_journal_cache_alloc_failures++; return (NULL); } g_journal_cache_used += size; mtx_unlock(&g_journal_cache_mtx); flags &= ~M_NOWAIT; #ifndef GJ_MEMDEBUG p = malloc(size, M_JOURNAL, flags | M_WAITOK); #else mi = malloc(sizeof(*mi) + size, M_JOURNAL, flags | M_WAITOK); p = (u_char *)mi + sizeof(*mi); mi->mi_size = size; stack_save(&mi->mi_stack); #endif return (p); } static void gj_free(void *p, size_t size) { #ifdef GJ_MEMDEBUG struct meminfo *mi; #endif KASSERT(p != NULL, ("p=NULL")); KASSERT(size > 0, ("size=0")); mtx_lock(&g_journal_cache_mtx); KASSERT(g_journal_cache_used >= size, ("Freeing too much?")); g_journal_cache_used -= size; mtx_unlock(&g_journal_cache_mtx); #ifdef GJ_MEMDEBUG mi = p = (void *)((u_char *)p - sizeof(*mi)); if (mi->mi_size != size) { printf("GJOURNAL: Size mismatch! %zu != %zu\n", size, mi->mi_size); printf("GJOURNAL: Alloc backtrace:\n"); stack_print(&mi->mi_stack); printf("GJOURNAL: Free backtrace:\n"); kdb_backtrace(); } #endif free(p, M_JOURNAL); } static void * gj_realloc(void *p, size_t size, size_t oldsize) { void *np; #ifndef GJ_MEMDEBUG mtx_lock(&g_journal_cache_mtx); g_journal_cache_used -= oldsize; g_journal_cache_used += size; mtx_unlock(&g_journal_cache_mtx); np = realloc(p, size, M_JOURNAL, M_WAITOK); #else np = gj_malloc(size, M_WAITOK); bcopy(p, np, MIN(oldsize, size)); gj_free(p, oldsize); #endif return (np); } static void g_journal_check_overflow(struct g_journal_softc *sc) { off_t length, used; if ((sc->sc_active.jj_offset < sc->sc_inactive.jj_offset && sc->sc_journal_offset >= sc->sc_inactive.jj_offset) || (sc->sc_active.jj_offset > sc->sc_inactive.jj_offset && sc->sc_journal_offset >= sc->sc_inactive.jj_offset && sc->sc_journal_offset < sc->sc_active.jj_offset)) { panic("Journal overflow " "(id = %u joffset=%jd active=%jd inactive=%jd)", (unsigned)sc->sc_id, (intmax_t)sc->sc_journal_offset, (intmax_t)sc->sc_active.jj_offset, (intmax_t)sc->sc_inactive.jj_offset); } if (sc->sc_active.jj_offset < sc->sc_inactive.jj_offset) { length = sc->sc_inactive.jj_offset - sc->sc_active.jj_offset; used = sc->sc_journal_offset - sc->sc_active.jj_offset; } else { length = sc->sc_jend - sc->sc_active.jj_offset; length += sc->sc_inactive.jj_offset - sc->sc_jstart; if (sc->sc_journal_offset >= sc->sc_active.jj_offset) used = sc->sc_journal_offset - sc->sc_active.jj_offset; else { used = sc->sc_jend - sc->sc_active.jj_offset; used += sc->sc_journal_offset - sc->sc_jstart; } } /* Already woken up? */ if (g_journal_switcher_wokenup) return; /* * If the active journal takes more than g_journal_force_switch precent * of free journal space, we force journal switch. */ KASSERT(length > 0, ("length=%jd used=%jd active=%jd inactive=%jd joffset=%jd", (intmax_t)length, (intmax_t)used, (intmax_t)sc->sc_active.jj_offset, (intmax_t)sc->sc_inactive.jj_offset, (intmax_t)sc->sc_journal_offset)); if ((used * 100) / length > g_journal_force_switch) { g_journal_stats_journal_full++; GJ_DEBUG(1, "Journal %s %jd%% full, forcing journal switch.", sc->sc_name, (used * 100) / length); mtx_lock(&g_journal_cache_mtx); g_journal_switcher_wokenup = 1; wakeup(&g_journal_switcher_state); mtx_unlock(&g_journal_cache_mtx); } } static void g_journal_orphan(struct g_consumer *cp) { struct g_journal_softc *sc; char name[256]; int error; g_topology_assert(); sc = cp->geom->softc; strlcpy(name, cp->provider->name, sizeof(name)); GJ_DEBUG(0, "Lost provider %s.", name); if (sc == NULL) return; error = g_journal_destroy(sc); if (error == 0) GJ_DEBUG(0, "Journal %s destroyed.", name); else { GJ_DEBUG(0, "Cannot destroy journal %s (error=%d). " "Destroy it manually after last close.", sc->sc_name, error); } } static int g_journal_access(struct g_provider *pp, int acr, int acw, int ace) { struct g_journal_softc *sc; int dcr, dcw, dce; g_topology_assert(); GJ_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr, acw, ace); dcr = pp->acr + acr; dcw = pp->acw + acw; dce = pp->ace + ace; sc = pp->geom->softc; if (sc == NULL || (sc->sc_flags & GJF_DEVICE_DESTROY)) { if (acr <= 0 && acw <= 0 && ace <= 0) return (0); else return (ENXIO); } if (pp->acw == 0 && dcw > 0) { GJ_DEBUG(1, "Marking %s as dirty.", sc->sc_name); sc->sc_flags &= ~GJF_DEVICE_CLEAN; g_topology_unlock(); g_journal_metadata_update(sc); g_topology_lock(); } /* else if (pp->acw == 0 && dcw > 0 && JEMPTY(sc)) { GJ_DEBUG(1, "Marking %s as clean.", sc->sc_name); sc->sc_flags |= GJF_DEVICE_CLEAN; g_topology_unlock(); g_journal_metadata_update(sc); g_topology_lock(); } */ return (0); } static void g_journal_header_encode(struct g_journal_header *hdr, u_char *data) { bcopy(GJ_HEADER_MAGIC, data, sizeof(GJ_HEADER_MAGIC)); data += sizeof(GJ_HEADER_MAGIC); le32enc(data, hdr->jh_journal_id); data += 4; le32enc(data, hdr->jh_journal_next_id); } static int g_journal_header_decode(const u_char *data, struct g_journal_header *hdr) { bcopy(data, hdr->jh_magic, sizeof(hdr->jh_magic)); data += sizeof(hdr->jh_magic); if (bcmp(hdr->jh_magic, GJ_HEADER_MAGIC, sizeof(GJ_HEADER_MAGIC)) != 0) return (EINVAL); hdr->jh_journal_id = le32dec(data); data += 4; hdr->jh_journal_next_id = le32dec(data); return (0); } static void g_journal_flush_cache(struct g_journal_softc *sc) { struct bintime bt; int error; if (sc->sc_bio_flush == 0) return; GJ_TIMER_START(1, &bt); if (sc->sc_bio_flush & GJ_FLUSH_JOURNAL) { error = g_io_flush(sc->sc_jconsumer); GJ_DEBUG(error == 0 ? 2 : 0, "Flush cache of %s: error=%d.", sc->sc_jconsumer->provider->name, error); } if (sc->sc_bio_flush & GJ_FLUSH_DATA) { /* * TODO: This could be called in parallel with the * previous call. */ error = g_io_flush(sc->sc_dconsumer); GJ_DEBUG(error == 0 ? 2 : 0, "Flush cache of %s: error=%d.", sc->sc_dconsumer->provider->name, error); } GJ_TIMER_STOP(1, &bt, "Cache flush time"); } static int g_journal_write_header(struct g_journal_softc *sc) { struct g_journal_header hdr; struct g_consumer *cp; u_char *buf; int error; cp = sc->sc_jconsumer; buf = gj_malloc(cp->provider->sectorsize, M_WAITOK); strlcpy(hdr.jh_magic, GJ_HEADER_MAGIC, sizeof(hdr.jh_magic)); hdr.jh_journal_id = sc->sc_journal_id; hdr.jh_journal_next_id = sc->sc_journal_next_id; g_journal_header_encode(&hdr, buf); error = g_write_data(cp, sc->sc_journal_offset, buf, cp->provider->sectorsize); /* if (error == 0) */ sc->sc_journal_offset += cp->provider->sectorsize; gj_free(buf, cp->provider->sectorsize); return (error); } /* * Every journal record has a header and data following it. * Functions below are used to decode the header before storing it to * little endian and to encode it after reading to system endianness. */ static void g_journal_record_header_encode(struct g_journal_record_header *hdr, u_char *data) { struct g_journal_entry *ent; u_int i; bcopy(GJ_RECORD_HEADER_MAGIC, data, sizeof(GJ_RECORD_HEADER_MAGIC)); data += sizeof(GJ_RECORD_HEADER_MAGIC); le32enc(data, hdr->jrh_journal_id); data += 8; le16enc(data, hdr->jrh_nentries); data += 2; bcopy(hdr->jrh_sum, data, sizeof(hdr->jrh_sum)); data += 8; for (i = 0; i < hdr->jrh_nentries; i++) { ent = &hdr->jrh_entries[i]; le64enc(data, ent->je_joffset); data += 8; le64enc(data, ent->je_offset); data += 8; le64enc(data, ent->je_length); data += 8; } } static int g_journal_record_header_decode(const u_char *data, struct g_journal_record_header *hdr) { struct g_journal_entry *ent; u_int i; bcopy(data, hdr->jrh_magic, sizeof(hdr->jrh_magic)); data += sizeof(hdr->jrh_magic); if (strcmp(hdr->jrh_magic, GJ_RECORD_HEADER_MAGIC) != 0) return (EINVAL); hdr->jrh_journal_id = le32dec(data); data += 8; hdr->jrh_nentries = le16dec(data); data += 2; if (hdr->jrh_nentries > GJ_RECORD_HEADER_NENTRIES) return (EINVAL); bcopy(data, hdr->jrh_sum, sizeof(hdr->jrh_sum)); data += 8; for (i = 0; i < hdr->jrh_nentries; i++) { ent = &hdr->jrh_entries[i]; ent->je_joffset = le64dec(data); data += 8; ent->je_offset = le64dec(data); data += 8; ent->je_length = le64dec(data); data += 8; } return (0); } /* * Function reads metadata from a provider (via the given consumer), decodes * it to system endianness and verifies its correctness. */ static int g_journal_metadata_read(struct g_consumer *cp, struct g_journal_metadata *md) { struct g_provider *pp; u_char *buf; int error; g_topology_assert(); error = g_access(cp, 1, 0, 0); if (error != 0) return (error); pp = cp->provider; g_topology_unlock(); /* Metadata is stored in last sector. */ buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize, &error); g_topology_lock(); g_access(cp, -1, 0, 0); if (buf == NULL) { GJ_DEBUG(1, "Cannot read metadata from %s (error=%d).", cp->provider->name, error); return (error); } /* Decode metadata. */ error = journal_metadata_decode(buf, md); g_free(buf); /* Is this is gjournal provider at all? */ if (strcmp(md->md_magic, G_JOURNAL_MAGIC) != 0) return (EINVAL); /* * Are we able to handle this version of metadata? * We only maintain backward compatibility. */ if (md->md_version > G_JOURNAL_VERSION) { GJ_DEBUG(0, "Kernel module is too old to handle metadata from %s.", cp->provider->name); return (EINVAL); } /* Is checksum correct? */ if (error != 0) { GJ_DEBUG(0, "MD5 metadata hash mismatch for provider %s.", cp->provider->name); return (error); } return (0); } /* * Two functions below are responsible for updating metadata. * Only metadata on the data provider is updated (we need to update * information about active journal in there). */ static void g_journal_metadata_done(struct bio *bp) { /* * There is not much we can do on error except informing about it. */ if (bp->bio_error != 0) { GJ_LOGREQ(0, bp, "Cannot update metadata (error=%d).", bp->bio_error); } else { GJ_LOGREQ(2, bp, "Metadata updated."); } gj_free(bp->bio_data, bp->bio_length); g_destroy_bio(bp); } static void g_journal_metadata_update(struct g_journal_softc *sc) { struct g_journal_metadata md; struct g_consumer *cp; struct bio *bp; u_char *sector; cp = sc->sc_dconsumer; sector = gj_malloc(cp->provider->sectorsize, M_WAITOK); strlcpy(md.md_magic, G_JOURNAL_MAGIC, sizeof(md.md_magic)); md.md_version = G_JOURNAL_VERSION; md.md_id = sc->sc_id; md.md_type = sc->sc_orig_type; md.md_jstart = sc->sc_jstart; md.md_jend = sc->sc_jend; md.md_joffset = sc->sc_inactive.jj_offset; md.md_jid = sc->sc_journal_previous_id; md.md_flags = 0; if (sc->sc_flags & GJF_DEVICE_CLEAN) md.md_flags |= GJ_FLAG_CLEAN; if (sc->sc_flags & GJF_DEVICE_HARDCODED) strlcpy(md.md_provider, sc->sc_name, sizeof(md.md_provider)); else bzero(md.md_provider, sizeof(md.md_provider)); md.md_provsize = cp->provider->mediasize; journal_metadata_encode(&md, sector); /* * Flush the cache, so we know all data are on disk. * We write here informations like "journal is consistent", so we need * to be sure it is. Without BIO_FLUSH here, we can end up in situation * where metadata is stored on disk, but not all data. */ g_journal_flush_cache(sc); bp = g_alloc_bio(); bp->bio_offset = cp->provider->mediasize - cp->provider->sectorsize; bp->bio_length = cp->provider->sectorsize; bp->bio_data = sector; bp->bio_cmd = BIO_WRITE; if (!(sc->sc_flags & GJF_DEVICE_DESTROY)) { bp->bio_done = g_journal_metadata_done; g_io_request(bp, cp); } else { bp->bio_done = NULL; g_io_request(bp, cp); biowait(bp, "gjmdu"); g_journal_metadata_done(bp); } /* * Be sure metadata reached the disk. */ g_journal_flush_cache(sc); } /* * This is where the I/O request comes from the GEOM. */ static void g_journal_start(struct bio *bp) { struct g_journal_softc *sc; sc = bp->bio_to->geom->softc; GJ_LOGREQ(3, bp, "Request received."); switch (bp->bio_cmd) { case BIO_READ: case BIO_WRITE: mtx_lock(&sc->sc_mtx); bioq_insert_tail(&sc->sc_regular_queue, bp); wakeup(sc); mtx_unlock(&sc->sc_mtx); return; case BIO_GETATTR: if (strcmp(bp->bio_attribute, "GJOURNAL::provider") == 0) { strlcpy(bp->bio_data, bp->bio_to->name, bp->bio_length); bp->bio_completed = strlen(bp->bio_to->name) + 1; g_io_deliver(bp, 0); return; } /* FALLTHROUGH */ case BIO_SPEEDUP: case BIO_DELETE: default: g_io_deliver(bp, EOPNOTSUPP); return; } } static void g_journal_std_done(struct bio *bp) { struct g_journal_softc *sc; sc = bp->bio_from->geom->softc; mtx_lock(&sc->sc_mtx); bioq_insert_tail(&sc->sc_back_queue, bp); wakeup(sc); mtx_unlock(&sc->sc_mtx); } static struct bio * g_journal_new_bio(off_t start, off_t end, off_t joffset, u_char *data, int flags) { struct bio *bp; bp = g_alloc_bio(); bp->bio_offset = start; bp->bio_joffset = joffset; bp->bio_length = end - start; bp->bio_cmd = BIO_WRITE; bp->bio_done = g_journal_std_done; if (data == NULL) bp->bio_data = NULL; else { bp->bio_data = gj_malloc(bp->bio_length, flags); if (bp->bio_data != NULL) bcopy(data, bp->bio_data, bp->bio_length); } return (bp); } #define g_journal_insert_bio(head, bp, flags) \ g_journal_insert((head), (bp)->bio_offset, \ (bp)->bio_offset + (bp)->bio_length, (bp)->bio_joffset, \ (bp)->bio_data, flags) /* * The function below does a lot more than just inserting bio to the queue. * It keeps the queue sorted by offset and ensures that there are no doubled * data (it combines bios where ranges overlap). * * The function returns the number of bios inserted (as bio can be splitted). */ static int g_journal_insert(struct bio **head, off_t nstart, off_t nend, off_t joffset, u_char *data, int flags) { struct bio *nbp, *cbp, *pbp; off_t cstart, cend; u_char *tmpdata; int n; GJ_DEBUG(3, "INSERT(%p): (%jd, %jd, %jd)", *head, nstart, nend, joffset); n = 0; pbp = NULL; GJQ_FOREACH(*head, cbp) { cstart = cbp->bio_offset; cend = cbp->bio_offset + cbp->bio_length; if (nstart >= cend) { /* * +-------------+ * | | * | current | +-------------+ * | bio | | | * | | | new | * +-------------+ | bio | * | | * +-------------+ */ GJ_DEBUG(3, "INSERT(%p): 1", *head); } else if (nend <= cstart) { /* * +-------------+ * | | * +-------------+ | current | * | | | bio | * | new | | | * | bio | +-------------+ * | | * +-------------+ */ nbp = g_journal_new_bio(nstart, nend, joffset, data, flags); if (pbp == NULL) *head = nbp; else pbp->bio_next = nbp; nbp->bio_next = cbp; n++; GJ_DEBUG(3, "INSERT(%p): 2 (nbp=%p pbp=%p)", *head, nbp, pbp); goto end; } else if (nstart <= cstart && nend >= cend) { /* * +-------------+ +-------------+ * | current bio | | current bio | * +---+-------------+---+ +-------------+---+ * | | | | | | | * | | | | | | | * | +-------------+ | +-------------+ | * | new bio | | new bio | * +---------------------+ +-----------------+ * * +-------------+ +-------------+ * | current bio | | current bio | * +---+-------------+ +-------------+ * | | | | | * | | | | | * | +-------------+ +-------------+ * | new bio | | new bio | * +-----------------+ +-------------+ */ g_journal_stats_bytes_skipped += cbp->bio_length; cbp->bio_offset = nstart; cbp->bio_joffset = joffset; cbp->bio_length = cend - nstart; if (cbp->bio_data != NULL) { gj_free(cbp->bio_data, cend - cstart); cbp->bio_data = NULL; } if (data != NULL) { cbp->bio_data = gj_malloc(cbp->bio_length, flags); if (cbp->bio_data != NULL) { bcopy(data, cbp->bio_data, cbp->bio_length); } data += cend - nstart; } joffset += cend - nstart; nstart = cend; GJ_DEBUG(3, "INSERT(%p): 3 (cbp=%p)", *head, cbp); } else if (nstart > cstart && nend >= cend) { /* * +-----------------+ +-------------+ * | current bio | | current bio | * | +-------------+ | +---------+---+ * | | | | | | | * | | | | | | | * +---+-------------+ +---+---------+ | * | new bio | | new bio | * +-------------+ +-------------+ */ g_journal_stats_bytes_skipped += cend - nstart; nbp = g_journal_new_bio(nstart, cend, joffset, data, flags); nbp->bio_next = cbp->bio_next; cbp->bio_next = nbp; cbp->bio_length = nstart - cstart; if (cbp->bio_data != NULL) { cbp->bio_data = gj_realloc(cbp->bio_data, cbp->bio_length, cend - cstart); } if (data != NULL) data += cend - nstart; joffset += cend - nstart; nstart = cend; n++; GJ_DEBUG(3, "INSERT(%p): 4 (cbp=%p)", *head, cbp); } else if (nstart > cstart && nend < cend) { /* * +---------------------+ * | current bio | * | +-------------+ | * | | | | * | | | | * +---+-------------+---+ * | new bio | * +-------------+ */ g_journal_stats_bytes_skipped += nend - nstart; nbp = g_journal_new_bio(nstart, nend, joffset, data, flags); nbp->bio_next = cbp->bio_next; cbp->bio_next = nbp; if (cbp->bio_data == NULL) tmpdata = NULL; else tmpdata = cbp->bio_data + nend - cstart; nbp = g_journal_new_bio(nend, cend, cbp->bio_joffset + nend - cstart, tmpdata, flags); nbp->bio_next = ((struct bio *)cbp->bio_next)->bio_next; ((struct bio *)cbp->bio_next)->bio_next = nbp; cbp->bio_length = nstart - cstart; if (cbp->bio_data != NULL) { cbp->bio_data = gj_realloc(cbp->bio_data, cbp->bio_length, cend - cstart); } n += 2; GJ_DEBUG(3, "INSERT(%p): 5 (cbp=%p)", *head, cbp); goto end; } else if (nstart <= cstart && nend < cend) { /* * +-----------------+ +-------------+ * | current bio | | current bio | * +-------------+ | +---+---------+ | * | | | | | | | * | | | | | | | * +-------------+---+ | +---------+---+ * | new bio | | new bio | * +-------------+ +-------------+ */ g_journal_stats_bytes_skipped += nend - nstart; nbp = g_journal_new_bio(nstart, nend, joffset, data, flags); if (pbp == NULL) *head = nbp; else pbp->bio_next = nbp; nbp->bio_next = cbp; cbp->bio_offset = nend; cbp->bio_length = cend - nend; cbp->bio_joffset += nend - cstart; tmpdata = cbp->bio_data; if (tmpdata != NULL) { cbp->bio_data = gj_malloc(cbp->bio_length, flags); if (cbp->bio_data != NULL) { bcopy(tmpdata + nend - cstart, cbp->bio_data, cbp->bio_length); } gj_free(tmpdata, cend - cstart); } n++; GJ_DEBUG(3, "INSERT(%p): 6 (cbp=%p)", *head, cbp); goto end; } if (nstart == nend) goto end; pbp = cbp; } nbp = g_journal_new_bio(nstart, nend, joffset, data, flags); if (pbp == NULL) *head = nbp; else pbp->bio_next = nbp; nbp->bio_next = NULL; n++; GJ_DEBUG(3, "INSERT(%p): 8 (nbp=%p pbp=%p)", *head, nbp, pbp); end: if (g_journal_debug >= 3) { GJQ_FOREACH(*head, cbp) { GJ_DEBUG(3, "ELEMENT: %p (%jd, %jd, %jd, %p)", cbp, (intmax_t)cbp->bio_offset, (intmax_t)cbp->bio_length, (intmax_t)cbp->bio_joffset, cbp->bio_data); } GJ_DEBUG(3, "INSERT(%p): DONE %d", *head, n); } return (n); } /* * The function combines neighbour bios trying to squeeze as much data as * possible into one bio. * * The function returns the number of bios combined (negative value). */ static int g_journal_optimize(struct bio *head) { struct bio *cbp, *pbp; int n; n = 0; pbp = NULL; GJQ_FOREACH(head, cbp) { /* Skip bios which has to be read first. */ if (cbp->bio_data == NULL) { pbp = NULL; continue; } /* There is no previous bio yet. */ if (pbp == NULL) { pbp = cbp; continue; } /* Is this a neighbour bio? */ if (pbp->bio_offset + pbp->bio_length != cbp->bio_offset) { /* Be sure that bios queue is sorted. */ KASSERT(pbp->bio_offset + pbp->bio_length < cbp->bio_offset, ("poffset=%jd plength=%jd coffset=%jd", (intmax_t)pbp->bio_offset, (intmax_t)pbp->bio_length, (intmax_t)cbp->bio_offset)); pbp = cbp; continue; } /* Be sure we don't end up with too big bio. */ if (pbp->bio_length + cbp->bio_length > maxphys) { pbp = cbp; continue; } /* Ok, we can join bios. */ GJ_LOGREQ(4, pbp, "Join: "); GJ_LOGREQ(4, cbp, "and: "); pbp->bio_data = gj_realloc(pbp->bio_data, pbp->bio_length + cbp->bio_length, pbp->bio_length); bcopy(cbp->bio_data, pbp->bio_data + pbp->bio_length, cbp->bio_length); gj_free(cbp->bio_data, cbp->bio_length); pbp->bio_length += cbp->bio_length; pbp->bio_next = cbp->bio_next; g_destroy_bio(cbp); cbp = pbp; g_journal_stats_combined_ios++; n--; GJ_LOGREQ(4, pbp, "Got: "); } return (n); } /* * TODO: Update comment. * These are functions responsible for copying one portion of data from journal * to the destination provider. * The order goes like this: * 1. Read the header, which contains informations about data blocks * following it. * 2. Read the data blocks from the journal. * 3. Write the data blocks on the data provider. * * g_journal_copy_start() * g_journal_copy_done() - got finished write request, logs potential errors. */ /* * When there is no data in cache, this function is used to read it. */ static void g_journal_read_first(struct g_journal_softc *sc, struct bio *bp) { struct bio *cbp; /* * We were short in memory, so data was freed. * In that case we need to read it back from journal. */ cbp = g_alloc_bio(); cbp->bio_cflags = bp->bio_cflags; cbp->bio_parent = bp; cbp->bio_offset = bp->bio_joffset; cbp->bio_length = bp->bio_length; cbp->bio_data = gj_malloc(bp->bio_length, M_WAITOK); cbp->bio_cmd = BIO_READ; cbp->bio_done = g_journal_std_done; GJ_LOGREQ(4, cbp, "READ FIRST"); g_io_request(cbp, sc->sc_jconsumer); g_journal_cache_misses++; } static void g_journal_copy_send(struct g_journal_softc *sc) { struct bio *bioq, *bp, *lbp; bioq = lbp = NULL; mtx_lock(&sc->sc_mtx); for (; sc->sc_copy_in_progress < g_journal_parallel_copies;) { bp = GJQ_FIRST(sc->sc_inactive.jj_queue); if (bp == NULL) break; GJQ_REMOVE(sc->sc_inactive.jj_queue, bp); sc->sc_copy_in_progress++; GJQ_INSERT_AFTER(bioq, bp, lbp); lbp = bp; } mtx_unlock(&sc->sc_mtx); if (g_journal_do_optimize) sc->sc_copy_in_progress += g_journal_optimize(bioq); while ((bp = GJQ_FIRST(bioq)) != NULL) { GJQ_REMOVE(bioq, bp); GJQ_INSERT_HEAD(sc->sc_copy_queue, bp); bp->bio_cflags = GJ_BIO_COPY; if (bp->bio_data == NULL) g_journal_read_first(sc, bp); else { bp->bio_joffset = 0; GJ_LOGREQ(4, bp, "SEND"); g_io_request(bp, sc->sc_dconsumer); } } } static void g_journal_copy_start(struct g_journal_softc *sc) { /* * Remember in metadata that we're starting to copy journaled data * to the data provider. * In case of power failure, we will copy these data once again on boot. */ if (!sc->sc_journal_copying) { sc->sc_journal_copying = 1; GJ_DEBUG(1, "Starting copy of journal."); g_journal_metadata_update(sc); } g_journal_copy_send(sc); } /* * Data block has been read from the journal provider. */ static int g_journal_copy_read_done(struct bio *bp) { struct g_journal_softc *sc; struct g_consumer *cp; struct bio *pbp; KASSERT(bp->bio_cflags == GJ_BIO_COPY, ("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_COPY)); sc = bp->bio_from->geom->softc; pbp = bp->bio_parent; if (bp->bio_error != 0) { GJ_DEBUG(0, "Error while reading data from %s (error=%d).", bp->bio_to->name, bp->bio_error); /* * We will not be able to deliver WRITE request as well. */ gj_free(bp->bio_data, bp->bio_length); g_destroy_bio(pbp); g_destroy_bio(bp); sc->sc_copy_in_progress--; return (1); } pbp->bio_data = bp->bio_data; cp = sc->sc_dconsumer; g_io_request(pbp, cp); GJ_LOGREQ(4, bp, "READ DONE"); g_destroy_bio(bp); return (0); } /* * Data block has been written to the data provider. */ static void g_journal_copy_write_done(struct bio *bp) { struct g_journal_softc *sc; KASSERT(bp->bio_cflags == GJ_BIO_COPY, ("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_COPY)); sc = bp->bio_from->geom->softc; sc->sc_copy_in_progress--; if (bp->bio_error != 0) { GJ_LOGREQ(0, bp, "[copy] Error while writing data (error=%d)", bp->bio_error); } GJQ_REMOVE(sc->sc_copy_queue, bp); gj_free(bp->bio_data, bp->bio_length); GJ_LOGREQ(4, bp, "DONE"); g_destroy_bio(bp); if (sc->sc_copy_in_progress == 0) { /* * This was the last write request for this journal. */ GJ_DEBUG(1, "Data has been copied."); sc->sc_journal_copying = 0; } } static void g_journal_flush_done(struct bio *bp); /* * Flush one record onto active journal provider. */ static void g_journal_flush(struct g_journal_softc *sc) { struct g_journal_record_header hdr; struct g_journal_entry *ent; struct g_provider *pp; struct bio **bioq; struct bio *bp, *fbp, *pbp; off_t joffset; u_char *data, hash[16]; MD5_CTX ctx; u_int i; if (sc->sc_current_count == 0) return; pp = sc->sc_jprovider; GJ_VALIDATE_OFFSET(sc->sc_journal_offset, sc); joffset = sc->sc_journal_offset; GJ_DEBUG(2, "Storing %d journal entries on %s at %jd.", sc->sc_current_count, pp->name, (intmax_t)joffset); /* * Store 'journal id', so we know to which journal this record belongs. */ hdr.jrh_journal_id = sc->sc_journal_id; /* Could be less than g_journal_record_entries if called due timeout. */ hdr.jrh_nentries = MIN(sc->sc_current_count, g_journal_record_entries); strlcpy(hdr.jrh_magic, GJ_RECORD_HEADER_MAGIC, sizeof(hdr.jrh_magic)); bioq = &sc->sc_active.jj_queue; GJQ_LAST(sc->sc_flush_queue, pbp); fbp = g_alloc_bio(); fbp->bio_parent = NULL; fbp->bio_cflags = GJ_BIO_JOURNAL; fbp->bio_offset = -1; fbp->bio_joffset = joffset; fbp->bio_length = pp->sectorsize; fbp->bio_cmd = BIO_WRITE; fbp->bio_done = g_journal_std_done; GJQ_INSERT_AFTER(sc->sc_flush_queue, fbp, pbp); pbp = fbp; fbp->bio_to = pp; GJ_LOGREQ(4, fbp, "FLUSH_OUT"); joffset += pp->sectorsize; sc->sc_flush_count++; if (sc->sc_flags & GJF_DEVICE_CHECKSUM) MD5Init(&ctx); for (i = 0; i < hdr.jrh_nentries; i++) { bp = sc->sc_current_queue; KASSERT(bp != NULL, ("NULL bp")); bp->bio_to = pp; GJ_LOGREQ(4, bp, "FLUSHED"); sc->sc_current_queue = bp->bio_next; bp->bio_next = NULL; sc->sc_current_count--; /* Add to the header. */ ent = &hdr.jrh_entries[i]; ent->je_offset = bp->bio_offset; ent->je_joffset = joffset; ent->je_length = bp->bio_length; data = bp->bio_data; if (sc->sc_flags & GJF_DEVICE_CHECKSUM) MD5Update(&ctx, data, ent->je_length); g_reset_bio(bp); bp->bio_cflags = GJ_BIO_JOURNAL; bp->bio_offset = ent->je_offset; bp->bio_joffset = ent->je_joffset; bp->bio_length = ent->je_length; bp->bio_data = data; bp->bio_cmd = BIO_WRITE; bp->bio_done = g_journal_std_done; GJQ_INSERT_AFTER(sc->sc_flush_queue, bp, pbp); pbp = bp; bp->bio_to = pp; GJ_LOGREQ(4, bp, "FLUSH_OUT"); joffset += bp->bio_length; sc->sc_flush_count++; /* * Add request to the active sc_journal_queue queue. * This is our cache. After journal switch we don't have to * read the data from the inactive journal, because we keep * it in memory. */ g_journal_insert(bioq, ent->je_offset, ent->je_offset + ent->je_length, ent->je_joffset, data, M_NOWAIT); } /* * After all requests, store valid header. */ data = gj_malloc(pp->sectorsize, M_WAITOK); if (sc->sc_flags & GJF_DEVICE_CHECKSUM) { MD5Final(hash, &ctx); bcopy(hash, hdr.jrh_sum, sizeof(hdr.jrh_sum)); } g_journal_record_header_encode(&hdr, data); fbp->bio_data = data; sc->sc_journal_offset = joffset; g_journal_check_overflow(sc); } /* * Flush request finished. */ static void g_journal_flush_done(struct bio *bp) { struct g_journal_softc *sc; struct g_consumer *cp; KASSERT((bp->bio_cflags & GJ_BIO_MASK) == GJ_BIO_JOURNAL, ("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_JOURNAL)); cp = bp->bio_from; sc = cp->geom->softc; sc->sc_flush_in_progress--; if (bp->bio_error != 0) { GJ_LOGREQ(0, bp, "[flush] Error while writing data (error=%d)", bp->bio_error); } gj_free(bp->bio_data, bp->bio_length); GJ_LOGREQ(4, bp, "DONE"); g_destroy_bio(bp); } static void g_journal_release_delayed(struct g_journal_softc *sc); static void g_journal_flush_send(struct g_journal_softc *sc) { struct g_consumer *cp; struct bio *bioq, *bp, *lbp; cp = sc->sc_jconsumer; bioq = lbp = NULL; while (sc->sc_flush_in_progress < g_journal_parallel_flushes) { /* Send one flush requests to the active journal. */ bp = GJQ_FIRST(sc->sc_flush_queue); if (bp != NULL) { GJQ_REMOVE(sc->sc_flush_queue, bp); sc->sc_flush_count--; bp->bio_offset = bp->bio_joffset; bp->bio_joffset = 0; sc->sc_flush_in_progress++; GJQ_INSERT_AFTER(bioq, bp, lbp); lbp = bp; } /* Try to release delayed requests. */ g_journal_release_delayed(sc); /* If there are no requests to flush, leave. */ if (GJQ_FIRST(sc->sc_flush_queue) == NULL) break; } if (g_journal_do_optimize) sc->sc_flush_in_progress += g_journal_optimize(bioq); while ((bp = GJQ_FIRST(bioq)) != NULL) { GJQ_REMOVE(bioq, bp); GJ_LOGREQ(3, bp, "Flush request send"); g_io_request(bp, cp); } } static void g_journal_add_current(struct g_journal_softc *sc, struct bio *bp) { int n; GJ_LOGREQ(4, bp, "CURRENT %d", sc->sc_current_count); n = g_journal_insert_bio(&sc->sc_current_queue, bp, M_WAITOK); sc->sc_current_count += n; n = g_journal_optimize(sc->sc_current_queue); sc->sc_current_count += n; /* * For requests which are added to the current queue we deliver * response immediately. */ bp->bio_completed = bp->bio_length; g_io_deliver(bp, 0); if (sc->sc_current_count >= g_journal_record_entries) { /* * Let's flush one record onto active journal provider. */ g_journal_flush(sc); } } static void g_journal_release_delayed(struct g_journal_softc *sc) { struct bio *bp; for (;;) { /* The flush queue is full, exit. */ if (sc->sc_flush_count >= g_journal_accept_immediately) return; bp = bioq_takefirst(&sc->sc_delayed_queue); if (bp == NULL) return; sc->sc_delayed_count--; g_journal_add_current(sc, bp); } } /* * Add I/O request to the current queue. If we have enough requests for one * journal record we flush them onto active journal provider. */ static void g_journal_add_request(struct g_journal_softc *sc, struct bio *bp) { /* * The flush queue is full, we need to delay the request. */ if (sc->sc_delayed_count > 0 || sc->sc_flush_count >= g_journal_accept_immediately) { GJ_LOGREQ(4, bp, "DELAYED"); bioq_insert_tail(&sc->sc_delayed_queue, bp); sc->sc_delayed_count++; return; } KASSERT(TAILQ_EMPTY(&sc->sc_delayed_queue.queue), ("DELAYED queue not empty.")); g_journal_add_current(sc, bp); } static void g_journal_read_done(struct bio *bp); /* * Try to find requested data in cache. */ static struct bio * g_journal_read_find(struct bio *head, int sorted, struct bio *pbp, off_t ostart, off_t oend) { off_t cstart, cend; struct bio *bp; GJQ_FOREACH(head, bp) { if (bp->bio_offset == -1) continue; cstart = MAX(ostart, bp->bio_offset); cend = MIN(oend, bp->bio_offset + bp->bio_length); if (cend <= ostart) continue; else if (cstart >= oend) { if (!sorted) continue; else { bp = NULL; break; } } if (bp->bio_data == NULL) break; GJ_DEBUG(3, "READ(%p): (%jd, %jd) (bp=%p)", head, cstart, cend, bp); bcopy(bp->bio_data + cstart - bp->bio_offset, pbp->bio_data + cstart - pbp->bio_offset, cend - cstart); pbp->bio_completed += cend - cstart; if (pbp->bio_completed == pbp->bio_length) { /* * Cool, the whole request was in cache, deliver happy * message. */ g_io_deliver(pbp, 0); return (pbp); } break; } return (bp); } /* * This function is used for collecting data on read. * The complexity is because parts of the data can be stored in four different * places: * - in memory - the data not yet send to the active journal provider * - in the active journal * - in the inactive journal * - in the data provider */ static void g_journal_read(struct g_journal_softc *sc, struct bio *pbp, off_t ostart, off_t oend) { struct bio *bp, *nbp, *head; off_t cstart, cend; u_int i, sorted = 0; GJ_DEBUG(3, "READ: (%jd, %jd)", ostart, oend); cstart = cend = -1; bp = NULL; head = NULL; for (i = 1; i <= 5; i++) { switch (i) { case 1: /* Not-yet-send data. */ head = sc->sc_current_queue; sorted = 1; break; case 2: /* Skip flush queue as they are also in active queue */ continue; case 3: /* Active journal. */ head = sc->sc_active.jj_queue; sorted = 1; break; case 4: /* Inactive journal. */ /* * XXX: Here could be a race with g_journal_lowmem(). */ head = sc->sc_inactive.jj_queue; sorted = 1; break; case 5: /* In-flight to the data provider. */ head = sc->sc_copy_queue; sorted = 0; break; default: panic("gjournal %s: i=%d", __func__, i); } bp = g_journal_read_find(head, sorted, pbp, ostart, oend); if (bp == pbp) { /* Got the whole request. */ GJ_DEBUG(2, "Got the whole request from %u.", i); return; } else if (bp != NULL) { cstart = MAX(ostart, bp->bio_offset); cend = MIN(oend, bp->bio_offset + bp->bio_length); GJ_DEBUG(2, "Got part of the request from %u (%jd-%jd).", i, (intmax_t)cstart, (intmax_t)cend); break; } } if (bp != NULL) { if (bp->bio_data == NULL) { nbp = g_duplicate_bio(pbp); nbp->bio_cflags = GJ_BIO_READ; nbp->bio_data = pbp->bio_data + cstart - pbp->bio_offset; nbp->bio_offset = bp->bio_joffset + cstart - bp->bio_offset; nbp->bio_length = cend - cstart; nbp->bio_done = g_journal_read_done; g_io_request(nbp, sc->sc_jconsumer); } /* * If we don't have the whole request yet, call g_journal_read() * recursively. */ if (ostart < cstart) g_journal_read(sc, pbp, ostart, cstart); if (oend > cend) g_journal_read(sc, pbp, cend, oend); } else { /* * No data in memory, no data in journal. * Its time for asking data provider. */ GJ_DEBUG(3, "READ(data): (%jd, %jd)", ostart, oend); nbp = g_duplicate_bio(pbp); nbp->bio_cflags = GJ_BIO_READ; nbp->bio_data = pbp->bio_data + ostart - pbp->bio_offset; nbp->bio_offset = ostart; nbp->bio_length = oend - ostart; nbp->bio_done = g_journal_read_done; g_io_request(nbp, sc->sc_dconsumer); /* We have the whole request, return here. */ return; } } /* * Function responsible for handling finished READ requests. * Actually, g_std_done() could be used here, the only difference is that we * log error. */ static void g_journal_read_done(struct bio *bp) { struct bio *pbp; KASSERT(bp->bio_cflags == GJ_BIO_READ, ("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_READ)); pbp = bp->bio_parent; pbp->bio_inbed++; pbp->bio_completed += bp->bio_length; if (bp->bio_error != 0) { if (pbp->bio_error == 0) pbp->bio_error = bp->bio_error; GJ_DEBUG(0, "Error while reading data from %s (error=%d).", bp->bio_to->name, bp->bio_error); } g_destroy_bio(bp); if (pbp->bio_children == pbp->bio_inbed && pbp->bio_completed == pbp->bio_length) { /* We're done. */ g_io_deliver(pbp, 0); } } /* * Deactive current journal and active next one. */ static void g_journal_switch(struct g_journal_softc *sc) { struct g_provider *pp; if (JEMPTY(sc)) { GJ_DEBUG(3, "No need for %s switch.", sc->sc_name); pp = LIST_FIRST(&sc->sc_geom->provider); if (!(sc->sc_flags & GJF_DEVICE_CLEAN) && pp->acw == 0) { sc->sc_flags |= GJF_DEVICE_CLEAN; GJ_DEBUG(1, "Marking %s as clean.", sc->sc_name); g_journal_metadata_update(sc); } } else { GJ_DEBUG(3, "Switching journal %s.", sc->sc_geom->name); pp = sc->sc_jprovider; sc->sc_journal_previous_id = sc->sc_journal_id; sc->sc_journal_id = sc->sc_journal_next_id; sc->sc_journal_next_id = arc4random(); GJ_VALIDATE_OFFSET(sc->sc_journal_offset, sc); g_journal_write_header(sc); sc->sc_inactive.jj_offset = sc->sc_active.jj_offset; sc->sc_inactive.jj_queue = sc->sc_active.jj_queue; sc->sc_active.jj_offset = sc->sc_journal_offset - pp->sectorsize; sc->sc_active.jj_queue = NULL; /* * Switch is done, start copying data from the (now) inactive * journal to the data provider. */ g_journal_copy_start(sc); } mtx_lock(&sc->sc_mtx); sc->sc_flags &= ~GJF_DEVICE_SWITCH; mtx_unlock(&sc->sc_mtx); } static void g_journal_initialize(struct g_journal_softc *sc) { sc->sc_journal_id = arc4random(); sc->sc_journal_next_id = arc4random(); sc->sc_journal_previous_id = sc->sc_journal_id; sc->sc_journal_offset = sc->sc_jstart; sc->sc_inactive.jj_offset = sc->sc_jstart; g_journal_write_header(sc); sc->sc_active.jj_offset = sc->sc_jstart; } static void g_journal_mark_as_dirty(struct g_journal_softc *sc) { const struct g_journal_desc *desc; int i; GJ_DEBUG(1, "Marking file system %s as dirty.", sc->sc_name); for (i = 0; (desc = g_journal_filesystems[i]) != NULL; i++) desc->jd_dirty(sc->sc_dconsumer); } /* * Function read record header from the given journal. * It is very simlar to g_read_data(9), but it doesn't allocate memory for bio * and data on every call. */ static int g_journal_sync_read(struct g_consumer *cp, struct bio *bp, off_t offset, void *data) { int error; g_reset_bio(bp); bp->bio_cmd = BIO_READ; bp->bio_done = NULL; bp->bio_offset = offset; bp->bio_length = cp->provider->sectorsize; bp->bio_data = data; g_io_request(bp, cp); error = biowait(bp, "gjs_read"); return (error); } #if 0 /* * Function is called when we start the journal device and we detect that * one of the journals was not fully copied. * The purpose of this function is to read all records headers from journal * and placed them in the inactive queue, so we can start journal * synchronization process and the journal provider itself. * Design decision was taken to not synchronize the whole journal here as it * can take too much time. Reading headers only and delaying synchronization * process until after journal provider is started should be the best choice. */ #endif static void g_journal_sync(struct g_journal_softc *sc) { struct g_journal_record_header rhdr; struct g_journal_entry *ent; struct g_journal_header jhdr; struct g_consumer *cp; struct bio *bp, *fbp, *tbp; off_t joffset, offset; u_char *buf, sum[16]; uint64_t id; MD5_CTX ctx; int error, found, i; found = 0; fbp = NULL; cp = sc->sc_jconsumer; bp = g_alloc_bio(); buf = gj_malloc(cp->provider->sectorsize, M_WAITOK); offset = joffset = sc->sc_inactive.jj_offset = sc->sc_journal_offset; GJ_DEBUG(2, "Looking for termination at %jd.", (intmax_t)joffset); /* * Read and decode first journal header. */ error = g_journal_sync_read(cp, bp, offset, buf); if (error != 0) { GJ_DEBUG(0, "Error while reading journal header from %s.", cp->provider->name); goto end; } error = g_journal_header_decode(buf, &jhdr); if (error != 0) { GJ_DEBUG(0, "Cannot decode journal header from %s.", cp->provider->name); goto end; } id = sc->sc_journal_id; if (jhdr.jh_journal_id != sc->sc_journal_id) { GJ_DEBUG(1, "Journal ID mismatch at %jd (0x%08x != 0x%08x).", (intmax_t)offset, (u_int)jhdr.jh_journal_id, (u_int)id); goto end; } offset += cp->provider->sectorsize; id = sc->sc_journal_next_id = jhdr.jh_journal_next_id; for (;;) { /* * If the biggest record won't fit, look for a record header or * journal header from the beginning. */ GJ_VALIDATE_OFFSET(offset, sc); error = g_journal_sync_read(cp, bp, offset, buf); if (error != 0) { /* * Not good. Having an error while reading header * means, that we cannot read next headers and in * consequence we cannot find termination. */ GJ_DEBUG(0, "Error while reading record header from %s.", cp->provider->name); break; } error = g_journal_record_header_decode(buf, &rhdr); if (error != 0) { GJ_DEBUG(2, "Not a record header at %jd (error=%d).", (intmax_t)offset, error); /* * This is not a record header. * If we are lucky, this is next journal header. */ error = g_journal_header_decode(buf, &jhdr); if (error != 0) { GJ_DEBUG(1, "Not a journal header at %jd (error=%d).", (intmax_t)offset, error); /* * Nope, this is not journal header, which * bascially means that journal is not * terminated properly. */ error = ENOENT; break; } /* * Ok. This is header of _some_ journal. Now we need to * verify if this is header of the _next_ journal. */ if (jhdr.jh_journal_id != id) { GJ_DEBUG(1, "Journal ID mismatch at %jd " "(0x%08x != 0x%08x).", (intmax_t)offset, (u_int)jhdr.jh_journal_id, (u_int)id); error = ENOENT; break; } /* Found termination. */ found++; GJ_DEBUG(1, "Found termination at %jd (id=0x%08x).", (intmax_t)offset, (u_int)id); sc->sc_active.jj_offset = offset; sc->sc_journal_offset = offset + cp->provider->sectorsize; sc->sc_journal_id = id; id = sc->sc_journal_next_id = jhdr.jh_journal_next_id; while ((tbp = fbp) != NULL) { fbp = tbp->bio_next; GJ_LOGREQ(3, tbp, "Adding request."); g_journal_insert_bio(&sc->sc_inactive.jj_queue, tbp, M_WAITOK); } /* Skip journal's header. */ offset += cp->provider->sectorsize; continue; } /* Skip record's header. */ offset += cp->provider->sectorsize; /* * Add information about every record entry to the inactive * queue. */ if (sc->sc_flags & GJF_DEVICE_CHECKSUM) MD5Init(&ctx); for (i = 0; i < rhdr.jrh_nentries; i++) { ent = &rhdr.jrh_entries[i]; GJ_DEBUG(3, "Insert entry: %jd %jd.", (intmax_t)ent->je_offset, (intmax_t)ent->je_length); g_journal_insert(&fbp, ent->je_offset, ent->je_offset + ent->je_length, ent->je_joffset, NULL, M_WAITOK); if (sc->sc_flags & GJF_DEVICE_CHECKSUM) { u_char *buf2; /* * TODO: Should use faster function (like * g_journal_sync_read()). */ buf2 = g_read_data(cp, offset, ent->je_length, NULL); if (buf2 == NULL) GJ_DEBUG(0, "Cannot read data at %jd.", (intmax_t)offset); else { MD5Update(&ctx, buf2, ent->je_length); g_free(buf2); } } /* Skip entry's data. */ offset += ent->je_length; } if (sc->sc_flags & GJF_DEVICE_CHECKSUM) { MD5Final(sum, &ctx); if (bcmp(sum, rhdr.jrh_sum, sizeof(rhdr.jrh_sum)) != 0) { GJ_DEBUG(0, "MD5 hash mismatch at %jd!", (intmax_t)offset); } } } end: gj_free(bp->bio_data, cp->provider->sectorsize); g_destroy_bio(bp); /* Remove bios from unterminated journal. */ while ((tbp = fbp) != NULL) { fbp = tbp->bio_next; g_destroy_bio(tbp); } if (found < 1 && joffset > 0) { GJ_DEBUG(0, "Journal on %s is broken/corrupted. Initializing.", sc->sc_name); while ((tbp = sc->sc_inactive.jj_queue) != NULL) { sc->sc_inactive.jj_queue = tbp->bio_next; g_destroy_bio(tbp); } g_journal_initialize(sc); g_journal_mark_as_dirty(sc); } else { GJ_DEBUG(0, "Journal %s consistent.", sc->sc_name); g_journal_copy_start(sc); } } /* * Wait for requests. * If we have requests in the current queue, flush them after 3 seconds from the * last flush. In this way we don't wait forever (or for journal switch) with * storing not full records on journal. */ static void g_journal_wait(struct g_journal_softc *sc, time_t last_write) { int error, timeout; GJ_DEBUG(3, "%s: enter", __func__); if (sc->sc_current_count == 0) { if (g_journal_debug < 2) msleep(sc, &sc->sc_mtx, PRIBIO | PDROP, "gj:work", 0); else { /* * If we have debug turned on, show number of elements * in various queues. */ for (;;) { error = msleep(sc, &sc->sc_mtx, PRIBIO, "gj:work", hz * 3); if (error == 0) { mtx_unlock(&sc->sc_mtx); break; } GJ_DEBUG(3, "Report: current count=%d", sc->sc_current_count); GJ_DEBUG(3, "Report: flush count=%d", sc->sc_flush_count); GJ_DEBUG(3, "Report: flush in progress=%d", sc->sc_flush_in_progress); GJ_DEBUG(3, "Report: copy in progress=%d", sc->sc_copy_in_progress); GJ_DEBUG(3, "Report: delayed=%d", sc->sc_delayed_count); } } GJ_DEBUG(3, "%s: exit 1", __func__); return; } /* * Flush even not full records every 3 seconds. */ timeout = (last_write + 3 - time_second) * hz; if (timeout <= 0) { mtx_unlock(&sc->sc_mtx); g_journal_flush(sc); g_journal_flush_send(sc); GJ_DEBUG(3, "%s: exit 2", __func__); return; } error = msleep(sc, &sc->sc_mtx, PRIBIO | PDROP, "gj:work", timeout); if (error == EWOULDBLOCK) g_journal_flush_send(sc); GJ_DEBUG(3, "%s: exit 3", __func__); } /* * Worker thread. */ static void g_journal_worker(void *arg) { struct g_journal_softc *sc; struct g_geom *gp; struct g_provider *pp; struct bio *bp; time_t last_write; int type; thread_lock(curthread); sched_prio(curthread, PRIBIO); thread_unlock(curthread); sc = arg; type = 0; /* gcc */ if (sc->sc_flags & GJF_DEVICE_CLEAN) { GJ_DEBUG(0, "Journal %s clean.", sc->sc_name); g_journal_initialize(sc); } else { g_journal_sync(sc); } /* * Check if we can use BIO_FLUSH. */ sc->sc_bio_flush = 0; if (g_io_flush(sc->sc_jconsumer) == 0) { sc->sc_bio_flush |= GJ_FLUSH_JOURNAL; GJ_DEBUG(1, "BIO_FLUSH supported by %s.", sc->sc_jconsumer->provider->name); } else { GJ_DEBUG(0, "BIO_FLUSH not supported by %s.", sc->sc_jconsumer->provider->name); } if (sc->sc_jconsumer != sc->sc_dconsumer) { if (g_io_flush(sc->sc_dconsumer) == 0) { sc->sc_bio_flush |= GJ_FLUSH_DATA; GJ_DEBUG(1, "BIO_FLUSH supported by %s.", sc->sc_dconsumer->provider->name); } else { GJ_DEBUG(0, "BIO_FLUSH not supported by %s.", sc->sc_dconsumer->provider->name); } } gp = sc->sc_geom; g_topology_lock(); pp = g_new_providerf(gp, "%s.journal", sc->sc_name); pp->mediasize = sc->sc_mediasize; /* * There could be a problem when data provider and journal providers * have different sectorsize, but such scenario is prevented on journal * creation. */ pp->sectorsize = sc->sc_sectorsize; g_error_provider(pp, 0); g_topology_unlock(); last_write = time_second; if (sc->sc_rootmount != NULL) { GJ_DEBUG(1, "root_mount_rel %p", sc->sc_rootmount); root_mount_rel(sc->sc_rootmount); sc->sc_rootmount = NULL; } for (;;) { /* Get first request from the queue. */ mtx_lock(&sc->sc_mtx); bp = bioq_first(&sc->sc_back_queue); if (bp != NULL) type = (bp->bio_cflags & GJ_BIO_MASK); if (bp == NULL) { bp = bioq_first(&sc->sc_regular_queue); if (bp != NULL) type = GJ_BIO_REGULAR; } if (bp == NULL) { try_switch: if ((sc->sc_flags & GJF_DEVICE_SWITCH) || (sc->sc_flags & GJF_DEVICE_DESTROY)) { if (sc->sc_current_count > 0) { mtx_unlock(&sc->sc_mtx); g_journal_flush(sc); g_journal_flush_send(sc); continue; } if (sc->sc_flush_in_progress > 0) goto sleep; if (sc->sc_copy_in_progress > 0) goto sleep; } if (sc->sc_flags & GJF_DEVICE_SWITCH) { mtx_unlock(&sc->sc_mtx); g_journal_switch(sc); wakeup(&sc->sc_journal_copying); continue; } if (sc->sc_flags & GJF_DEVICE_DESTROY) { GJ_DEBUG(1, "Shutting down worker " "thread for %s.", gp->name); sc->sc_worker = NULL; wakeup(&sc->sc_worker); mtx_unlock(&sc->sc_mtx); kproc_exit(0); } sleep: g_journal_wait(sc, last_write); continue; } /* * If we're in switch process, we need to delay all new * write requests until its done. */ if ((sc->sc_flags & GJF_DEVICE_SWITCH) && type == GJ_BIO_REGULAR && bp->bio_cmd == BIO_WRITE) { GJ_LOGREQ(2, bp, "WRITE on SWITCH"); goto try_switch; } if (type == GJ_BIO_REGULAR) bioq_remove(&sc->sc_regular_queue, bp); else bioq_remove(&sc->sc_back_queue, bp); mtx_unlock(&sc->sc_mtx); switch (type) { case GJ_BIO_REGULAR: /* Regular request. */ switch (bp->bio_cmd) { case BIO_READ: g_journal_read(sc, bp, bp->bio_offset, bp->bio_offset + bp->bio_length); break; case BIO_WRITE: last_write = time_second; g_journal_add_request(sc, bp); g_journal_flush_send(sc); break; default: panic("Invalid bio_cmd (%d).", bp->bio_cmd); } break; case GJ_BIO_COPY: switch (bp->bio_cmd) { case BIO_READ: if (g_journal_copy_read_done(bp)) g_journal_copy_send(sc); break; case BIO_WRITE: g_journal_copy_write_done(bp); g_journal_copy_send(sc); break; default: panic("Invalid bio_cmd (%d).", bp->bio_cmd); } break; case GJ_BIO_JOURNAL: g_journal_flush_done(bp); g_journal_flush_send(sc); break; case GJ_BIO_READ: default: panic("Invalid bio (%d).", type); } } } static void g_journal_destroy_event(void *arg, int flags __unused) { struct g_journal_softc *sc; g_topology_assert(); sc = arg; g_journal_destroy(sc); } static void g_journal_timeout(void *arg) { struct g_journal_softc *sc; sc = arg; GJ_DEBUG(0, "Timeout. Journal %s cannot be completed.", sc->sc_geom->name); g_post_event(g_journal_destroy_event, sc, M_NOWAIT, NULL); } static struct g_geom * g_journal_create(struct g_class *mp, struct g_provider *pp, const struct g_journal_metadata *md) { struct g_journal_softc *sc; struct g_geom *gp; struct g_consumer *cp; int error; sc = NULL; /* gcc */ g_topology_assert(); /* * There are two possibilities: * 1. Data and both journals are on the same provider. * 2. Data and journals are all on separated providers. */ /* Look for journal device with the same ID. */ LIST_FOREACH(gp, &mp->geom, geom) { sc = gp->softc; if (sc == NULL) continue; if (sc->sc_id == md->md_id) break; } if (gp == NULL) sc = NULL; else if (sc != NULL && (sc->sc_type & md->md_type) != 0) { GJ_DEBUG(1, "Journal device %u already configured.", sc->sc_id); return (NULL); } if (md->md_type == 0 || (md->md_type & ~GJ_TYPE_COMPLETE) != 0) { GJ_DEBUG(0, "Invalid type on %s.", pp->name); return (NULL); } if (md->md_type & GJ_TYPE_DATA) { GJ_DEBUG(0, "Journal %u: %s contains data.", md->md_id, pp->name); } if (md->md_type & GJ_TYPE_JOURNAL) { GJ_DEBUG(0, "Journal %u: %s contains journal.", md->md_id, pp->name); } if (sc == NULL) { /* Action geom. */ sc = malloc(sizeof(*sc), M_JOURNAL, M_WAITOK | M_ZERO); sc->sc_id = md->md_id; sc->sc_type = 0; sc->sc_flags = 0; sc->sc_worker = NULL; gp = g_new_geomf(mp, "gjournal %u", sc->sc_id); gp->start = g_journal_start; gp->orphan = g_journal_orphan; gp->access = g_journal_access; gp->softc = sc; gp->flags |= G_GEOM_VOLATILE_BIO; sc->sc_geom = gp; mtx_init(&sc->sc_mtx, "gjournal", NULL, MTX_DEF); bioq_init(&sc->sc_back_queue); bioq_init(&sc->sc_regular_queue); bioq_init(&sc->sc_delayed_queue); sc->sc_delayed_count = 0; sc->sc_current_queue = NULL; sc->sc_current_count = 0; sc->sc_flush_queue = NULL; sc->sc_flush_count = 0; sc->sc_flush_in_progress = 0; sc->sc_copy_queue = NULL; sc->sc_copy_in_progress = 0; sc->sc_inactive.jj_queue = NULL; sc->sc_active.jj_queue = NULL; sc->sc_rootmount = root_mount_hold("GJOURNAL"); GJ_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount); callout_init(&sc->sc_callout, 1); if (md->md_type != GJ_TYPE_COMPLETE) { /* * Journal and data are on separate providers. * At this point we have only one of them. * We setup a timeout in case the other part will not * appear, so we won't wait forever. */ callout_reset(&sc->sc_callout, 5 * hz, g_journal_timeout, sc); } } /* Remember type of the data provider. */ if (md->md_type & GJ_TYPE_DATA) sc->sc_orig_type = md->md_type; sc->sc_type |= md->md_type; cp = NULL; if (md->md_type & GJ_TYPE_DATA) { if (md->md_flags & GJ_FLAG_CLEAN) sc->sc_flags |= GJF_DEVICE_CLEAN; if (md->md_flags & GJ_FLAG_CHECKSUM) sc->sc_flags |= GJF_DEVICE_CHECKSUM; cp = g_new_consumer(gp); error = g_attach(cp, pp); KASSERT(error == 0, ("Cannot attach to %s (error=%d).", pp->name, error)); error = g_access(cp, 1, 1, 1); if (error != 0) { GJ_DEBUG(0, "Cannot access %s (error=%d).", pp->name, error); g_journal_destroy(sc); return (NULL); } sc->sc_dconsumer = cp; sc->sc_mediasize = pp->mediasize - pp->sectorsize; sc->sc_sectorsize = pp->sectorsize; sc->sc_jstart = md->md_jstart; sc->sc_jend = md->md_jend; if (md->md_provider[0] != '\0') sc->sc_flags |= GJF_DEVICE_HARDCODED; sc->sc_journal_offset = md->md_joffset; sc->sc_journal_id = md->md_jid; sc->sc_journal_previous_id = md->md_jid; } if (md->md_type & GJ_TYPE_JOURNAL) { if (cp == NULL) { cp = g_new_consumer(gp); error = g_attach(cp, pp); KASSERT(error == 0, ("Cannot attach to %s (error=%d).", pp->name, error)); error = g_access(cp, 1, 1, 1); if (error != 0) { GJ_DEBUG(0, "Cannot access %s (error=%d).", pp->name, error); g_journal_destroy(sc); return (NULL); } } else { /* * Journal is on the same provider as data, which means * that data provider ends where journal starts. */ sc->sc_mediasize = md->md_jstart; } sc->sc_jconsumer = cp; } /* Start switcher kproc if needed. */ if (g_journal_switcher_proc == NULL) g_journal_start_switcher(mp); if ((sc->sc_type & GJ_TYPE_COMPLETE) != GJ_TYPE_COMPLETE) { /* Journal is not complete yet. */ return (gp); } else { /* Journal complete, cancel timeout. */ callout_drain(&sc->sc_callout); } error = kproc_create(g_journal_worker, sc, &sc->sc_worker, 0, 0, "g_journal %s", sc->sc_name); if (error != 0) { GJ_DEBUG(0, "Cannot create worker thread for %s.journal.", sc->sc_name); g_journal_destroy(sc); return (NULL); } return (gp); } static void g_journal_destroy_consumer(void *arg, int flags __unused) { struct g_consumer *cp; g_topology_assert(); cp = arg; g_detach(cp); g_destroy_consumer(cp); } static int g_journal_destroy(struct g_journal_softc *sc) { struct g_geom *gp; struct g_provider *pp; struct g_consumer *cp; g_topology_assert(); if (sc == NULL) return (ENXIO); gp = sc->sc_geom; pp = LIST_FIRST(&gp->provider); if (pp != NULL) { if (pp->acr != 0 || pp->acw != 0 || pp->ace != 0) { GJ_DEBUG(1, "Device %s is still open (r%dw%de%d).", pp->name, pp->acr, pp->acw, pp->ace); return (EBUSY); } g_error_provider(pp, ENXIO); g_journal_flush(sc); g_journal_flush_send(sc); g_journal_switch(sc); } sc->sc_flags |= (GJF_DEVICE_DESTROY | GJF_DEVICE_CLEAN); g_topology_unlock(); if (sc->sc_rootmount != NULL) { GJ_DEBUG(1, "root_mount_rel %p", sc->sc_rootmount); root_mount_rel(sc->sc_rootmount); sc->sc_rootmount = NULL; } callout_drain(&sc->sc_callout); mtx_lock(&sc->sc_mtx); wakeup(sc); while (sc->sc_worker != NULL) msleep(&sc->sc_worker, &sc->sc_mtx, PRIBIO, "gj:destroy", 0); mtx_unlock(&sc->sc_mtx); if (pp != NULL) { GJ_DEBUG(1, "Marking %s as clean.", sc->sc_name); g_journal_metadata_update(sc); g_topology_lock(); g_wither_provider(pp, ENXIO); } else { g_topology_lock(); } mtx_destroy(&sc->sc_mtx); if (sc->sc_current_count != 0) { GJ_DEBUG(0, "Warning! Number of current requests %d.", sc->sc_current_count); } gp->softc = NULL; LIST_FOREACH(cp, &gp->consumer, consumer) { if (cp->acr + cp->acw + cp->ace > 0) g_access(cp, -1, -1, -1); /* * We keep all consumers open for writting, so if I'll detach * and destroy consumer here, I'll get providers for taste, so * journal will be started again. * Sending an event here, prevents this from happening. */ g_post_event(g_journal_destroy_consumer, cp, M_WAITOK, NULL); } g_wither_geom(gp, ENXIO); free(sc, M_JOURNAL); return (0); } static void g_journal_taste_orphan(struct g_consumer *cp) { KASSERT(1 == 0, ("%s called while tasting %s.", __func__, cp->provider->name)); } static struct g_geom * g_journal_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) { struct g_journal_metadata md; struct g_consumer *cp; struct g_geom *gp; int error; g_topology_assert(); g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name); GJ_DEBUG(2, "Tasting %s.", pp->name); if (pp->geom->class == mp) return (NULL); gp = g_new_geomf(mp, "journal:taste"); /* This orphan function should be never called. */ gp->orphan = g_journal_taste_orphan; cp = g_new_consumer(gp); error = g_attach(cp, pp); if (error == 0) { error = g_journal_metadata_read(cp, &md); g_detach(cp); } g_destroy_consumer(cp); g_destroy_geom(gp); if (error != 0) return (NULL); gp = NULL; if (md.md_provider[0] != '\0' && !g_compare_names(md.md_provider, pp->name)) return (NULL); if (md.md_provsize != 0 && md.md_provsize != pp->mediasize) return (NULL); if (g_journal_debug >= 2) journal_metadata_dump(&md); gp = g_journal_create(mp, pp, &md); return (gp); } static struct g_journal_softc * g_journal_find_device(struct g_class *mp, const char *name) { struct g_journal_softc *sc; struct g_geom *gp; struct g_provider *pp; if (strncmp(name, _PATH_DEV, 5) == 0) name += 5; LIST_FOREACH(gp, &mp->geom, geom) { sc = gp->softc; if (sc == NULL) continue; if (sc->sc_flags & GJF_DEVICE_DESTROY) continue; if ((sc->sc_type & GJ_TYPE_COMPLETE) != GJ_TYPE_COMPLETE) continue; pp = LIST_FIRST(&gp->provider); if (strcmp(sc->sc_name, name) == 0) return (sc); if (pp != NULL && strcmp(pp->name, name) == 0) return (sc); } return (NULL); } static void g_journal_ctl_destroy(struct gctl_req *req, struct g_class *mp) { struct g_journal_softc *sc; const char *name; char param[16]; int *nargs; int error, i; g_topology_assert(); nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); if (nargs == NULL) { gctl_error(req, "No '%s' argument.", "nargs"); return; } if (*nargs <= 0) { gctl_error(req, "Missing device(s)."); return; } for (i = 0; i < *nargs; i++) { snprintf(param, sizeof(param), "arg%d", i); name = gctl_get_asciiparam(req, param); if (name == NULL) { gctl_error(req, "No 'arg%d' argument.", i); return; } sc = g_journal_find_device(mp, name); if (sc == NULL) { gctl_error(req, "No such device: %s.", name); return; } error = g_journal_destroy(sc); if (error != 0) { gctl_error(req, "Cannot destroy device %s (error=%d).", LIST_FIRST(&sc->sc_geom->provider)->name, error); return; } } } static void g_journal_ctl_sync(struct gctl_req *req __unused, struct g_class *mp __unused) { g_topology_assert(); g_topology_unlock(); g_journal_sync_requested++; wakeup(&g_journal_switcher_state); while (g_journal_sync_requested > 0) tsleep(&g_journal_sync_requested, PRIBIO, "j:sreq", hz / 2); g_topology_lock(); } static void g_journal_config(struct gctl_req *req, struct g_class *mp, const char *verb) { uint32_t *version; g_topology_assert(); version = gctl_get_paraml(req, "version", sizeof(*version)); if (version == NULL) { gctl_error(req, "No '%s' argument.", "version"); return; } if (*version != G_JOURNAL_VERSION) { gctl_error(req, "Userland and kernel parts are out of sync."); return; } if (strcmp(verb, "destroy") == 0 || strcmp(verb, "stop") == 0) { g_journal_ctl_destroy(req, mp); return; } else if (strcmp(verb, "sync") == 0) { g_journal_ctl_sync(req, mp); return; } gctl_error(req, "Unknown verb."); } static void g_journal_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp) { struct g_journal_softc *sc; g_topology_assert(); sc = gp->softc; if (sc == NULL) return; if (pp != NULL) { /* Nothing here. */ } else if (cp != NULL) { int first = 1; sbuf_printf(sb, "%s", indent); if (cp == sc->sc_dconsumer) { sbuf_cat(sb, "Data"); first = 0; } if (cp == sc->sc_jconsumer) { if (!first) sbuf_cat(sb, ","); sbuf_cat(sb, "Journal"); } sbuf_cat(sb, "\n"); if (cp == sc->sc_jconsumer) { sbuf_printf(sb, "%jd\n", (intmax_t)sc->sc_jstart); sbuf_printf(sb, "%jd\n", (intmax_t)sc->sc_jend); } } else { sbuf_printf(sb, "%s%u\n", indent, (u_int)sc->sc_id); } } static eventhandler_tag g_journal_event_shutdown = NULL; static eventhandler_tag g_journal_event_lowmem = NULL; static void g_journal_shutdown(void *arg, int howto __unused) { struct g_class *mp; struct g_geom *gp, *gp2; if (KERNEL_PANICKED()) return; mp = arg; g_topology_lock(); LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) { if (gp->softc == NULL) continue; GJ_DEBUG(0, "Shutting down geom %s.", gp->name); g_journal_destroy(gp->softc); } g_topology_unlock(); } /* * Free cached requests from inactive queue in case of low memory. * We free GJ_FREE_AT_ONCE elements at once. */ #define GJ_FREE_AT_ONCE 4 static void g_journal_lowmem(void *arg, int howto __unused) { struct g_journal_softc *sc; struct g_class *mp; struct g_geom *gp; struct bio *bp; u_int nfree = GJ_FREE_AT_ONCE; g_journal_stats_low_mem++; mp = arg; g_topology_lock(); LIST_FOREACH(gp, &mp->geom, geom) { sc = gp->softc; if (sc == NULL || (sc->sc_flags & GJF_DEVICE_DESTROY)) continue; mtx_lock(&sc->sc_mtx); for (bp = sc->sc_inactive.jj_queue; nfree > 0 && bp != NULL; nfree--, bp = bp->bio_next) { /* * This is safe to free the bio_data, because: * 1. If bio_data is NULL it will be read from the * inactive journal. * 2. If bp is sent down, it is first removed from the * inactive queue, so it's impossible to free the * data from under in-flight bio. * On the other hand, freeing elements from the active * queue, is not safe. */ if (bp->bio_data != NULL) { GJ_DEBUG(2, "Freeing data from %s.", sc->sc_name); gj_free(bp->bio_data, bp->bio_length); bp->bio_data = NULL; } } mtx_unlock(&sc->sc_mtx); if (nfree == 0) break; } g_topology_unlock(); } static void g_journal_switcher(void *arg); static void g_journal_init(struct g_class *mp) { /* Pick a conservative value if provided value sucks. */ if (g_journal_cache_divisor <= 0 || (vm_kmem_size / g_journal_cache_divisor == 0)) { g_journal_cache_divisor = 5; } if (g_journal_cache_limit > 0) { g_journal_cache_limit = vm_kmem_size / g_journal_cache_divisor; g_journal_cache_low = (g_journal_cache_limit / 100) * g_journal_cache_switch; } g_journal_event_shutdown = EVENTHANDLER_REGISTER(shutdown_post_sync, g_journal_shutdown, mp, EVENTHANDLER_PRI_FIRST); if (g_journal_event_shutdown == NULL) GJ_DEBUG(0, "Warning! Cannot register shutdown event."); g_journal_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, g_journal_lowmem, mp, EVENTHANDLER_PRI_FIRST); if (g_journal_event_lowmem == NULL) GJ_DEBUG(0, "Warning! Cannot register lowmem event."); } static void g_journal_fini(struct g_class *mp) { if (g_journal_event_shutdown != NULL) { EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_journal_event_shutdown); } if (g_journal_event_lowmem != NULL) EVENTHANDLER_DEREGISTER(vm_lowmem, g_journal_event_lowmem); if (g_journal_switcher_proc != NULL) g_journal_stop_switcher(); } DECLARE_GEOM_CLASS(g_journal_class, g_journal); static const struct g_journal_desc * g_journal_find_desc(const char *fstype) { const struct g_journal_desc *desc; int i; for (desc = g_journal_filesystems[i = 0]; desc != NULL; desc = g_journal_filesystems[++i]) { if (strcmp(desc->jd_fstype, fstype) == 0) break; } return (desc); } static void g_journal_switch_wait(struct g_journal_softc *sc) { struct bintime bt; mtx_assert(&sc->sc_mtx, MA_OWNED); if (g_journal_debug >= 2) { if (sc->sc_flush_in_progress > 0) { GJ_DEBUG(2, "%d requests flushing.", sc->sc_flush_in_progress); } if (sc->sc_copy_in_progress > 0) { GJ_DEBUG(2, "%d requests copying.", sc->sc_copy_in_progress); } if (sc->sc_flush_count > 0) { GJ_DEBUG(2, "%d requests to flush.", sc->sc_flush_count); } if (sc->sc_delayed_count > 0) { GJ_DEBUG(2, "%d requests delayed.", sc->sc_delayed_count); } } g_journal_stats_switches++; if (sc->sc_copy_in_progress > 0) g_journal_stats_wait_for_copy++; GJ_TIMER_START(1, &bt); sc->sc_flags &= ~GJF_DEVICE_BEFORE_SWITCH; sc->sc_flags |= GJF_DEVICE_SWITCH; wakeup(sc); while (sc->sc_flags & GJF_DEVICE_SWITCH) { msleep(&sc->sc_journal_copying, &sc->sc_mtx, PRIBIO, "gj:switch", 0); } GJ_TIMER_STOP(1, &bt, "Switch time of %s", sc->sc_name); } static void g_journal_do_switch(struct g_class *classp) { struct g_journal_softc *sc; const struct g_journal_desc *desc; struct g_geom *gp; struct mount *mp; struct bintime bt; char *mountpoint; int error, save; g_topology_lock(); LIST_FOREACH(gp, &classp->geom, geom) { sc = gp->softc; if (sc == NULL) continue; if (sc->sc_flags & GJF_DEVICE_DESTROY) continue; if ((sc->sc_type & GJ_TYPE_COMPLETE) != GJ_TYPE_COMPLETE) continue; mtx_lock(&sc->sc_mtx); sc->sc_flags |= GJF_DEVICE_BEFORE_SWITCH; mtx_unlock(&sc->sc_mtx); } g_topology_unlock(); mtx_lock(&mountlist_mtx); TAILQ_FOREACH(mp, &mountlist, mnt_list) { if (mp->mnt_gjprovider == NULL) continue; if (mp->mnt_flag & MNT_RDONLY) continue; desc = g_journal_find_desc(mp->mnt_stat.f_fstypename); if (desc == NULL) continue; if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) continue; /* mtx_unlock(&mountlist_mtx) was done inside vfs_busy() */ g_topology_lock(); sc = g_journal_find_device(classp, mp->mnt_gjprovider); g_topology_unlock(); if (sc == NULL) { GJ_DEBUG(0, "Cannot find journal geom for %s.", mp->mnt_gjprovider); goto next; } else if (JEMPTY(sc)) { mtx_lock(&sc->sc_mtx); sc->sc_flags &= ~GJF_DEVICE_BEFORE_SWITCH; mtx_unlock(&sc->sc_mtx); GJ_DEBUG(3, "No need for %s switch.", sc->sc_name); goto next; } mountpoint = mp->mnt_stat.f_mntonname; error = vn_start_write(NULL, &mp, V_WAIT); if (error != 0) { GJ_DEBUG(0, "vn_start_write(%s) failed (error=%d).", mountpoint, error); goto next; } save = curthread_pflags_set(TDP_SYNCIO); GJ_TIMER_START(1, &bt); vfs_periodic(mp, MNT_NOWAIT); GJ_TIMER_STOP(1, &bt, "Msync time of %s", mountpoint); GJ_TIMER_START(1, &bt); error = VFS_SYNC(mp, MNT_NOWAIT); if (error == 0) GJ_TIMER_STOP(1, &bt, "Sync time of %s", mountpoint); else { GJ_DEBUG(0, "Cannot sync file system %s (error=%d).", mountpoint, error); } curthread_pflags_restore(save); vn_finished_write(mp); if (error != 0) goto next; /* * Send BIO_FLUSH before freezing the file system, so it can be * faster after the freeze. */ GJ_TIMER_START(1, &bt); g_journal_flush_cache(sc); GJ_TIMER_STOP(1, &bt, "BIO_FLUSH time of %s", sc->sc_name); GJ_TIMER_START(1, &bt); error = vfs_write_suspend(mp, VS_SKIP_UNMOUNT); GJ_TIMER_STOP(1, &bt, "Suspend time of %s", mountpoint); if (error != 0) { GJ_DEBUG(0, "Cannot suspend file system %s (error=%d).", mountpoint, error); goto next; } error = desc->jd_clean(mp); if (error != 0) goto next; mtx_lock(&sc->sc_mtx); g_journal_switch_wait(sc); mtx_unlock(&sc->sc_mtx); vfs_write_resume(mp, 0); next: mtx_lock(&mountlist_mtx); vfs_unbusy(mp); } mtx_unlock(&mountlist_mtx); sc = NULL; for (;;) { g_topology_lock(); LIST_FOREACH(gp, &g_journal_class.geom, geom) { sc = gp->softc; if (sc == NULL) continue; mtx_lock(&sc->sc_mtx); if ((sc->sc_type & GJ_TYPE_COMPLETE) == GJ_TYPE_COMPLETE && !(sc->sc_flags & GJF_DEVICE_DESTROY) && (sc->sc_flags & GJF_DEVICE_BEFORE_SWITCH)) { break; } mtx_unlock(&sc->sc_mtx); sc = NULL; } g_topology_unlock(); if (sc == NULL) break; mtx_assert(&sc->sc_mtx, MA_OWNED); g_journal_switch_wait(sc); mtx_unlock(&sc->sc_mtx); } } static void g_journal_start_switcher(struct g_class *mp) { int error; g_topology_assert(); MPASS(g_journal_switcher_proc == NULL); g_journal_switcher_state = GJ_SWITCHER_WORKING; error = kproc_create(g_journal_switcher, mp, &g_journal_switcher_proc, 0, 0, "g_journal switcher"); KASSERT(error == 0, ("Cannot create switcher thread.")); } static void g_journal_stop_switcher(void) { g_topology_assert(); MPASS(g_journal_switcher_proc != NULL); g_journal_switcher_state = GJ_SWITCHER_DIE; wakeup(&g_journal_switcher_state); while (g_journal_switcher_state != GJ_SWITCHER_DIED) tsleep(&g_journal_switcher_state, PRIBIO, "jfini:wait", hz / 5); GJ_DEBUG(1, "Switcher died."); g_journal_switcher_proc = NULL; } /* * TODO: Kill switcher thread on last geom destruction? */ static void g_journal_switcher(void *arg) { struct g_class *mp; struct bintime bt; int error; mp = arg; curthread->td_pflags |= TDP_NORUNNINGBUF; for (;;) { g_journal_switcher_wokenup = 0; error = tsleep(&g_journal_switcher_state, PRIBIO, "jsw:wait", g_journal_switch_time * hz); if (g_journal_switcher_state == GJ_SWITCHER_DIE) { g_journal_switcher_state = GJ_SWITCHER_DIED; GJ_DEBUG(1, "Switcher exiting."); wakeup(&g_journal_switcher_state); kproc_exit(0); } if (error == 0 && g_journal_sync_requested == 0) { GJ_DEBUG(1, "Out of cache, force switch (used=%jd " "limit=%jd).", (intmax_t)g_journal_cache_used, (intmax_t)g_journal_cache_limit); } GJ_TIMER_START(1, &bt); g_journal_do_switch(mp); GJ_TIMER_STOP(1, &bt, "Entire switch time"); if (g_journal_sync_requested > 0) { g_journal_sync_requested = 0; wakeup(&g_journal_sync_requested); } } } diff --git a/sys/i386/i386/perfmon.c b/sys/i386/i386/perfmon.c index 3f35b59faa80..cb8c0acfc0dc 100644 --- a/sys/i386/i386/perfmon.c +++ b/sys/i386/i386/perfmon.c @@ -1,406 +1,406 @@ /*- * Copyright 1996 Massachusetts Institute of Technology * * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby * granted, provided that both the above copyright notice and this * permission notice appear in all copies, that both the above * copyright notice and this permission notice appear in all * supporting documentation, and that the name of M.I.T. not be used * in advertising or publicity pertaining to distribution of the * software without specific, written prior permission. M.I.T. makes * no representations about the suitability of this software for any * purpose. It is provided "as is" without express or implied * warranty. * * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #ifndef SMP #include #endif #include #include #include static int perfmon_inuse; static int perfmon_cpuok; #ifndef SMP static int msr_ctl[NPMC]; #endif static int msr_pmc[NPMC]; static unsigned int ctl_shadow[NPMC]; static quad_t pmc_shadow[NPMC]; /* used when ctr is stopped on P5 */ static int (*writectl)(int); #ifndef SMP static int writectl5(int); static int writectl6(int); #endif static d_close_t perfmon_close; static d_open_t perfmon_open; static d_ioctl_t perfmon_ioctl; /* - * XXX perfmon_init_dev(void *) is a split from the perfmon_init() funtion. + * XXX perfmon_init_dev(void *) is a split from the perfmon_init() function. * This solves a problem for DEVFS users. It loads the "perfmon" driver after * the DEVFS subsystem has been kicked into action. The SI_ORDER_ANY is to * assure that it is the most lowest priority task which, guarantees the * above. */ static void perfmon_init_dev(void *); SYSINIT(cpu, SI_SUB_DRIVERS, SI_ORDER_ANY, perfmon_init_dev, NULL); static struct cdevsw perfmon_cdevsw = { .d_version = D_VERSION, .d_flags = D_NEEDGIANT, .d_open = perfmon_open, .d_close = perfmon_close, .d_ioctl = perfmon_ioctl, .d_name = "perfmon", }; /* * Must be called after cpu_class is set up. */ void perfmon_init(void) { #ifndef SMP switch(cpu_class) { case CPUCLASS_586: perfmon_cpuok = 1; msr_ctl[0] = MSR_P5_CESR; msr_ctl[1] = MSR_P5_CESR; msr_pmc[0] = MSR_P5_CTR0; msr_pmc[1] = MSR_P5_CTR1; writectl = writectl5; break; case CPUCLASS_686: perfmon_cpuok = 1; msr_ctl[0] = MSR_EVNTSEL0; msr_ctl[1] = MSR_EVNTSEL1; msr_pmc[0] = MSR_PERFCTR0; msr_pmc[1] = MSR_PERFCTR1; writectl = writectl6; break; default: perfmon_cpuok = 0; break; } #endif /* SMP */ } static void perfmon_init_dev(dummy) void *dummy; { make_dev(&perfmon_cdevsw, 32, UID_ROOT, GID_KMEM, 0640, "perfmon"); } int perfmon_avail(void) { return perfmon_cpuok; } int perfmon_setup(int pmc, unsigned int control) { register_t saveintr; if (pmc < 0 || pmc >= NPMC) return EINVAL; perfmon_inuse |= (1 << pmc); control &= ~(PMCF_SYS_FLAGS << 16); saveintr = intr_disable(); ctl_shadow[pmc] = control; writectl(pmc); wrmsr(msr_pmc[pmc], pmc_shadow[pmc] = 0); intr_restore(saveintr); return 0; } int perfmon_get(int pmc, unsigned int *control) { if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { *control = ctl_shadow[pmc]; return 0; } return EBUSY; /* XXX reversed sense */ } int perfmon_fini(int pmc) { if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { perfmon_stop(pmc); ctl_shadow[pmc] = 0; perfmon_inuse &= ~(1 << pmc); return 0; } return EBUSY; /* XXX reversed sense */ } int perfmon_start(int pmc) { register_t saveintr; if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { saveintr = intr_disable(); ctl_shadow[pmc] |= (PMCF_EN << 16); wrmsr(msr_pmc[pmc], pmc_shadow[pmc]); writectl(pmc); intr_restore(saveintr); return 0; } return EBUSY; } int perfmon_stop(int pmc) { register_t saveintr; if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { saveintr = intr_disable(); pmc_shadow[pmc] = rdmsr(msr_pmc[pmc]) & 0xffffffffffULL; ctl_shadow[pmc] &= ~(PMCF_EN << 16); writectl(pmc); intr_restore(saveintr); return 0; } return EBUSY; } int perfmon_read(int pmc, quad_t *val) { if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { if (ctl_shadow[pmc] & (PMCF_EN << 16)) *val = rdmsr(msr_pmc[pmc]) & 0xffffffffffULL; else *val = pmc_shadow[pmc]; return 0; } return EBUSY; } int perfmon_reset(int pmc) { if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { wrmsr(msr_pmc[pmc], pmc_shadow[pmc] = 0); return 0; } return EBUSY; } #ifndef SMP /* * Unfortunately, the performance-monitoring registers are laid out * differently in the P5 and P6. We keep everything in P6 format * internally (except for the event code), and convert to P5 * format as needed on those CPUs. The writectl function pointer * is set up to point to one of these functions by perfmon_init(). */ int writectl6(int pmc) { if (pmc > 0 && !(ctl_shadow[pmc] & (PMCF_EN << 16))) { wrmsr(msr_ctl[pmc], 0); } else { wrmsr(msr_ctl[pmc], ctl_shadow[pmc]); } return 0; } #define P5FLAG_P 0x200 #define P5FLAG_E 0x100 #define P5FLAG_USR 0x80 #define P5FLAG_OS 0x40 int writectl5(int pmc) { quad_t newval = 0; if (ctl_shadow[1] & (PMCF_EN << 16)) { if (ctl_shadow[1] & (PMCF_USR << 16)) newval |= P5FLAG_USR << 16; if (ctl_shadow[1] & (PMCF_OS << 16)) newval |= P5FLAG_OS << 16; if (!(ctl_shadow[1] & (PMCF_E << 16))) newval |= P5FLAG_E << 16; newval |= (ctl_shadow[1] & 0x3f) << 16; } if (ctl_shadow[0] & (PMCF_EN << 16)) { if (ctl_shadow[0] & (PMCF_USR << 16)) newval |= P5FLAG_USR; if (ctl_shadow[0] & (PMCF_OS << 16)) newval |= P5FLAG_OS; if (!(ctl_shadow[0] & (PMCF_E << 16))) newval |= P5FLAG_E; newval |= ctl_shadow[0] & 0x3f; } wrmsr(msr_ctl[0], newval); return 0; /* XXX should check for unimplemented bits */ } #endif /* !SMP */ /* * Now the user-mode interface, called from a subdevice of mem.c. */ static int writer; static int writerpmc; static int perfmon_open(struct cdev *dev, int flags, int fmt, struct thread *td) { if (!perfmon_cpuok) return ENXIO; if (flags & FWRITE) { if (writer) { return EBUSY; } else { writer = 1; writerpmc = 0; } } return 0; } static int perfmon_close(struct cdev *dev, int flags, int fmt, struct thread *td) { if (flags & FWRITE) { int i; for (i = 0; i < NPMC; i++) { if (writerpmc & (1 << i)) perfmon_fini(i); } writer = 0; } return 0; } static int perfmon_ioctl(struct cdev *dev, u_long cmd, caddr_t param, int flags, struct thread *td) { struct pmc *pmc; struct pmc_data *pmcd; struct pmc_tstamp *pmct; uint64_t freq; int *ip; int rv; switch(cmd) { case PMIOSETUP: if (!(flags & FWRITE)) return EPERM; pmc = (struct pmc *)param; rv = perfmon_setup(pmc->pmc_num, pmc->pmc_val); if (!rv) { writerpmc |= (1 << pmc->pmc_num); } break; case PMIOGET: pmc = (struct pmc *)param; rv = perfmon_get(pmc->pmc_num, &pmc->pmc_val); break; case PMIOSTART: if (!(flags & FWRITE)) return EPERM; ip = (int *)param; rv = perfmon_start(*ip); break; case PMIOSTOP: if (!(flags & FWRITE)) return EPERM; ip = (int *)param; rv = perfmon_stop(*ip); break; case PMIORESET: if (!(flags & FWRITE)) return EPERM; ip = (int *)param; rv = perfmon_reset(*ip); break; case PMIOREAD: pmcd = (struct pmc_data *)param; rv = perfmon_read(pmcd->pmcd_num, &pmcd->pmcd_value); break; case PMIOTSTAMP: freq = atomic_load_acq_64(&tsc_freq); if (freq == 0) { rv = ENOTTY; break; } pmct = (struct pmc_tstamp *)param; /* XXX interface loses precision. */ pmct->pmct_rate = freq / 1000000; pmct->pmct_value = rdtsc(); rv = 0; break; default: rv = ENOTTY; } return rv; } diff --git a/usr.bin/tip/tip/tip.h b/usr.bin/tip/tip/tip.h index 298b08cb6692..a5d53b9a6aad 100644 --- a/usr.bin/tip/tip/tip.h +++ b/usr.bin/tip/tip/tip.h @@ -1,358 +1,358 @@ /* $OpenBSD: tip.h,v 1.27 2006/08/18 03:06:18 jason Exp $ */ /* $NetBSD: tip.h,v 1.7 1997/04/20 00:02:46 mellon Exp $ */ /* $FreeBSD$ */ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1989, 1993 * The Regents of the University of California. All rights reserved. * * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)tip.h 8.1 (Berkeley) 6/6/93 */ /* * tip - terminal interface program */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef EXTERN #define EXTERN extern #endif /* * Remote host attributes */ EXTERN char *DV; /* UNIX device(s) to open */ EXTERN char *EL; /* chars marking an EOL */ EXTERN char *CM; /* initial connection message */ EXTERN char *IE; /* EOT to expect on input */ EXTERN char *OE; /* EOT to send to complete FT */ EXTERN char *CU; /* call unit if making a phone call */ EXTERN char *AT; /* acu type */ EXTERN char *PN; /* phone number(s) */ EXTERN char *DI; /* disconnect string */ EXTERN char *PA; /* parity to be generated */ EXTERN char *PH; /* phone number file */ EXTERN char *RM; /* remote file name */ EXTERN char *HO; /* host name */ EXTERN long BR; /* line speed for conversation */ EXTERN long FS; /* frame size for transfers */ EXTERN short DU; /* this host is dialed up */ EXTERN short HW; /* this device is hardwired, see hunt.c */ EXTERN char *ES; /* escape character */ EXTERN char *EX; /* exceptions */ EXTERN char *FO; /* force (literal next) char*/ EXTERN char *RC; /* raise character */ EXTERN char *RE; /* script record file */ EXTERN char *PR; /* remote prompt */ EXTERN long DL; /* line delay for file transfers to remote */ EXTERN long CL; /* char delay for file transfers to remote */ EXTERN long ET; /* echocheck timeout */ EXTERN long LD; /* line disc */ EXTERN short HD; /* this host is half duplex - do local echo */ /* * String value table */ typedef struct { char *v_name; /* whose name is it */ char v_type; /* for interpreting set's */ char v_access; /* protection of touchy ones */ char *v_abrev; /* possible abbreviation */ char *v_value; /* casted to a union later */ } value_t; #define STRING 01 /* string valued */ #define BOOL 02 /* true-false value */ #define NUMBER 04 /* numeric value */ #define CHAR 010 /* character value */ #define WRITE 01 /* write access to variable */ #define READ 02 /* read access */ #define CHANGED 01 /* low bit is used to show modification */ #define PUBLIC 1 /* public access rights */ #define PRIVATE 03 /* private to definer */ #define ROOT 05 /* root defined */ #define TRUE 1 #define FALSE 0 #define ENVIRON 020 /* initialize out of the environment */ #define IREMOTE 040 /* initialize out of remote structure */ #define INIT 0100 /* static data space used for initialization */ #define TMASK 017 /* * Definition of ACU line description */ typedef struct { char *acu_name; int (*acu_dialer)(char *, char *); void (*acu_disconnect)(void); void (*acu_abort)(void); } acu_t; #define equal(a, b) (strcmp(a,b)==0)/* A nice function to string compare */ /* * variable manipulation stuff -- * if we defined the value entry in value_t, then we couldn't * initialize it in vars.c, so we cast it as needed to keep lint * happy. */ #define value(v) vtable[v].v_value #define lvalue(v) (long)(intptr_t)vtable[v].v_value #define number(v) ((long)(intptr_t)(v)) #define boolean(v) ((short)(intptr_t)(v)) #define character(v) ((char)(intptr_t)(v)) #define setnumber(v,n) do { (v) = (char *)(intptr_t)(n); } while (0) #define setboolean(v,n) do { (v) = (char *)(intptr_t)(n); } while (0) #define setcharacter(v,n) do { (v) = (char *)(intptr_t)(n); } while (0) /* * Escape command table definitions -- * lookup in this table is performed when ``escapec'' is recognized * at the beginning of a line (as defined by the eolmarks variable). */ typedef struct { char e_char; /* char to match on */ char e_flags; /* experimental, privileged */ char *e_help; /* help string */ void (*e_func)(int); /* command */ } esctable_t; #define NORM 00 /* normal protection, execute anyone */ #define EXP 01 /* experimental, mark it with a `*' on help */ #define PRIV 02 /* privileged, root execute only */ extern int vflag; /* verbose during reading of .tiprc file */ extern int noesc; /* no escape `~' char */ extern value_t vtable[]; /* variable table */ #ifndef ACULOG #define logent(a, b, c, d) #define loginit() #endif /* * Definition of indices into variable table so * value(DEFINE) turns into a static address. */ #define BEAUTIFY 0 #define BAUDRATE 1 #define DIALTIMEOUT 2 #define EOFREAD 3 #define EOFWRITE 4 #define EOL 5 #define ESCAPE 6 #define EXCEPTIONS 7 #define FORCE 8 #define FRAMESIZE 9 #define HOST 10 #define LOG 11 #define PHONES 12 #define PROMPT 13 #define RAISE 14 #define RAISECHAR 15 #define RECORD 16 #define REMOTE 17 #define SCRIPT 18 #define TABEXPAND 19 #define VERBOSE 20 #define SHELL 21 #define HOME 22 #define ECHOCHECK 23 #define DISCONNECT 24 #define TAND 25 #define LDELAY 26 #define CDELAY 27 #define ETIMEOUT 28 #define RAWFTP 29 #define HALFDUPLEX 30 #define LECHO 31 #define PARITY 32 #define HARDWAREFLOW 33 #define LINEDISC 34 #define DC 35 #define NOVAL ((value_t *)NULL) #define NOACU ((acu_t *)NULL) #define NOSTR ((char *)NULL) #define NOFILE ((FILE *)NULL) #define NOPWD ((struct passwd *)0) EXTERN struct termios term; /* current mode of terminal */ EXTERN struct termios defterm; /* initial mode of terminal */ EXTERN struct termios defchars; /* current mode with initial chars */ EXTERN int gotdefterm; EXTERN FILE *fscript; /* FILE for scripting */ EXTERN int fildes[2]; /* file transfer synchronization channel */ -EXTERN int repdes[2]; /* read process sychronization channel */ +EXTERN int repdes[2]; /* read process synchronization channel */ EXTERN int FD; /* open file descriptor to remote host */ EXTERN int AC; /* open file descriptor to dialer (v831 only) */ EXTERN int vflag; /* print .tiprc initialization sequence */ EXTERN int noesc; /* no `~' escape char */ EXTERN int sfd; /* for ~< operation */ EXTERN pid_t tipin_pid; /* pid of tipin */ EXTERN pid_t tipout_pid; /* pid of tipout */ EXTERN uid_t uid, euid; /* real and effective user id's */ EXTERN gid_t gid, egid; /* real and effective group id's */ EXTERN int stop; /* stop transfer session flag */ EXTERN int quit; /* same; but on other end */ EXTERN int intflag; /* recognized interrupt */ EXTERN int stoprompt; /* for interrupting a prompt session */ EXTERN int timedout; /* ~> transfer timedout */ EXTERN int cumode; /* simulating the "cu" program */ EXTERN int bits8; /* terminal is 8-bit mode */ #define STRIP_PAR (bits8 ? 0377 : 0177) EXTERN char fname[PATH_MAX]; /* file name buffer for ~< */ EXTERN char copyname[PATH_MAX]; /* file name buffer for ~> */ EXTERN char ccc; /* synchronization character */ EXTERN char *uucplock; /* name of lock file for uucp's */ EXTERN int odisc; /* initial tty line discipline */ extern int disc; /* current tty discpline */ extern char *__progname; /* program name */ char *con(void); char *ctrl(char); char *expand(char *); char *getremote(char *); char *interp(char *); int any(int, char *); int biz22w_dialer(char *, char *); int biz22f_dialer(char *, char *); int biz31w_dialer(char *, char *); int biz31f_dialer(char *, char *); int cour_dialer(char *, char *); int df02_dialer(char *, char *); int df03_dialer(char *, char *); int dn_dialer(char *, char *); int hay_dialer(char *, char *); int prompt(char *, char *, size_t); size_t size(char *); int t3000_dialer(char *, char *); int ttysetup(int); int uu_lock(char *); int uu_unlock(char *); int v3451_dialer(char *, char *); int v831_dialer(char *, char *); int ven_dialer(char *, char *); int vstring(char *, char *); long hunt(char *); void biz22_disconnect(void); void biz22_abort(void); void biz31_disconnect(void); void biz31_abort(void); void chdirectory(int); void cleanup(int); void consh(int); void cour_abort(void); void cour_disconnect(void); void cu_put(int); void cu_take(int); void cumain(int, char **); void daemon_uid(void); void df_abort(void); void df_disconnect(void); void disconnect(char *); void dn_abort(void); void dn_disconnect(void); void finish(int); void genbrk(int); void getfl(int); void hay_abort(void); void hay_disconnect(void); void help(int); void listvariables(int); void logent(char *, char *, char *, char *); void loginit(void); void parwrite(int, char *, size_t); void pipefile(int); void pipeout(int); void raw(void); void sendfile(int); void setparity(char *); void setscript(void); void shell(int); void shell_uid(void); void suspend(int); void t3000_disconnect(void); void t3000_abort(void); void timeout(int); void tipabort(char *); void tipout(void); void user_uid(void); void unexcl(void); void unraw(void); void v3451_abort(void); void v3451_disconnect(void); void v831_disconnect(void); void v831_abort(void); void variable(int); void ven_disconnect(void); void ven_abort(void); void vinit(void); void vlex(char *);