Index: head/sys/amd64/amd64/bpf_jit_machdep.c =================================================================== --- head/sys/amd64/amd64/bpf_jit_machdep.c (revision 367185) +++ head/sys/amd64/amd64/bpf_jit_machdep.c (revision 367186) @@ -1,653 +1,653 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) * Copyright (C) 2005-2017 Jung-uk Kim * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Politecnico di Torino nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef _KERNEL #include "opt_bpf.h" #include #include #include #include #include #include #include #else #include #include #include #include #endif #include #include #include #include /* * Emit routine to update the jump table. */ static void emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) { if (stream->refs != NULL) (stream->refs)[stream->bpf_pc] += len; stream->cur_ip += len; } /* * Emit routine to output the actual binary code. */ static void emit_code(bpf_bin_stream *stream, u_int value, u_int len) { switch (len) { case 1: stream->ibuf[stream->cur_ip] = (u_char)value; stream->cur_ip++; break; case 2: *((u_short *)(void *)(stream->ibuf + stream->cur_ip)) = (u_short)value; stream->cur_ip += 2; break; case 4: *((u_int *)(void *)(stream->ibuf + stream->cur_ip)) = value; stream->cur_ip += 4; break; } return; } /* * Scan the filter program and find possible optimization. */ static int bpf_jit_optimize(struct bpf_insn *prog, u_int nins) { int flags; u_int i; /* Do we return immediately? */ if (BPF_CLASS(prog[0].code) == BPF_RET) return (BPF_JIT_FRET); for (flags = 0, i = 0; i < nins; i++) { switch (prog[i].code) { case BPF_LD|BPF_W|BPF_ABS: case BPF_LD|BPF_H|BPF_ABS: case BPF_LD|BPF_B|BPF_ABS: case BPF_LD|BPF_W|BPF_IND: case BPF_LD|BPF_H|BPF_IND: case BPF_LD|BPF_B|BPF_IND: case BPF_LDX|BPF_MSH|BPF_B: flags |= BPF_JIT_FPKT; break; case BPF_LD|BPF_MEM: case BPF_LDX|BPF_MEM: case BPF_ST: case BPF_STX: flags |= BPF_JIT_FMEM; break; case BPF_LD|BPF_W|BPF_LEN: case BPF_LDX|BPF_W|BPF_LEN: flags |= BPF_JIT_FLEN; break; case BPF_JMP|BPF_JA: case BPF_JMP|BPF_JGT|BPF_K: case BPF_JMP|BPF_JGE|BPF_K: case BPF_JMP|BPF_JEQ|BPF_K: case BPF_JMP|BPF_JSET|BPF_K: case BPF_JMP|BPF_JGT|BPF_X: case BPF_JMP|BPF_JGE|BPF_X: case BPF_JMP|BPF_JEQ|BPF_X: case BPF_JMP|BPF_JSET|BPF_X: flags |= BPF_JIT_FJMP; break; } if (flags == BPF_JIT_FLAG_ALL) break; } return (flags); } /* * Function that does the real stuff. */ bpf_filter_func bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) { bpf_bin_stream stream; struct bpf_insn *ins; int flags, fret, fpkt, fmem, fjmp, flen; u_int i, pass; /* * NOTE: Do not modify the name of this variable, as it's used by * the macros to emit code. */ emit_func emitm; flags = bpf_jit_optimize(prog, nins); fret = (flags & BPF_JIT_FRET) != 0; fpkt = (flags & BPF_JIT_FPKT) != 0; fmem = (flags & BPF_JIT_FMEM) != 0; fjmp = (flags & BPF_JIT_FJMP) != 0; flen = (flags & BPF_JIT_FLEN) != 0; if (fret) nins = 1; memset(&stream, 0, sizeof(stream)); /* Allocate the reference table for the jumps. */ if (fjmp) { #ifdef _KERNEL stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT | M_ZERO); #else stream.refs = calloc(nins + 1, sizeof(u_int)); #endif if (stream.refs == NULL) return (NULL); } /* * The first pass will emit the lengths of the instructions * to create the reference table. */ emitm = emit_length; for (pass = 0; pass < 2; pass++) { ins = prog; /* Create the procedure header. */ if (fmem) { PUSH(RBP); MOVrq(RSP, RBP); SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP); } if (flen) MOVrd2(ESI, R9D); if (fpkt) { MOVrq2(RDI, R8); MOVrd(EDX, EDI); } for (i = 0; i < nins; i++) { stream.bpf_pc++; switch (ins->code) { default: #ifdef _KERNEL return (NULL); #else abort(); #endif case BPF_RET|BPF_K: MOVid(ins->k, EAX); if (fmem) LEAVE(); RET(); break; case BPF_RET|BPF_A: if (fmem) LEAVE(); RET(); break; case BPF_LD|BPF_W|BPF_ABS: MOVid(ins->k, ESI); CMPrd(EDI, ESI); JAb(12); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); if (fmem) { JAEb(4); ZEROrd(EAX); LEAVE(); } else { JAEb(3); ZEROrd(EAX); } RET(); MOVrq3(R8, RCX); MOVobd(RCX, RSI, EAX); BSWAP(EAX); break; case BPF_LD|BPF_H|BPF_ABS: ZEROrd(EAX); MOVid(ins->k, ESI); CMPrd(EDI, ESI); JAb(12); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); if (fmem) { JAEb(2); LEAVE(); } else JAEb(1); RET(); MOVrq3(R8, RCX); MOVobw(RCX, RSI, AX); SWAP_AX(); break; case BPF_LD|BPF_B|BPF_ABS: ZEROrd(EAX); MOVid(ins->k, ESI); CMPrd(EDI, ESI); if (fmem) { JBb(2); LEAVE(); } else JBb(1); RET(); MOVrq3(R8, RCX); MOVobb(RCX, RSI, AL); break; case BPF_LD|BPF_W|BPF_LEN: MOVrd3(R9D, EAX); break; case BPF_LDX|BPF_W|BPF_LEN: MOVrd3(R9D, EDX); break; case BPF_LD|BPF_W|BPF_IND: CMPrd(EDI, EDX); JAb(27); MOVid(ins->k, ESI); MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); JBb(14); ADDrd(EDX, ESI); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); if (fmem) { JAEb(4); ZEROrd(EAX); LEAVE(); } else { JAEb(3); ZEROrd(EAX); } RET(); MOVrq3(R8, RCX); MOVobd(RCX, RSI, EAX); BSWAP(EAX); break; case BPF_LD|BPF_H|BPF_IND: ZEROrd(EAX); CMPrd(EDI, EDX); JAb(27); MOVid(ins->k, ESI); MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); JBb(14); ADDrd(EDX, ESI); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); if (fmem) { JAEb(2); LEAVE(); } else JAEb(1); RET(); MOVrq3(R8, RCX); MOVobw(RCX, RSI, AX); SWAP_AX(); break; case BPF_LD|BPF_B|BPF_IND: ZEROrd(EAX); CMPrd(EDI, EDX); JAEb(13); MOVid(ins->k, ESI); MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); if (fmem) { JAb(2); LEAVE(); } else JAb(1); RET(); MOVrq3(R8, RCX); ADDrd(EDX, ESI); MOVobb(RCX, RSI, AL); break; case BPF_LDX|BPF_MSH|BPF_B: MOVid(ins->k, ESI); CMPrd(EDI, ESI); if (fmem) { JBb(4); ZEROrd(EAX); LEAVE(); } else { JBb(3); ZEROrd(EAX); } RET(); ZEROrd(EDX); MOVrq3(R8, RCX); MOVobb(RCX, RSI, DL); ANDib(0x0f, DL); SHLib(2, EDX); break; case BPF_LD|BPF_IMM: MOVid(ins->k, EAX); break; case BPF_LDX|BPF_IMM: MOVid(ins->k, EDX); break; case BPF_LD|BPF_MEM: MOVid(ins->k * sizeof(uint32_t), ESI); MOVobd(RSP, RSI, EAX); break; case BPF_LDX|BPF_MEM: MOVid(ins->k * sizeof(uint32_t), ESI); MOVobd(RSP, RSI, EDX); break; case BPF_ST: /* * XXX this command and the following could * be optimized if the previous instruction * was already of this type */ MOVid(ins->k * sizeof(uint32_t), ESI); MOVomd(EAX, RSP, RSI); break; case BPF_STX: MOVid(ins->k * sizeof(uint32_t), ESI); MOVomd(EDX, RSP, RSI); break; case BPF_JMP|BPF_JA: JUMP(ins->k); break; case BPF_JMP|BPF_JGT|BPF_K: case BPF_JMP|BPF_JGE|BPF_K: case BPF_JMP|BPF_JEQ|BPF_K: case BPF_JMP|BPF_JSET|BPF_K: case BPF_JMP|BPF_JGT|BPF_X: case BPF_JMP|BPF_JGE|BPF_X: case BPF_JMP|BPF_JEQ|BPF_X: case BPF_JMP|BPF_JSET|BPF_X: if (ins->jt == ins->jf) { JUMP(ins->jt); break; } switch (ins->code) { case BPF_JMP|BPF_JGT|BPF_K: CMPid(ins->k, EAX); JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_K: CMPid(ins->k, EAX); JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_K: CMPid(ins->k, EAX); JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_K: TESTid(ins->k, EAX); JCC(JNE, JE); break; case BPF_JMP|BPF_JGT|BPF_X: CMPrd(EDX, EAX); JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_X: CMPrd(EDX, EAX); JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_X: CMPrd(EDX, EAX); JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_X: TESTrd(EDX, EAX); JCC(JNE, JE); break; } break; case BPF_ALU|BPF_ADD|BPF_X: ADDrd(EDX, EAX); break; case BPF_ALU|BPF_SUB|BPF_X: SUBrd(EDX, EAX); break; case BPF_ALU|BPF_MUL|BPF_X: MOVrd(EDX, ECX); MULrd(EDX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_DIV|BPF_X: case BPF_ALU|BPF_MOD|BPF_X: TESTrd(EDX, EDX); if (fmem) { JNEb(4); ZEROrd(EAX); LEAVE(); } else { JNEb(3); ZEROrd(EAX); } RET(); MOVrd(EDX, ECX); ZEROrd(EDX); DIVrd(ECX); if (BPF_OP(ins->code) == BPF_MOD) MOVrd(EDX, EAX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_AND|BPF_X: ANDrd(EDX, EAX); break; case BPF_ALU|BPF_OR|BPF_X: ORrd(EDX, EAX); break; case BPF_ALU|BPF_XOR|BPF_X: XORrd(EDX, EAX); break; case BPF_ALU|BPF_LSH|BPF_X: MOVrd(EDX, ECX); SHL_CLrb(EAX); break; case BPF_ALU|BPF_RSH|BPF_X: MOVrd(EDX, ECX); SHR_CLrb(EAX); break; case BPF_ALU|BPF_ADD|BPF_K: ADD_EAXi(ins->k); break; case BPF_ALU|BPF_SUB|BPF_K: SUB_EAXi(ins->k); break; case BPF_ALU|BPF_MUL|BPF_K: MOVrd(EDX, ECX); MOVid(ins->k, EDX); MULrd(EDX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_DIV|BPF_K: case BPF_ALU|BPF_MOD|BPF_K: MOVrd(EDX, ECX); ZEROrd(EDX); MOVid(ins->k, ESI); DIVrd(ESI); if (BPF_OP(ins->code) == BPF_MOD) MOVrd(EDX, EAX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_AND|BPF_K: ANDid(ins->k, EAX); break; case BPF_ALU|BPF_OR|BPF_K: ORid(ins->k, EAX); break; case BPF_ALU|BPF_XOR|BPF_K: XORid(ins->k, EAX); break; case BPF_ALU|BPF_LSH|BPF_K: SHLib((ins->k) & 0xff, EAX); break; case BPF_ALU|BPF_RSH|BPF_K: SHRib((ins->k) & 0xff, EAX); break; case BPF_ALU|BPF_NEG: NEGd(EAX); break; case BPF_MISC|BPF_TAX: MOVrd(EAX, EDX); break; case BPF_MISC|BPF_TXA: MOVrd(EDX, EAX); break; } ins++; } if (pass > 0) continue; *size = stream.cur_ip; #ifdef _KERNEL - stream.ibuf = malloc(*size, M_BPFJIT, M_EXEC | M_NOWAIT); + stream.ibuf = malloc_exec(*size, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); if (stream.ibuf == MAP_FAILED) { stream.ibuf = NULL; break; } #endif /* * Modify the reference table to contain the offsets and * not the lengths of the instructions. */ if (fjmp) for (i = 1; i < nins + 1; i++) stream.refs[i] += stream.refs[i - 1]; /* Reset the counters. */ stream.cur_ip = 0; stream.bpf_pc = 0; /* The second pass creates the actual code. */ emitm = emit_code; } /* * The reference table is needed only during compilation, * now we can free it. */ if (fjmp) #ifdef _KERNEL free(stream.refs, M_BPFJIT); #else free(stream.refs); #endif #ifndef _KERNEL if (stream.ibuf != NULL && mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) { munmap(stream.ibuf, *size); stream.ibuf = NULL; } #endif return ((bpf_filter_func)(void *)stream.ibuf); } Index: head/sys/i386/i386/bpf_jit_machdep.c =================================================================== --- head/sys/i386/i386/bpf_jit_machdep.c (revision 367185) +++ head/sys/i386/i386/bpf_jit_machdep.c (revision 367186) @@ -1,683 +1,683 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) * Copyright (C) 2005-2017 Jung-uk Kim * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Politecnico di Torino nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef _KERNEL #include "opt_bpf.h" #include #include #include #include #include #include #include #else #include #include #include #include #endif #include #include #include #include /* * Emit routine to update the jump table. */ static void emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) { if (stream->refs != NULL) (stream->refs)[stream->bpf_pc] += len; stream->cur_ip += len; } /* * Emit routine to output the actual binary code. */ static void emit_code(bpf_bin_stream *stream, u_int value, u_int len) { switch (len) { case 1: stream->ibuf[stream->cur_ip] = (u_char)value; stream->cur_ip++; break; case 2: *((u_short *)(void *)(stream->ibuf + stream->cur_ip)) = (u_short)value; stream->cur_ip += 2; break; case 4: *((u_int *)(void *)(stream->ibuf + stream->cur_ip)) = value; stream->cur_ip += 4; break; } return; } /* * Scan the filter program and find possible optimization. */ static int bpf_jit_optimize(struct bpf_insn *prog, u_int nins) { int flags; u_int i; /* Do we return immediately? */ if (BPF_CLASS(prog[0].code) == BPF_RET) return (BPF_JIT_FRET); for (flags = 0, i = 0; i < nins; i++) { switch (prog[i].code) { case BPF_LD|BPF_W|BPF_ABS: case BPF_LD|BPF_H|BPF_ABS: case BPF_LD|BPF_B|BPF_ABS: case BPF_LD|BPF_W|BPF_IND: case BPF_LD|BPF_H|BPF_IND: case BPF_LD|BPF_B|BPF_IND: case BPF_LDX|BPF_MSH|BPF_B: flags |= BPF_JIT_FPKT; break; case BPF_LD|BPF_MEM: case BPF_LDX|BPF_MEM: case BPF_ST: case BPF_STX: flags |= BPF_JIT_FMEM; break; case BPF_JMP|BPF_JA: case BPF_JMP|BPF_JGT|BPF_K: case BPF_JMP|BPF_JGE|BPF_K: case BPF_JMP|BPF_JEQ|BPF_K: case BPF_JMP|BPF_JSET|BPF_K: case BPF_JMP|BPF_JGT|BPF_X: case BPF_JMP|BPF_JGE|BPF_X: case BPF_JMP|BPF_JEQ|BPF_X: case BPF_JMP|BPF_JSET|BPF_X: flags |= BPF_JIT_FJMP; break; case BPF_ALU|BPF_DIV|BPF_K: case BPF_ALU|BPF_MOD|BPF_K: flags |= BPF_JIT_FADK; break; } if (flags == BPF_JIT_FLAG_ALL) break; } return (flags); } /* * Function that does the real stuff. */ bpf_filter_func bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) { bpf_bin_stream stream; struct bpf_insn *ins; int flags, fret, fpkt, fmem, fjmp, fadk; int save_esp; u_int i, pass; /* * NOTE: Do not modify the name of this variable, as it's used by * the macros to emit code. */ emit_func emitm; flags = bpf_jit_optimize(prog, nins); fret = (flags & BPF_JIT_FRET) != 0; fpkt = (flags & BPF_JIT_FPKT) != 0; fmem = (flags & BPF_JIT_FMEM) != 0; fjmp = (flags & BPF_JIT_FJMP) != 0; fadk = (flags & BPF_JIT_FADK) != 0; save_esp = (fpkt || fmem || fadk); /* Stack is used. */ if (fret) nins = 1; memset(&stream, 0, sizeof(stream)); /* Allocate the reference table for the jumps. */ if (fjmp) { #ifdef _KERNEL stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT | M_ZERO); #else stream.refs = calloc(nins + 1, sizeof(u_int)); #endif if (stream.refs == NULL) return (NULL); } /* * The first pass will emit the lengths of the instructions * to create the reference table. */ emitm = emit_length; for (pass = 0; pass < 2; pass++) { ins = prog; /* Create the procedure header. */ if (save_esp) { PUSH(EBP); MOVrd(ESP, EBP); } if (fmem) SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP); if (save_esp) PUSH(ESI); if (fpkt) { PUSH(EDI); PUSH(EBX); MOVodd(8, EBP, EBX); MOVodd(16, EBP, EDI); } for (i = 0; i < nins; i++) { stream.bpf_pc++; switch (ins->code) { default: #ifdef _KERNEL return (NULL); #else abort(); #endif case BPF_RET|BPF_K: MOVid(ins->k, EAX); if (save_esp) { if (fpkt) { POP(EBX); POP(EDI); } POP(ESI); LEAVE(); } RET(); break; case BPF_RET|BPF_A: if (save_esp) { if (fpkt) { POP(EBX); POP(EDI); } POP(ESI); LEAVE(); } RET(); break; case BPF_LD|BPF_W|BPF_ABS: MOVid(ins->k, ESI); CMPrd(EDI, ESI); JAb(12); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); JAEb(7); ZEROrd(EAX); POP(EBX); POP(EDI); POP(ESI); LEAVE(); RET(); MOVobd(EBX, ESI, EAX); BSWAP(EAX); break; case BPF_LD|BPF_H|BPF_ABS: ZEROrd(EAX); MOVid(ins->k, ESI); CMPrd(EDI, ESI); JAb(12); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); JAEb(5); POP(EBX); POP(EDI); POP(ESI); LEAVE(); RET(); MOVobw(EBX, ESI, AX); SWAP_AX(); break; case BPF_LD|BPF_B|BPF_ABS: ZEROrd(EAX); MOVid(ins->k, ESI); CMPrd(EDI, ESI); JBb(5); POP(EBX); POP(EDI); POP(ESI); LEAVE(); RET(); MOVobb(EBX, ESI, AL); break; case BPF_LD|BPF_W|BPF_LEN: if (save_esp) MOVodd(12, EBP, EAX); else { MOVrd(ESP, ECX); MOVodd(12, ECX, EAX); } break; case BPF_LDX|BPF_W|BPF_LEN: if (save_esp) MOVodd(12, EBP, EDX); else { MOVrd(ESP, ECX); MOVodd(12, ECX, EDX); } break; case BPF_LD|BPF_W|BPF_IND: CMPrd(EDI, EDX); JAb(27); MOVid(ins->k, ESI); MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); JBb(14); ADDrd(EDX, ESI); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); JAEb(7); ZEROrd(EAX); POP(EBX); POP(EDI); POP(ESI); LEAVE(); RET(); MOVobd(EBX, ESI, EAX); BSWAP(EAX); break; case BPF_LD|BPF_H|BPF_IND: ZEROrd(EAX); CMPrd(EDI, EDX); JAb(27); MOVid(ins->k, ESI); MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); JBb(14); ADDrd(EDX, ESI); MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); JAEb(5); POP(EBX); POP(EDI); POP(ESI); LEAVE(); RET(); MOVobw(EBX, ESI, AX); SWAP_AX(); break; case BPF_LD|BPF_B|BPF_IND: ZEROrd(EAX); CMPrd(EDI, EDX); JAEb(13); MOVid(ins->k, ESI); MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); JAb(5); POP(EBX); POP(EDI); POP(ESI); LEAVE(); RET(); ADDrd(EDX, ESI); MOVobb(EBX, ESI, AL); break; case BPF_LDX|BPF_MSH|BPF_B: MOVid(ins->k, ESI); CMPrd(EDI, ESI); JBb(7); ZEROrd(EAX); POP(EBX); POP(EDI); POP(ESI); LEAVE(); RET(); ZEROrd(EDX); MOVobb(EBX, ESI, DL); ANDib(0x0f, DL); SHLib(2, EDX); break; case BPF_LD|BPF_IMM: MOVid(ins->k, EAX); break; case BPF_LDX|BPF_IMM: MOVid(ins->k, EDX); break; case BPF_LD|BPF_MEM: MOVrd(EBP, ECX); MOVid(((int)ins->k - BPF_MEMWORDS) * sizeof(uint32_t), ESI); MOVobd(ECX, ESI, EAX); break; case BPF_LDX|BPF_MEM: MOVrd(EBP, ECX); MOVid(((int)ins->k - BPF_MEMWORDS) * sizeof(uint32_t), ESI); MOVobd(ECX, ESI, EDX); break; case BPF_ST: /* * XXX this command and the following could * be optimized if the previous instruction * was already of this type */ MOVrd(EBP, ECX); MOVid(((int)ins->k - BPF_MEMWORDS) * sizeof(uint32_t), ESI); MOVomd(EAX, ECX, ESI); break; case BPF_STX: MOVrd(EBP, ECX); MOVid(((int)ins->k - BPF_MEMWORDS) * sizeof(uint32_t), ESI); MOVomd(EDX, ECX, ESI); break; case BPF_JMP|BPF_JA: JUMP(ins->k); break; case BPF_JMP|BPF_JGT|BPF_K: case BPF_JMP|BPF_JGE|BPF_K: case BPF_JMP|BPF_JEQ|BPF_K: case BPF_JMP|BPF_JSET|BPF_K: case BPF_JMP|BPF_JGT|BPF_X: case BPF_JMP|BPF_JGE|BPF_X: case BPF_JMP|BPF_JEQ|BPF_X: case BPF_JMP|BPF_JSET|BPF_X: if (ins->jt == ins->jf) { JUMP(ins->jt); break; } switch (ins->code) { case BPF_JMP|BPF_JGT|BPF_K: CMPid(ins->k, EAX); JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_K: CMPid(ins->k, EAX); JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_K: CMPid(ins->k, EAX); JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_K: TESTid(ins->k, EAX); JCC(JNE, JE); break; case BPF_JMP|BPF_JGT|BPF_X: CMPrd(EDX, EAX); JCC(JA, JBE); break; case BPF_JMP|BPF_JGE|BPF_X: CMPrd(EDX, EAX); JCC(JAE, JB); break; case BPF_JMP|BPF_JEQ|BPF_X: CMPrd(EDX, EAX); JCC(JE, JNE); break; case BPF_JMP|BPF_JSET|BPF_X: TESTrd(EDX, EAX); JCC(JNE, JE); break; } break; case BPF_ALU|BPF_ADD|BPF_X: ADDrd(EDX, EAX); break; case BPF_ALU|BPF_SUB|BPF_X: SUBrd(EDX, EAX); break; case BPF_ALU|BPF_MUL|BPF_X: MOVrd(EDX, ECX); MULrd(EDX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_DIV|BPF_X: case BPF_ALU|BPF_MOD|BPF_X: TESTrd(EDX, EDX); if (save_esp) { if (fpkt) { JNEb(7); ZEROrd(EAX); POP(EBX); POP(EDI); } else { JNEb(5); ZEROrd(EAX); } POP(ESI); LEAVE(); } else { JNEb(3); ZEROrd(EAX); } RET(); MOVrd(EDX, ECX); ZEROrd(EDX); DIVrd(ECX); if (BPF_OP(ins->code) == BPF_MOD) MOVrd(EDX, EAX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_AND|BPF_X: ANDrd(EDX, EAX); break; case BPF_ALU|BPF_OR|BPF_X: ORrd(EDX, EAX); break; case BPF_ALU|BPF_XOR|BPF_X: XORrd(EDX, EAX); break; case BPF_ALU|BPF_LSH|BPF_X: MOVrd(EDX, ECX); SHL_CLrb(EAX); break; case BPF_ALU|BPF_RSH|BPF_X: MOVrd(EDX, ECX); SHR_CLrb(EAX); break; case BPF_ALU|BPF_ADD|BPF_K: ADD_EAXi(ins->k); break; case BPF_ALU|BPF_SUB|BPF_K: SUB_EAXi(ins->k); break; case BPF_ALU|BPF_MUL|BPF_K: MOVrd(EDX, ECX); MOVid(ins->k, EDX); MULrd(EDX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_DIV|BPF_K: case BPF_ALU|BPF_MOD|BPF_K: MOVrd(EDX, ECX); ZEROrd(EDX); MOVid(ins->k, ESI); DIVrd(ESI); if (BPF_OP(ins->code) == BPF_MOD) MOVrd(EDX, EAX); MOVrd(ECX, EDX); break; case BPF_ALU|BPF_AND|BPF_K: ANDid(ins->k, EAX); break; case BPF_ALU|BPF_OR|BPF_K: ORid(ins->k, EAX); break; case BPF_ALU|BPF_XOR|BPF_K: XORid(ins->k, EAX); break; case BPF_ALU|BPF_LSH|BPF_K: SHLib((ins->k) & 0xff, EAX); break; case BPF_ALU|BPF_RSH|BPF_K: SHRib((ins->k) & 0xff, EAX); break; case BPF_ALU|BPF_NEG: NEGd(EAX); break; case BPF_MISC|BPF_TAX: MOVrd(EAX, EDX); break; case BPF_MISC|BPF_TXA: MOVrd(EDX, EAX); break; } ins++; } if (pass > 0) continue; *size = stream.cur_ip; #ifdef _KERNEL - stream.ibuf = malloc(*size, M_BPFJIT, M_EXEC | M_NOWAIT); + stream.ibuf = malloc_exec(*size, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); if (stream.ibuf == MAP_FAILED) { stream.ibuf = NULL; break; } #endif /* * Modify the reference table to contain the offsets and * not the lengths of the instructions. */ if (fjmp) for (i = 1; i < nins + 1; i++) stream.refs[i] += stream.refs[i - 1]; /* Reset the counters. */ stream.cur_ip = 0; stream.bpf_pc = 0; /* The second pass creates the actual code. */ emitm = emit_code; } /* * The reference table is needed only during compilation, * now we can free it. */ if (fjmp) #ifdef _KERNEL free(stream.refs, M_BPFJIT); #else free(stream.refs); #endif #ifndef _KERNEL if (stream.ibuf != NULL && mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) { munmap(stream.ibuf, *size); stream.ibuf = NULL; } #endif return ((bpf_filter_func)(void *)stream.ibuf); } Index: head/sys/kern/kern_malloc.c =================================================================== --- head/sys/kern/kern_malloc.c (revision 367185) +++ head/sys/kern/kern_malloc.c (revision 367186) @@ -1,1479 +1,1543 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1987, 1991, 1993 * The Regents of the University of California. * Copyright (c) 2005-2009 Robert N. M. Watson * Copyright (c) 2008 Otto Moerbeek (mallocarray) * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)kern_malloc.c 8.3 (Berkeley) 1/4/94 */ /* * Kernel malloc(9) implementation -- general purpose kernel memory allocator * based on memory types. Back end is implemented using the UMA(9) zone * allocator. A set of fixed-size buckets are used for smaller allocations, * and a special UMA allocation interface is used for larger allocations. * Callers declare memory types, and statistics are maintained independently * for each memory type. Statistics are maintained per-CPU for performance * reasons. See malloc(9) and comments in malloc.h for a detailed * description. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef EPOCH_TRACE #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEBUG_MEMGUARD #include #endif #ifdef DEBUG_REDZONE #include #endif #if defined(INVARIANTS) && defined(__i386__) #include #endif #include #ifdef KDTRACE_HOOKS #include bool __read_frequently dtrace_malloc_enabled; dtrace_malloc_probe_func_t __read_mostly dtrace_malloc_probe; #endif #if defined(INVARIANTS) || defined(MALLOC_MAKE_FAILURES) || \ defined(DEBUG_MEMGUARD) || defined(DEBUG_REDZONE) #define MALLOC_DEBUG 1 #endif /* * When realloc() is called, if the new size is sufficiently smaller than * the old size, realloc() will allocate a new, smaller block to avoid * wasting memory. 'Sufficiently smaller' is defined as: newsize <= * oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'. */ #ifndef REALLOC_FRACTION #define REALLOC_FRACTION 1 /* new block if <= half the size */ #endif /* * Centrally define some common malloc types. */ MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches"); MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory"); MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers"); static struct malloc_type *kmemstatistics; static int kmemcount; #define KMEM_ZSHIFT 4 #define KMEM_ZBASE 16 #define KMEM_ZMASK (KMEM_ZBASE - 1) #define KMEM_ZMAX 65536 #define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT) static uint8_t kmemsize[KMEM_ZSIZE + 1]; #ifndef MALLOC_DEBUG_MAXZONES #define MALLOC_DEBUG_MAXZONES 1 #endif static int numzones = MALLOC_DEBUG_MAXZONES; /* * Small malloc(9) memory allocations are allocated from a set of UMA buckets * of various sizes. * * XXX: The comment here used to read "These won't be powers of two for * long." It's possible that a significant amount of wasted memory could be * recovered by tuning the sizes of these buckets. */ struct { int kz_size; const char *kz_name; uma_zone_t kz_zone[MALLOC_DEBUG_MAXZONES]; } kmemzones[] = { {16, "16", }, {32, "32", }, {64, "64", }, {128, "128", }, {256, "256", }, {512, "512", }, {1024, "1024", }, {2048, "2048", }, {4096, "4096", }, {8192, "8192", }, {16384, "16384", }, {32768, "32768", }, {65536, "65536", }, {0, NULL}, }; /* * Zone to allocate malloc type descriptions from. For ABI reasons, memory * types are described by a data structure passed by the declaring code, but * the malloc(9) implementation has its own data structure describing the * type and statistics. This permits the malloc(9)-internal data structures * to be modified without breaking binary-compiled kernel modules that * declare malloc types. */ static uma_zone_t mt_zone; static uma_zone_t mt_stats_zone; u_long vm_kmem_size; SYSCTL_ULONG(_vm, OID_AUTO, kmem_size, CTLFLAG_RDTUN, &vm_kmem_size, 0, "Size of kernel memory"); static u_long kmem_zmax = KMEM_ZMAX; SYSCTL_ULONG(_vm, OID_AUTO, kmem_zmax, CTLFLAG_RDTUN, &kmem_zmax, 0, "Maximum allocation size that malloc(9) would use UMA as backend"); static u_long vm_kmem_size_min; SYSCTL_ULONG(_vm, OID_AUTO, kmem_size_min, CTLFLAG_RDTUN, &vm_kmem_size_min, 0, "Minimum size of kernel memory"); static u_long vm_kmem_size_max; SYSCTL_ULONG(_vm, OID_AUTO, kmem_size_max, CTLFLAG_RDTUN, &vm_kmem_size_max, 0, "Maximum size of kernel memory"); static u_int vm_kmem_size_scale; SYSCTL_UINT(_vm, OID_AUTO, kmem_size_scale, CTLFLAG_RDTUN, &vm_kmem_size_scale, 0, "Scale factor for kernel memory size"); static int sysctl_kmem_map_size(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, kmem_map_size, CTLFLAG_RD | CTLTYPE_ULONG | CTLFLAG_MPSAFE, NULL, 0, sysctl_kmem_map_size, "LU", "Current kmem allocation size"); static int sysctl_kmem_map_free(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, kmem_map_free, CTLFLAG_RD | CTLTYPE_ULONG | CTLFLAG_MPSAFE, NULL, 0, sysctl_kmem_map_free, "LU", "Free space in kmem"); /* * The malloc_mtx protects the kmemstatistics linked list. */ struct mtx malloc_mtx; #ifdef MALLOC_PROFILE uint64_t krequests[KMEM_ZSIZE + 1]; static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS); #endif static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS); /* * time_uptime of the last malloc(9) failure (induced or real). */ static time_t t_malloc_fail; #if defined(MALLOC_MAKE_FAILURES) || (MALLOC_DEBUG_MAXZONES > 1) static SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Kernel malloc debugging options"); #endif /* * malloc(9) fault injection -- cause malloc failures every (n) mallocs when * the caller specifies M_NOWAIT. If set to 0, no failures are caused. */ #ifdef MALLOC_MAKE_FAILURES static int malloc_failure_rate; static int malloc_nowait_count; static int malloc_failure_count; SYSCTL_INT(_debug_malloc, OID_AUTO, failure_rate, CTLFLAG_RWTUN, &malloc_failure_rate, 0, "Every (n) mallocs with M_NOWAIT will fail"); SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD, &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures"); #endif static int sysctl_kmem_map_size(SYSCTL_HANDLER_ARGS) { u_long size; size = uma_size(); return (sysctl_handle_long(oidp, &size, 0, req)); } static int sysctl_kmem_map_free(SYSCTL_HANDLER_ARGS) { u_long size, limit; /* The sysctl is unsigned, implement as a saturation value. */ size = uma_size(); limit = uma_limit(); if (size > limit) size = 0; else size = limit - size; return (sysctl_handle_long(oidp, &size, 0, req)); } /* * malloc(9) uma zone separation -- sub-page buffer overruns in one * malloc type will affect only a subset of other malloc types. */ #if MALLOC_DEBUG_MAXZONES > 1 static void tunable_set_numzones(void) { TUNABLE_INT_FETCH("debug.malloc.numzones", &numzones); /* Sanity check the number of malloc uma zones. */ if (numzones <= 0) numzones = 1; if (numzones > MALLOC_DEBUG_MAXZONES) numzones = MALLOC_DEBUG_MAXZONES; } SYSINIT(numzones, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_set_numzones, NULL); SYSCTL_INT(_debug_malloc, OID_AUTO, numzones, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &numzones, 0, "Number of malloc uma subzones"); /* * Any number that changes regularly is an okay choice for the * offset. Build numbers are pretty good of you have them. */ static u_int zone_offset = __FreeBSD_version; TUNABLE_INT("debug.malloc.zone_offset", &zone_offset); SYSCTL_UINT(_debug_malloc, OID_AUTO, zone_offset, CTLFLAG_RDTUN, &zone_offset, 0, "Separate malloc types by examining the " "Nth character in the malloc type short description."); static void mtp_set_subzone(struct malloc_type *mtp) { struct malloc_type_internal *mtip; const char *desc; size_t len; u_int val; mtip = mtp->ks_handle; desc = mtp->ks_shortdesc; if (desc == NULL || (len = strlen(desc)) == 0) val = 0; else val = desc[zone_offset % len]; mtip->mti_zone = (val % numzones); } static inline u_int mtp_get_subzone(struct malloc_type *mtp) { struct malloc_type_internal *mtip; mtip = mtp->ks_handle; KASSERT(mtip->mti_zone < numzones, ("mti_zone %u out of range %d", mtip->mti_zone, numzones)); return (mtip->mti_zone); } #elif MALLOC_DEBUG_MAXZONES == 0 #error "MALLOC_DEBUG_MAXZONES must be positive." #else static void mtp_set_subzone(struct malloc_type *mtp) { struct malloc_type_internal *mtip; mtip = mtp->ks_handle; mtip->mti_zone = 0; } static inline u_int mtp_get_subzone(struct malloc_type *mtp) { return (0); } #endif /* MALLOC_DEBUG_MAXZONES > 1 */ int malloc_last_fail(void) { return (time_uptime - t_malloc_fail); } /* * An allocation has succeeded -- update malloc type statistics for the * amount of bucket size. Occurs within a critical section so that the * thread isn't preempted and doesn't migrate while updating per-PCU * statistics. */ static void malloc_type_zone_allocated(struct malloc_type *mtp, unsigned long size, int zindx) { struct malloc_type_internal *mtip; struct malloc_type_stats *mtsp; critical_enter(); mtip = mtp->ks_handle; mtsp = zpcpu_get(mtip->mti_stats); if (size > 0) { mtsp->mts_memalloced += size; mtsp->mts_numallocs++; } if (zindx != -1) mtsp->mts_size |= 1 << zindx; #ifdef KDTRACE_HOOKS if (__predict_false(dtrace_malloc_enabled)) { uint32_t probe_id = mtip->mti_probes[DTMALLOC_PROBE_MALLOC]; if (probe_id != 0) (dtrace_malloc_probe)(probe_id, (uintptr_t) mtp, (uintptr_t) mtip, (uintptr_t) mtsp, size, zindx); } #endif critical_exit(); } void malloc_type_allocated(struct malloc_type *mtp, unsigned long size) { if (size > 0) malloc_type_zone_allocated(mtp, size, -1); } /* * A free operation has occurred -- update malloc type statistics for the * amount of the bucket size. Occurs within a critical section so that the * thread isn't preempted and doesn't migrate while updating per-CPU * statistics. */ void malloc_type_freed(struct malloc_type *mtp, unsigned long size) { struct malloc_type_internal *mtip; struct malloc_type_stats *mtsp; critical_enter(); mtip = mtp->ks_handle; mtsp = zpcpu_get(mtip->mti_stats); mtsp->mts_memfreed += size; mtsp->mts_numfrees++; #ifdef KDTRACE_HOOKS if (__predict_false(dtrace_malloc_enabled)) { uint32_t probe_id = mtip->mti_probes[DTMALLOC_PROBE_FREE]; if (probe_id != 0) (dtrace_malloc_probe)(probe_id, (uintptr_t) mtp, (uintptr_t) mtip, (uintptr_t) mtsp, size, 0); } #endif critical_exit(); } /* * contigmalloc: * * Allocate a block of physically contiguous memory. * * If M_NOWAIT is set, this routine will not block and return NULL if * the allocation fails. */ void * contigmalloc(unsigned long size, struct malloc_type *type, int flags, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary) { void *ret; ret = (void *)kmem_alloc_contig(size, flags, low, high, alignment, boundary, VM_MEMATTR_DEFAULT); if (ret != NULL) malloc_type_allocated(type, round_page(size)); return (ret); } void * contigmalloc_domainset(unsigned long size, struct malloc_type *type, struct domainset *ds, int flags, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary) { void *ret; ret = (void *)kmem_alloc_contig_domainset(ds, size, flags, low, high, alignment, boundary, VM_MEMATTR_DEFAULT); if (ret != NULL) malloc_type_allocated(type, round_page(size)); return (ret); } /* * contigfree: * * Free a block of memory allocated by contigmalloc. * * This routine may not block. */ void contigfree(void *addr, unsigned long size, struct malloc_type *type) { kmem_free((vm_offset_t)addr, size); malloc_type_freed(type, round_page(size)); } #ifdef MALLOC_DEBUG static int malloc_dbg(caddr_t *vap, size_t *sizep, struct malloc_type *mtp, int flags) { #ifdef INVARIANTS int indx; KASSERT(mtp->ks_magic == M_MAGIC, ("malloc: bad malloc type magic")); /* * Check that exactly one of M_WAITOK or M_NOWAIT is specified. */ indx = flags & (M_WAITOK | M_NOWAIT); if (indx != M_NOWAIT && indx != M_WAITOK) { static struct timeval lasterr; static int curerr, once; if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) { printf("Bad malloc flags: %x\n", indx); kdb_backtrace(); flags |= M_WAITOK; once++; } } #endif #ifdef MALLOC_MAKE_FAILURES if ((flags & M_NOWAIT) && (malloc_failure_rate != 0)) { atomic_add_int(&malloc_nowait_count, 1); if ((malloc_nowait_count % malloc_failure_rate) == 0) { atomic_add_int(&malloc_failure_count, 1); t_malloc_fail = time_uptime; *vap = NULL; return (EJUSTRETURN); } } #endif if (flags & M_WAITOK) { KASSERT(curthread->td_intr_nesting_level == 0, ("malloc(M_WAITOK) in interrupt context")); if (__predict_false(!THREAD_CAN_SLEEP())) { #ifdef EPOCH_TRACE epoch_trace_list(curthread); #endif KASSERT(1, ("malloc(M_WAITOK) with sleeping prohibited")); } } KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), ("malloc: called with spinlock or critical section held")); #ifdef DEBUG_MEMGUARD if (memguard_cmp_mtp(mtp, *sizep)) { *vap = memguard_alloc(*sizep, flags); if (*vap != NULL) return (EJUSTRETURN); /* This is unfortunate but should not be fatal. */ } #endif #ifdef DEBUG_REDZONE *sizep = redzone_size_ntor(*sizep); #endif return (0); } #endif /* * Handle large allocations and frees by using kmem_malloc directly. */ static inline bool malloc_large_slab(uma_slab_t slab) { uintptr_t va; va = (uintptr_t)slab; return ((va & 1) != 0); } static inline size_t malloc_large_size(uma_slab_t slab) { uintptr_t va; va = (uintptr_t)slab; return (va >> 1); } static caddr_t malloc_large(size_t *size, struct domainset *policy, int flags) { vm_offset_t va; size_t sz; sz = roundup(*size, PAGE_SIZE); va = kmem_malloc_domainset(policy, sz, flags); if (va != 0) { /* The low bit is unused for slab pointers. */ vsetzoneslab(va, NULL, (void *)((sz << 1) | 1)); uma_total_inc(sz); *size = sz; } return ((caddr_t)va); } static void free_large(void *addr, size_t size) { kmem_free((vm_offset_t)addr, size); uma_total_dec(size); } /* * malloc: * * Allocate a block of memory. * * If M_NOWAIT is set, this routine will not block and return NULL if * the allocation fails. */ void * (malloc)(size_t size, struct malloc_type *mtp, int flags) { int indx; caddr_t va; uma_zone_t zone; #if defined(DEBUG_REDZONE) unsigned long osize = size; #endif + MPASS((flags & M_EXEC) == 0); #ifdef MALLOC_DEBUG va = NULL; if (malloc_dbg(&va, &size, mtp, flags) != 0) return (va); #endif - if (size <= kmem_zmax && (flags & M_EXEC) == 0) { + if (size <= kmem_zmax) { if (size & KMEM_ZMASK) size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; indx = kmemsize[size >> KMEM_ZSHIFT]; zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)]; #ifdef MALLOC_PROFILE krequests[size >> KMEM_ZSHIFT]++; #endif va = uma_zalloc(zone, flags); if (va != NULL) size = zone->uz_size; malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); } else { va = malloc_large(&size, DOMAINSET_RR(), flags); malloc_type_allocated(mtp, va == NULL ? 0 : size); } - if (flags & M_WAITOK) - KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); - else if (va == NULL) + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); t_malloc_fail = time_uptime; + } #ifdef DEBUG_REDZONE if (va != NULL) va = redzone_setup(va, osize); #endif return ((void *) va); } static void * malloc_domain(size_t *sizep, int *indxp, struct malloc_type *mtp, int domain, int flags) { uma_zone_t zone; caddr_t va; size_t size; int indx; size = *sizep; KASSERT(size <= kmem_zmax && (flags & M_EXEC) == 0, ("malloc_domain: Called with bad flag / size combination.")); if (size & KMEM_ZMASK) size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; indx = kmemsize[size >> KMEM_ZSHIFT]; zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)]; #ifdef MALLOC_PROFILE krequests[size >> KMEM_ZSHIFT]++; #endif va = uma_zalloc_domain(zone, NULL, domain, flags); if (va != NULL) *sizep = zone->uz_size; *indxp = indx; return ((void *)va); } void * malloc_domainset(size_t size, struct malloc_type *mtp, struct domainset *ds, int flags) { struct vm_domainset_iter di; - caddr_t ret; + caddr_t va; int domain; int indx; #if defined(DEBUG_REDZONE) unsigned long osize = size; #endif + MPASS((flags & M_EXEC) == 0); #ifdef MALLOC_DEBUG - ret= NULL; - if (malloc_dbg(&ret, &size, mtp, flags) != 0) - return (ret); + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); #endif - if (size <= kmem_zmax && (flags & M_EXEC) == 0) { + if (size <= kmem_zmax) { vm_domainset_iter_policy_init(&di, ds, &domain, &flags); do { - ret = malloc_domain(&size, &indx, mtp, domain, flags); - } while (ret == NULL && + va = malloc_domain(&size, &indx, mtp, domain, flags); + } while (va == NULL && vm_domainset_iter_policy(&di, &domain) == 0); - malloc_type_zone_allocated(mtp, ret == NULL ? 0 : size, indx); + malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); } else { /* Policy is handled by kmem. */ - ret = malloc_large(&size, ds, flags); - malloc_type_allocated(mtp, ret == NULL ? 0 : size); + va = malloc_large(&size, ds, flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); } + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); + t_malloc_fail = time_uptime; + } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); +#endif + return (va); +} - if (flags & M_WAITOK) - KASSERT(ret != NULL, ("malloc(M_WAITOK) returned NULL")); - else if (ret == NULL) +/* + * Allocate an executable area. + */ +void * +malloc_exec(size_t size, struct malloc_type *mtp, int flags) +{ + caddr_t va; +#if defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + + flags |= M_EXEC; +#ifdef MALLOC_DEBUG + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + va = malloc_large(&size, DOMAINSET_RR(), flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); t_malloc_fail = time_uptime; + } #ifdef DEBUG_REDZONE - if (ret != NULL) - ret = redzone_setup(ret, osize); + if (va != NULL) + va = redzone_setup(va, osize); #endif - return (ret); + return ((void *) va); +} + +void * +malloc_domainset_exec(size_t size, struct malloc_type *mtp, struct domainset *ds, + int flags) +{ + caddr_t va; +#if defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + + flags |= M_EXEC; +#ifdef MALLOC_DEBUG + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + /* Policy is handled by kmem. */ + va = malloc_large(&size, ds, flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); + t_malloc_fail = time_uptime; + } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); +#endif + return (va); } void * mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags) { if (WOULD_OVERFLOW(nmemb, size)) panic("mallocarray: %zu * %zu overflowed", nmemb, size); return (malloc(size * nmemb, type, flags)); } #ifdef INVARIANTS static void free_save_type(void *addr, struct malloc_type *mtp, u_long size) { struct malloc_type **mtpp = addr; /* * Cache a pointer to the malloc_type that most recently freed * this memory here. This way we know who is most likely to * have stepped on it later. * * This code assumes that size is a multiple of 8 bytes for * 64 bit machines */ mtpp = (struct malloc_type **) ((unsigned long)mtpp & ~UMA_ALIGN_PTR); mtpp += (size - sizeof(struct malloc_type *)) / sizeof(struct malloc_type *); *mtpp = mtp; } #endif #ifdef MALLOC_DEBUG static int free_dbg(void **addrp, struct malloc_type *mtp) { void *addr; addr = *addrp; KASSERT(mtp->ks_magic == M_MAGIC, ("free: bad malloc type magic")); KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), ("free: called with spinlock or critical section held")); /* free(NULL, ...) does nothing */ if (addr == NULL) return (EJUSTRETURN); #ifdef DEBUG_MEMGUARD if (is_memguard_addr(addr)) { memguard_free(addr); return (EJUSTRETURN); } #endif #ifdef DEBUG_REDZONE redzone_check(addr); *addrp = redzone_addr_ntor(addr); #endif return (0); } #endif /* * free: * * Free a block of memory allocated by malloc. * * This routine may not block. */ void free(void *addr, struct malloc_type *mtp) { uma_zone_t zone; uma_slab_t slab; u_long size; #ifdef MALLOC_DEBUG if (free_dbg(&addr, mtp) != 0) return; #endif /* free(NULL, ...) does nothing */ if (addr == NULL) return; vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab); if (slab == NULL) panic("free: address %p(%p) has not been allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); if (__predict_true(!malloc_large_slab(slab))) { size = zone->uz_size; #ifdef INVARIANTS free_save_type(addr, mtp, size); #endif uma_zfree_arg(zone, addr, slab); } else { size = malloc_large_size(slab); free_large(addr, size); } malloc_type_freed(mtp, size); } /* * zfree: * * Zero then free a block of memory allocated by malloc. * * This routine may not block. */ void zfree(void *addr, struct malloc_type *mtp) { uma_zone_t zone; uma_slab_t slab; u_long size; #ifdef MALLOC_DEBUG if (free_dbg(&addr, mtp) != 0) return; #endif /* free(NULL, ...) does nothing */ if (addr == NULL) return; vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab); if (slab == NULL) panic("free: address %p(%p) has not been allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); if (__predict_true(!malloc_large_slab(slab))) { size = zone->uz_size; #ifdef INVARIANTS free_save_type(addr, mtp, size); #endif explicit_bzero(addr, size); uma_zfree_arg(zone, addr, slab); } else { size = malloc_large_size(slab); explicit_bzero(addr, size); free_large(addr, size); } malloc_type_freed(mtp, size); } /* * realloc: change the size of a memory block */ void * realloc(void *addr, size_t size, struct malloc_type *mtp, int flags) { uma_zone_t zone; uma_slab_t slab; unsigned long alloc; void *newaddr; KASSERT(mtp->ks_magic == M_MAGIC, ("realloc: bad malloc type magic")); KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), ("realloc: called with spinlock or critical section held")); /* realloc(NULL, ...) is equivalent to malloc(...) */ if (addr == NULL) return (malloc(size, mtp, flags)); /* * XXX: Should report free of old memory and alloc of new memory to * per-CPU stats. */ #ifdef DEBUG_MEMGUARD if (is_memguard_addr(addr)) return (memguard_realloc(addr, size, mtp, flags)); #endif #ifdef DEBUG_REDZONE slab = NULL; zone = NULL; alloc = redzone_get_size(addr); #else vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab); /* Sanity check */ KASSERT(slab != NULL, ("realloc: address %p out of range", (void *)addr)); /* Get the size of the original block */ if (!malloc_large_slab(slab)) alloc = zone->uz_size; else alloc = malloc_large_size(slab); /* Reuse the original block if appropriate */ if (size <= alloc && (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE)) return (addr); #endif /* !DEBUG_REDZONE */ /* Allocate a new, bigger (or smaller) block */ if ((newaddr = malloc(size, mtp, flags)) == NULL) return (NULL); /* Copy over original contents */ bcopy(addr, newaddr, min(size, alloc)); free(addr, mtp); return (newaddr); } /* * reallocf: same as realloc() but free memory on failure. */ void * reallocf(void *addr, size_t size, struct malloc_type *mtp, int flags) { void *mem; if ((mem = realloc(addr, size, mtp, flags)) == NULL) free(addr, mtp); return (mem); } /* * malloc_usable_size: returns the usable size of the allocation. */ size_t malloc_usable_size(const void *addr) { #ifndef DEBUG_REDZONE uma_zone_t zone; uma_slab_t slab; #endif u_long size; if (addr == NULL) return (0); #ifdef DEBUG_MEMGUARD if (is_memguard_addr(__DECONST(void *, addr))) return (memguard_get_req_size(addr)); #endif #ifdef DEBUG_REDZONE size = redzone_get_size(__DECONST(void *, addr)); #else vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab); if (slab == NULL) panic("malloc_usable_size: address %p(%p) is not allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); if (!malloc_large_slab(slab)) size = zone->uz_size; else size = malloc_large_size(slab); #endif return (size); } CTASSERT(VM_KMEM_SIZE_SCALE >= 1); /* * Initialize the kernel memory (kmem) arena. */ void kmeminit(void) { u_long mem_size; u_long tmp; #ifdef VM_KMEM_SIZE if (vm_kmem_size == 0) vm_kmem_size = VM_KMEM_SIZE; #endif #ifdef VM_KMEM_SIZE_MIN if (vm_kmem_size_min == 0) vm_kmem_size_min = VM_KMEM_SIZE_MIN; #endif #ifdef VM_KMEM_SIZE_MAX if (vm_kmem_size_max == 0) vm_kmem_size_max = VM_KMEM_SIZE_MAX; #endif /* * Calculate the amount of kernel virtual address (KVA) space that is * preallocated to the kmem arena. In order to support a wide range * of machines, it is a function of the physical memory size, * specifically, * * min(max(physical memory size / VM_KMEM_SIZE_SCALE, * VM_KMEM_SIZE_MIN), VM_KMEM_SIZE_MAX) * * Every architecture must define an integral value for * VM_KMEM_SIZE_SCALE. However, the definitions of VM_KMEM_SIZE_MIN * and VM_KMEM_SIZE_MAX, which represent respectively the floor and * ceiling on this preallocation, are optional. Typically, * VM_KMEM_SIZE_MAX is itself a function of the available KVA space on * a given architecture. */ mem_size = vm_cnt.v_page_count; if (mem_size <= 32768) /* delphij XXX 128MB */ kmem_zmax = PAGE_SIZE; if (vm_kmem_size_scale < 1) vm_kmem_size_scale = VM_KMEM_SIZE_SCALE; /* * Check if we should use defaults for the "vm_kmem_size" * variable: */ if (vm_kmem_size == 0) { vm_kmem_size = mem_size / vm_kmem_size_scale; vm_kmem_size = vm_kmem_size * PAGE_SIZE < vm_kmem_size ? vm_kmem_size_max : vm_kmem_size * PAGE_SIZE; if (vm_kmem_size_min > 0 && vm_kmem_size < vm_kmem_size_min) vm_kmem_size = vm_kmem_size_min; if (vm_kmem_size_max > 0 && vm_kmem_size >= vm_kmem_size_max) vm_kmem_size = vm_kmem_size_max; } if (vm_kmem_size == 0) panic("Tune VM_KMEM_SIZE_* for the platform"); /* * The amount of KVA space that is preallocated to the * kmem arena can be set statically at compile-time or manually * through the kernel environment. However, it is still limited to * twice the physical memory size, which has been sufficient to handle * the most severe cases of external fragmentation in the kmem arena. */ if (vm_kmem_size / 2 / PAGE_SIZE > mem_size) vm_kmem_size = 2 * mem_size * PAGE_SIZE; vm_kmem_size = round_page(vm_kmem_size); #ifdef DEBUG_MEMGUARD tmp = memguard_fudge(vm_kmem_size, kernel_map); #else tmp = vm_kmem_size; #endif uma_set_limit(tmp); #ifdef DEBUG_MEMGUARD /* * Initialize MemGuard if support compiled in. MemGuard is a * replacement allocator used for detecting tamper-after-free * scenarios as they occur. It is only used for debugging. */ memguard_init(kernel_arena); #endif } /* * Initialize the kernel memory allocator */ /* ARGSUSED*/ static void mallocinit(void *dummy) { int i; uint8_t indx; mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF); kmeminit(); if (kmem_zmax < PAGE_SIZE || kmem_zmax > KMEM_ZMAX) kmem_zmax = KMEM_ZMAX; mt_stats_zone = uma_zcreate("mt_stats_zone", sizeof(struct malloc_type_stats), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); mt_zone = uma_zcreate("mt_zone", sizeof(struct malloc_type_internal), #ifdef INVARIANTS mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini, #else NULL, NULL, NULL, NULL, #endif UMA_ALIGN_PTR, UMA_ZONE_MALLOC); for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) { int size = kmemzones[indx].kz_size; const char *name = kmemzones[indx].kz_name; int subzone; for (subzone = 0; subzone < numzones; subzone++) { kmemzones[indx].kz_zone[subzone] = uma_zcreate(name, size, #ifdef INVARIANTS mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini, #else NULL, NULL, NULL, NULL, #endif UMA_ALIGN_PTR, UMA_ZONE_MALLOC); } for (;i <= size; i+= KMEM_ZBASE) kmemsize[i >> KMEM_ZSHIFT] = indx; } } SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_SECOND, mallocinit, NULL); void malloc_init(void *data) { struct malloc_type_internal *mtip; struct malloc_type *mtp; KASSERT(vm_cnt.v_page_count != 0, ("malloc_register before vm_init")); mtp = data; if (mtp->ks_magic != M_MAGIC) panic("malloc_init: bad malloc type magic"); mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO); mtip->mti_stats = uma_zalloc_pcpu(mt_stats_zone, M_WAITOK | M_ZERO); mtp->ks_handle = mtip; mtp_set_subzone(mtp); mtx_lock(&malloc_mtx); mtp->ks_next = kmemstatistics; kmemstatistics = mtp; kmemcount++; mtx_unlock(&malloc_mtx); } void malloc_uninit(void *data) { struct malloc_type_internal *mtip; struct malloc_type_stats *mtsp; struct malloc_type *mtp, *temp; uma_slab_t slab; long temp_allocs, temp_bytes; int i; mtp = data; KASSERT(mtp->ks_magic == M_MAGIC, ("malloc_uninit: bad malloc type magic")); KASSERT(mtp->ks_handle != NULL, ("malloc_deregister: cookie NULL")); mtx_lock(&malloc_mtx); mtip = mtp->ks_handle; mtp->ks_handle = NULL; if (mtp != kmemstatistics) { for (temp = kmemstatistics; temp != NULL; temp = temp->ks_next) { if (temp->ks_next == mtp) { temp->ks_next = mtp->ks_next; break; } } KASSERT(temp, ("malloc_uninit: type '%s' not found", mtp->ks_shortdesc)); } else kmemstatistics = mtp->ks_next; kmemcount--; mtx_unlock(&malloc_mtx); /* * Look for memory leaks. */ temp_allocs = temp_bytes = 0; for (i = 0; i <= mp_maxid; i++) { mtsp = zpcpu_get_cpu(mtip->mti_stats, i); temp_allocs += mtsp->mts_numallocs; temp_allocs -= mtsp->mts_numfrees; temp_bytes += mtsp->mts_memalloced; temp_bytes -= mtsp->mts_memfreed; } if (temp_allocs > 0 || temp_bytes > 0) { printf("Warning: memory type %s leaked memory on destroy " "(%ld allocations, %ld bytes leaked).\n", mtp->ks_shortdesc, temp_allocs, temp_bytes); } slab = vtoslab((vm_offset_t) mtip & (~UMA_SLAB_MASK)); uma_zfree_pcpu(mt_stats_zone, mtip->mti_stats); uma_zfree_arg(mt_zone, mtip, slab); } struct malloc_type * malloc_desc2type(const char *desc) { struct malloc_type *mtp; mtx_assert(&malloc_mtx, MA_OWNED); for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) { if (strcmp(mtp->ks_shortdesc, desc) == 0) return (mtp); } return (NULL); } static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS) { struct malloc_type_stream_header mtsh; struct malloc_type_internal *mtip; struct malloc_type_stats *mtsp, zeromts; struct malloc_type_header mth; struct malloc_type *mtp; int error, i; struct sbuf sbuf; error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL); mtx_lock(&malloc_mtx); bzero(&zeromts, sizeof(zeromts)); /* * Insert stream header. */ bzero(&mtsh, sizeof(mtsh)); mtsh.mtsh_version = MALLOC_TYPE_STREAM_VERSION; mtsh.mtsh_maxcpus = MAXCPU; mtsh.mtsh_count = kmemcount; (void)sbuf_bcat(&sbuf, &mtsh, sizeof(mtsh)); /* * Insert alternating sequence of type headers and type statistics. */ for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) { mtip = (struct malloc_type_internal *)mtp->ks_handle; /* * Insert type header. */ bzero(&mth, sizeof(mth)); strlcpy(mth.mth_name, mtp->ks_shortdesc, MALLOC_MAX_NAME); (void)sbuf_bcat(&sbuf, &mth, sizeof(mth)); /* * Insert type statistics for each CPU. */ for (i = 0; i <= mp_maxid; i++) { mtsp = zpcpu_get_cpu(mtip->mti_stats, i); (void)sbuf_bcat(&sbuf, mtsp, sizeof(*mtsp)); } /* * Fill in the missing CPUs. */ for (; i < MAXCPU; i++) { (void)sbuf_bcat(&sbuf, &zeromts, sizeof(zeromts)); } } mtx_unlock(&malloc_mtx); error = sbuf_finish(&sbuf); sbuf_delete(&sbuf); return (error); } SYSCTL_PROC(_kern, OID_AUTO, malloc_stats, CTLFLAG_RD | CTLTYPE_STRUCT | CTLFLAG_MPSAFE, 0, 0, sysctl_kern_malloc_stats, "s,malloc_type_ustats", "Return malloc types"); SYSCTL_INT(_kern, OID_AUTO, malloc_count, CTLFLAG_RD, &kmemcount, 0, "Count of kernel malloc types"); void malloc_type_list(malloc_type_list_func_t *func, void *arg) { struct malloc_type *mtp, **bufmtp; int count, i; size_t buflen; mtx_lock(&malloc_mtx); restart: mtx_assert(&malloc_mtx, MA_OWNED); count = kmemcount; mtx_unlock(&malloc_mtx); buflen = sizeof(struct malloc_type *) * count; bufmtp = malloc(buflen, M_TEMP, M_WAITOK); mtx_lock(&malloc_mtx); if (count < kmemcount) { free(bufmtp, M_TEMP); goto restart; } for (mtp = kmemstatistics, i = 0; mtp != NULL; mtp = mtp->ks_next, i++) bufmtp[i] = mtp; mtx_unlock(&malloc_mtx); for (i = 0; i < count; i++) (func)(bufmtp[i], arg); free(bufmtp, M_TEMP); } #ifdef DDB static int64_t get_malloc_stats(const struct malloc_type_internal *mtip, uint64_t *allocs, uint64_t *inuse) { const struct malloc_type_stats *mtsp; uint64_t frees, alloced, freed; int i; *allocs = 0; frees = 0; alloced = 0; freed = 0; for (i = 0; i <= mp_maxid; i++) { mtsp = zpcpu_get_cpu(mtip->mti_stats, i); *allocs += mtsp->mts_numallocs; frees += mtsp->mts_numfrees; alloced += mtsp->mts_memalloced; freed += mtsp->mts_memfreed; } *inuse = *allocs - frees; return (alloced - freed); } DB_SHOW_COMMAND(malloc, db_show_malloc) { const char *fmt_hdr, *fmt_entry; struct malloc_type *mtp; uint64_t allocs, inuse; int64_t size; /* variables for sorting */ struct malloc_type *last_mtype, *cur_mtype; int64_t cur_size, last_size; int ties; if (modif[0] == 'i') { fmt_hdr = "%s,%s,%s,%s\n"; fmt_entry = "\"%s\",%ju,%jdK,%ju\n"; } else { fmt_hdr = "%18s %12s %12s %12s\n"; fmt_entry = "%18s %12ju %12jdK %12ju\n"; } db_printf(fmt_hdr, "Type", "InUse", "MemUse", "Requests"); /* Select sort, largest size first. */ last_mtype = NULL; last_size = INT64_MAX; for (;;) { cur_mtype = NULL; cur_size = -1; ties = 0; for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) { /* * In the case of size ties, print out mtypes * in the order they are encountered. That is, * when we encounter the most recently output * mtype, we have already printed all preceding * ties, and we must print all following ties. */ if (mtp == last_mtype) { ties = 1; continue; } size = get_malloc_stats(mtp->ks_handle, &allocs, &inuse); if (size > cur_size && size < last_size + ties) { cur_size = size; cur_mtype = mtp; } } if (cur_mtype == NULL) break; size = get_malloc_stats(cur_mtype->ks_handle, &allocs, &inuse); db_printf(fmt_entry, cur_mtype->ks_shortdesc, inuse, howmany(size, 1024), allocs); if (db_pager_quit) break; last_mtype = cur_mtype; last_size = cur_size; } } #if MALLOC_DEBUG_MAXZONES > 1 DB_SHOW_COMMAND(multizone_matches, db_show_multizone_matches) { struct malloc_type_internal *mtip; struct malloc_type *mtp; u_int subzone; if (!have_addr) { db_printf("Usage: show multizone_matches \n"); return; } mtp = (void *)addr; if (mtp->ks_magic != M_MAGIC) { db_printf("Magic %lx does not match expected %x\n", mtp->ks_magic, M_MAGIC); return; } mtip = mtp->ks_handle; subzone = mtip->mti_zone; for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) { mtip = mtp->ks_handle; if (mtip->mti_zone != subzone) continue; db_printf("%s\n", mtp->ks_shortdesc); if (db_pager_quit) break; } } #endif /* MALLOC_DEBUG_MAXZONES > 1 */ #endif /* DDB */ #ifdef MALLOC_PROFILE static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS) { struct sbuf sbuf; uint64_t count; uint64_t waste; uint64_t mem; int error; int rsize; int size; int i; waste = 0; mem = 0; error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); sbuf_printf(&sbuf, "\n Size Requests Real Size\n"); for (i = 0; i < KMEM_ZSIZE; i++) { size = i << KMEM_ZSHIFT; rsize = kmemzones[kmemsize[i]].kz_size; count = (long long unsigned)krequests[i]; sbuf_printf(&sbuf, "%6d%28llu%11d\n", size, (unsigned long long)count, rsize); if ((rsize * count) > (size * count)) waste += (rsize * count) - (size * count); mem += (rsize * count); } sbuf_printf(&sbuf, "\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n", (unsigned long long)mem, (unsigned long long)waste); error = sbuf_finish(&sbuf); sbuf_delete(&sbuf); return (error); } SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling"); #endif /* MALLOC_PROFILE */ Index: head/sys/kern/link_elf.c =================================================================== --- head/sys/kern/link_elf.c (revision 367185) +++ head/sys/kern/link_elf.c (revision 367186) @@ -1,1942 +1,1942 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1998-2000 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include "opt_gdb.h" #include #include #ifdef GPROF #include #endif #include #include #include #ifdef SPARSE_MAPPING #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SPARSE_MAPPING #include #include #include #endif #include #include #include #include "linker_if.h" #define MAXSEGS 4 typedef struct elf_file { struct linker_file lf; /* Common fields */ int preloaded; /* Was file pre-loaded */ caddr_t address; /* Relocation address */ #ifdef SPARSE_MAPPING vm_object_t object; /* VM object to hold file pages */ #endif Elf_Dyn *dynamic; /* Symbol table etc. */ Elf_Hashelt nbuckets; /* DT_HASH info */ Elf_Hashelt nchains; const Elf_Hashelt *buckets; const Elf_Hashelt *chains; caddr_t hash; caddr_t strtab; /* DT_STRTAB */ int strsz; /* DT_STRSZ */ const Elf_Sym *symtab; /* DT_SYMTAB */ Elf_Addr *got; /* DT_PLTGOT */ const Elf_Rel *pltrel; /* DT_JMPREL */ int pltrelsize; /* DT_PLTRELSZ */ const Elf_Rela *pltrela; /* DT_JMPREL */ int pltrelasize; /* DT_PLTRELSZ */ const Elf_Rel *rel; /* DT_REL */ int relsize; /* DT_RELSZ */ const Elf_Rela *rela; /* DT_RELA */ int relasize; /* DT_RELASZ */ caddr_t modptr; const Elf_Sym *ddbsymtab; /* The symbol table we are using */ long ddbsymcnt; /* Number of symbols */ caddr_t ddbstrtab; /* String table */ long ddbstrcnt; /* number of bytes in string table */ caddr_t symbase; /* malloc'ed symbold base */ caddr_t strbase; /* malloc'ed string base */ caddr_t ctftab; /* CTF table */ long ctfcnt; /* number of bytes in CTF table */ caddr_t ctfoff; /* CTF offset table */ caddr_t typoff; /* Type offset table */ long typlen; /* Number of type entries. */ Elf_Addr pcpu_start; /* Pre-relocation pcpu set start. */ Elf_Addr pcpu_stop; /* Pre-relocation pcpu set stop. */ Elf_Addr pcpu_base; /* Relocated pcpu set address. */ #ifdef VIMAGE Elf_Addr vnet_start; /* Pre-relocation vnet set start. */ Elf_Addr vnet_stop; /* Pre-relocation vnet set stop. */ Elf_Addr vnet_base; /* Relocated vnet set address. */ #endif #ifdef GDB struct link_map gdb; /* hooks for gdb */ #endif } *elf_file_t; struct elf_set { Elf_Addr es_start; Elf_Addr es_stop; Elf_Addr es_base; TAILQ_ENTRY(elf_set) es_link; }; TAILQ_HEAD(elf_set_head, elf_set); #include static int link_elf_link_common_finish(linker_file_t); static int link_elf_link_preload(linker_class_t cls, const char *, linker_file_t *); static int link_elf_link_preload_finish(linker_file_t); static int link_elf_load_file(linker_class_t, const char *, linker_file_t *); static int link_elf_lookup_symbol(linker_file_t, const char *, c_linker_sym_t *); static int link_elf_symbol_values(linker_file_t, c_linker_sym_t, linker_symval_t *); static int link_elf_search_symbol(linker_file_t, caddr_t, c_linker_sym_t *, long *); static void link_elf_unload_file(linker_file_t); static void link_elf_unload_preload(linker_file_t); static int link_elf_lookup_set(linker_file_t, const char *, void ***, void ***, int *); static int link_elf_each_function_name(linker_file_t, int (*)(const char *, void *), void *); static int link_elf_each_function_nameval(linker_file_t, linker_function_nameval_callback_t, void *); static void link_elf_reloc_local(linker_file_t); static long link_elf_symtab_get(linker_file_t, const Elf_Sym **); static long link_elf_strtab_get(linker_file_t, caddr_t *); static int elf_lookup(linker_file_t, Elf_Size, int, Elf_Addr *); static kobj_method_t link_elf_methods[] = { KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol), KOBJMETHOD(linker_symbol_values, link_elf_symbol_values), KOBJMETHOD(linker_search_symbol, link_elf_search_symbol), KOBJMETHOD(linker_unload, link_elf_unload_file), KOBJMETHOD(linker_load_file, link_elf_load_file), KOBJMETHOD(linker_link_preload, link_elf_link_preload), KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish), KOBJMETHOD(linker_lookup_set, link_elf_lookup_set), KOBJMETHOD(linker_each_function_name, link_elf_each_function_name), KOBJMETHOD(linker_each_function_nameval, link_elf_each_function_nameval), KOBJMETHOD(linker_ctf_get, link_elf_ctf_get), KOBJMETHOD(linker_symtab_get, link_elf_symtab_get), KOBJMETHOD(linker_strtab_get, link_elf_strtab_get), KOBJMETHOD_END }; static struct linker_class link_elf_class = { #if ELF_TARG_CLASS == ELFCLASS32 "elf32", #else "elf64", #endif link_elf_methods, sizeof(struct elf_file) }; typedef int (*elf_reloc_fn)(linker_file_t lf, Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup); static int parse_dynamic(elf_file_t); static int relocate_file(elf_file_t); static int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, bool ifuncs); static int link_elf_preload_parse_symbols(elf_file_t); static struct elf_set_head set_pcpu_list; #ifdef VIMAGE static struct elf_set_head set_vnet_list; #endif static void elf_set_add(struct elf_set_head *list, Elf_Addr start, Elf_Addr stop, Elf_Addr base) { struct elf_set *set, *iter; set = malloc(sizeof(*set), M_LINKER, M_WAITOK); set->es_start = start; set->es_stop = stop; set->es_base = base; TAILQ_FOREACH(iter, list, es_link) { KASSERT((set->es_start < iter->es_start && set->es_stop < iter->es_stop) || (set->es_start > iter->es_start && set->es_stop > iter->es_stop), ("linker sets intersection: to insert: 0x%jx-0x%jx; inserted: 0x%jx-0x%jx", (uintmax_t)set->es_start, (uintmax_t)set->es_stop, (uintmax_t)iter->es_start, (uintmax_t)iter->es_stop)); if (iter->es_start > set->es_start) { TAILQ_INSERT_BEFORE(iter, set, es_link); break; } } if (iter == NULL) TAILQ_INSERT_TAIL(list, set, es_link); } static int elf_set_find(struct elf_set_head *list, Elf_Addr addr, Elf_Addr *start, Elf_Addr *base) { struct elf_set *set; TAILQ_FOREACH(set, list, es_link) { if (addr < set->es_start) return (0); if (addr < set->es_stop) { *start = set->es_start; *base = set->es_base; return (1); } } return (0); } static void elf_set_delete(struct elf_set_head *list, Elf_Addr start) { struct elf_set *set; TAILQ_FOREACH(set, list, es_link) { if (start < set->es_start) break; if (start == set->es_start) { TAILQ_REMOVE(list, set, es_link); free(set, M_LINKER); return; } } KASSERT(0, ("deleting unknown linker set (start = 0x%jx)", (uintmax_t)start)); } #ifdef GDB static void r_debug_state(struct r_debug *, struct link_map *); /* * A list of loaded modules for GDB to use for loading symbols. */ struct r_debug r_debug; #define GDB_STATE(s) do { \ r_debug.r_state = s; r_debug_state(NULL, NULL); \ } while (0) /* * Function for the debugger to set a breakpoint on to gain control. */ static void r_debug_state(struct r_debug *dummy_one __unused, struct link_map *dummy_two __unused) { } static void link_elf_add_gdb(struct link_map *l) { struct link_map *prev; l->l_next = NULL; if (r_debug.r_map == NULL) { /* Add first. */ l->l_prev = NULL; r_debug.r_map = l; } else { /* Append to list. */ for (prev = r_debug.r_map; prev->l_next != NULL; prev = prev->l_next) ; l->l_prev = prev; prev->l_next = l; } } static void link_elf_delete_gdb(struct link_map *l) { if (l->l_prev == NULL) { /* Remove first. */ if ((r_debug.r_map = l->l_next) != NULL) l->l_next->l_prev = NULL; } else { /* Remove any but first. */ if ((l->l_prev->l_next = l->l_next) != NULL) l->l_next->l_prev = l->l_prev; } } #endif /* GDB */ /* * The kernel symbol table starts here. */ extern struct _dynamic _DYNAMIC; static void link_elf_error(const char *filename, const char *s) { if (filename == NULL) printf("kldload: %s\n", s); else printf("kldload: %s: %s\n", filename, s); } static void link_elf_invoke_ctors(caddr_t addr, size_t size) { void (**ctor)(void); size_t i, cnt; if (addr == NULL || size == 0) return; cnt = size / sizeof(*ctor); ctor = (void *)addr; for (i = 0; i < cnt; i++) { if (ctor[i] != NULL) (*ctor[i])(); } } /* * Actions performed after linking/loading both the preloaded kernel and any * modules; whether preloaded or dynamicly loaded. */ static int link_elf_link_common_finish(linker_file_t lf) { #ifdef GDB elf_file_t ef = (elf_file_t)lf; char *newfilename; #endif int error; /* Notify MD code that a module is being loaded. */ error = elf_cpu_load_file(lf); if (error != 0) return (error); #ifdef GDB GDB_STATE(RT_ADD); ef->gdb.l_addr = lf->address; newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK); strcpy(newfilename, lf->filename); ef->gdb.l_name = newfilename; ef->gdb.l_ld = ef->dynamic; link_elf_add_gdb(&ef->gdb); GDB_STATE(RT_CONSISTENT); #endif /* Invoke .ctors */ link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size); return (0); } #ifdef RELOCATABLE_KERNEL /* * __startkernel and __endkernel are symbols set up as relocation canaries. * * They are defined in locore to reference linker script symbols at the * beginning and end of the LOAD area. This has the desired side effect of * giving us variables that have relative relocations pointing at them, so * relocation of the kernel object will cause the variables to be updated * automatically by the runtime linker when we initialize. * * There are two main reasons to relocate the kernel: * 1) If the loader needed to load the kernel at an alternate load address. * 2) If the kernel is switching address spaces on machines like POWER9 * under Radix where the high bits of the effective address are used to * differentiate between hypervisor, host, guest, and problem state. */ extern vm_offset_t __startkernel, __endkernel; #endif static unsigned long kern_relbase = KERNBASE; SYSCTL_ULONG(_kern, OID_AUTO, base_address, CTLFLAG_RD, SYSCTL_NULL_ULONG_PTR, KERNBASE, "Kernel base address"); SYSCTL_ULONG(_kern, OID_AUTO, relbase_address, CTLFLAG_RD, &kern_relbase, 0, "Kernel relocated base address"); static void link_elf_init(void* arg) { Elf_Dyn *dp; Elf_Addr *ctors_addrp; Elf_Size *ctors_sizep; caddr_t modptr, baseptr, sizeptr; elf_file_t ef; const char *modname; linker_add_class(&link_elf_class); dp = (Elf_Dyn *)&_DYNAMIC; modname = NULL; modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel"); if (modptr == NULL) modptr = preload_search_by_type("elf kernel"); modname = (char *)preload_search_info(modptr, MODINFO_NAME); if (modname == NULL) modname = "kernel"; linker_kernel_file = linker_make_file(modname, &link_elf_class); if (linker_kernel_file == NULL) panic("%s: Can't create linker structures for kernel", __func__); ef = (elf_file_t) linker_kernel_file; ef->preloaded = 1; #ifdef RELOCATABLE_KERNEL /* Compute relative displacement */ ef->address = (caddr_t) (__startkernel - KERNBASE); #else ef->address = 0; #endif #ifdef SPARSE_MAPPING ef->object = NULL; #endif ef->dynamic = dp; if (dp != NULL) parse_dynamic(ef); #ifdef RELOCATABLE_KERNEL linker_kernel_file->address = (caddr_t)__startkernel; linker_kernel_file->size = (intptr_t)(__endkernel - __startkernel); kern_relbase = (unsigned long)__startkernel; #else linker_kernel_file->address += KERNBASE; linker_kernel_file->size = -(intptr_t)linker_kernel_file->address; #endif if (modptr != NULL) { ef->modptr = modptr; baseptr = preload_search_info(modptr, MODINFO_ADDR); if (baseptr != NULL) linker_kernel_file->address = *(caddr_t *)baseptr; sizeptr = preload_search_info(modptr, MODINFO_SIZE); if (sizeptr != NULL) linker_kernel_file->size = *(size_t *)sizeptr; ctors_addrp = (Elf_Addr *)preload_search_info(modptr, MODINFO_METADATA | MODINFOMD_CTORS_ADDR); ctors_sizep = (Elf_Size *)preload_search_info(modptr, MODINFO_METADATA | MODINFOMD_CTORS_SIZE); if (ctors_addrp != NULL && ctors_sizep != NULL) { linker_kernel_file->ctors_addr = ef->address + *ctors_addrp; linker_kernel_file->ctors_size = *ctors_sizep; } } (void)link_elf_preload_parse_symbols(ef); #ifdef GDB r_debug.r_map = NULL; r_debug.r_brk = r_debug_state; r_debug.r_state = RT_CONSISTENT; #endif (void)link_elf_link_common_finish(linker_kernel_file); linker_kernel_file->flags |= LINKER_FILE_LINKED; TAILQ_INIT(&set_pcpu_list); #ifdef VIMAGE TAILQ_INIT(&set_vnet_list); #endif } SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, NULL); static int link_elf_preload_parse_symbols(elf_file_t ef) { caddr_t pointer; caddr_t ssym, esym, base; caddr_t strtab; int strcnt; Elf_Sym *symtab; int symcnt; if (ef->modptr == NULL) return (0); pointer = preload_search_info(ef->modptr, MODINFO_METADATA | MODINFOMD_SSYM); if (pointer == NULL) return (0); ssym = *(caddr_t *)pointer; pointer = preload_search_info(ef->modptr, MODINFO_METADATA | MODINFOMD_ESYM); if (pointer == NULL) return (0); esym = *(caddr_t *)pointer; base = ssym; symcnt = *(long *)base; base += sizeof(long); symtab = (Elf_Sym *)base; base += roundup(symcnt, sizeof(long)); if (base > esym || base < ssym) { printf("Symbols are corrupt!\n"); return (EINVAL); } strcnt = *(long *)base; base += sizeof(long); strtab = base; base += roundup(strcnt, sizeof(long)); if (base > esym || base < ssym) { printf("Symbols are corrupt!\n"); return (EINVAL); } ef->ddbsymtab = symtab; ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); ef->ddbstrtab = strtab; ef->ddbstrcnt = strcnt; return (0); } static int parse_dynamic(elf_file_t ef) { Elf_Dyn *dp; int plttype = DT_REL; for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) { switch (dp->d_tag) { case DT_HASH: { /* From src/libexec/rtld-elf/rtld.c */ const Elf_Hashelt *hashtab = (const Elf_Hashelt *) (ef->address + dp->d_un.d_ptr); ef->nbuckets = hashtab[0]; ef->nchains = hashtab[1]; ef->buckets = hashtab + 2; ef->chains = ef->buckets + ef->nbuckets; break; } case DT_STRTAB: ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr); break; case DT_STRSZ: ef->strsz = dp->d_un.d_val; break; case DT_SYMTAB: ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr); break; case DT_SYMENT: if (dp->d_un.d_val != sizeof(Elf_Sym)) return (ENOEXEC); break; case DT_PLTGOT: ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr); break; case DT_REL: ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); break; case DT_RELSZ: ef->relsize = dp->d_un.d_val; break; case DT_RELENT: if (dp->d_un.d_val != sizeof(Elf_Rel)) return (ENOEXEC); break; case DT_JMPREL: ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); break; case DT_PLTRELSZ: ef->pltrelsize = dp->d_un.d_val; break; case DT_RELA: ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr); break; case DT_RELASZ: ef->relasize = dp->d_un.d_val; break; case DT_RELAENT: if (dp->d_un.d_val != sizeof(Elf_Rela)) return (ENOEXEC); break; case DT_PLTREL: plttype = dp->d_un.d_val; if (plttype != DT_REL && plttype != DT_RELA) return (ENOEXEC); break; #ifdef GDB case DT_DEBUG: dp->d_un.d_ptr = (Elf_Addr)&r_debug; break; #endif } } if (plttype == DT_RELA) { ef->pltrela = (const Elf_Rela *)ef->pltrel; ef->pltrel = NULL; ef->pltrelasize = ef->pltrelsize; ef->pltrelsize = 0; } ef->ddbsymtab = ef->symtab; ef->ddbsymcnt = ef->nchains; ef->ddbstrtab = ef->strtab; ef->ddbstrcnt = ef->strsz; return elf_cpu_parse_dynamic(ef->address, ef->dynamic); } #define LS_PADDING 0x90909090 static int parse_dpcpu(elf_file_t ef) { int error, size; #if defined(__i386__) uint32_t pad; #endif ef->pcpu_start = 0; ef->pcpu_stop = 0; error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start, (void ***)&ef->pcpu_stop, NULL); /* Error just means there is no pcpu set to relocate. */ if (error != 0) return (0); size = (uintptr_t)ef->pcpu_stop - (uintptr_t)ef->pcpu_start; /* Empty set? */ if (size < 1) return (0); #if defined(__i386__) /* In case we do find __start/stop_set_ symbols double-check. */ if (size < 4) { uprintf("Kernel module '%s' must be recompiled with " "linker script\n", ef->lf.pathname); return (ENOEXEC); } /* Padding from linker-script correct? */ pad = *(uint32_t *)((uintptr_t)ef->pcpu_stop - sizeof(pad)); if (pad != LS_PADDING) { uprintf("Kernel module '%s' must be recompiled with " "linker script, invalid padding %#04x (%#04x)\n", ef->lf.pathname, pad, LS_PADDING); return (ENOEXEC); } /* If we only have valid padding, nothing to do. */ if (size == 4) return (0); #endif /* * Allocate space in the primary pcpu area. Copy in our * initialization from the data section and then initialize * all per-cpu storage from that. */ ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(size); if (ef->pcpu_base == 0) { printf("%s: pcpu module space is out of space; " "cannot allocate %d for %s\n", __func__, size, ef->lf.pathname); return (ENOSPC); } memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, size); dpcpu_copy((void *)ef->pcpu_base, size); elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop, ef->pcpu_base); return (0); } #ifdef VIMAGE static int parse_vnet(elf_file_t ef) { int error, size; #if defined(__i386__) uint32_t pad; #endif ef->vnet_start = 0; ef->vnet_stop = 0; error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start, (void ***)&ef->vnet_stop, NULL); /* Error just means there is no vnet data set to relocate. */ if (error != 0) return (0); size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start; /* Empty set? */ if (size < 1) return (0); #if defined(__i386__) /* In case we do find __start/stop_set_ symbols double-check. */ if (size < 4) { uprintf("Kernel module '%s' must be recompiled with " "linker script\n", ef->lf.pathname); return (ENOEXEC); } /* Padding from linker-script correct? */ pad = *(uint32_t *)((uintptr_t)ef->vnet_stop - sizeof(pad)); if (pad != LS_PADDING) { uprintf("Kernel module '%s' must be recompiled with " "linker script, invalid padding %#04x (%#04x)\n", ef->lf.pathname, pad, LS_PADDING); return (ENOEXEC); } /* If we only have valid padding, nothing to do. */ if (size == 4) return (0); #endif /* * Allocate space in the primary vnet area. Copy in our * initialization from the data section and then initialize * all per-vnet storage from that. */ ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size); if (ef->vnet_base == 0) { printf("%s: vnet module space is out of space; " "cannot allocate %d for %s\n", __func__, size, ef->lf.pathname); return (ENOSPC); } memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, size); vnet_data_copy((void *)ef->vnet_base, size); elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop, ef->vnet_base); return (0); } #endif #undef LS_PADDING /* * Apply the specified protection to the loadable segments of a preloaded linker * file. */ static int preload_protect(elf_file_t ef, vm_prot_t prot) { #ifdef __amd64__ Elf_Ehdr *hdr; Elf_Phdr *phdr, *phlimit; vm_prot_t nprot; int error; error = 0; hdr = (Elf_Ehdr *)ef->address; phdr = (Elf_Phdr *)(ef->address + hdr->e_phoff); phlimit = phdr + hdr->e_phnum; for (; phdr < phlimit; phdr++) { if (phdr->p_type != PT_LOAD) continue; nprot = prot | VM_PROT_READ; if ((phdr->p_flags & PF_W) != 0) nprot |= VM_PROT_WRITE; if ((phdr->p_flags & PF_X) != 0) nprot |= VM_PROT_EXECUTE; error = pmap_change_prot((vm_offset_t)ef->address + phdr->p_vaddr, round_page(phdr->p_memsz), nprot); if (error != 0) break; } return (error); #else return (0); #endif } #ifdef __arm__ /* * Locate the ARM exception/unwind table info for DDB and stack(9) use by * searching for the section header that describes it. There may be no unwind * info, for example in a module containing only data. */ static void link_elf_locate_exidx(linker_file_t lf, Elf_Shdr *shdr, int nhdr) { int i; for (i = 0; i < nhdr; i++) { if (shdr[i].sh_type == SHT_ARM_EXIDX) { lf->exidx_addr = shdr[i].sh_addr + lf->address; lf->exidx_size = shdr[i].sh_size; break; } } } /* * Locate the section headers metadata in a preloaded module, then use it to * locate the exception/unwind table in the module. The size of the metadata * block is stored in a uint32 word immediately before the data itself, and a * comment in preload_search_info() says it is safe to rely on that. */ static void link_elf_locate_exidx_preload(struct linker_file *lf, caddr_t modptr) { uint32_t *modinfo; Elf_Shdr *shdr; uint32_t nhdr; modinfo = (uint32_t *)preload_search_info(modptr, MODINFO_METADATA | MODINFOMD_SHDR); if (modinfo != NULL) { shdr = (Elf_Shdr *)modinfo; nhdr = modinfo[-1] / sizeof(Elf_Shdr); link_elf_locate_exidx(lf, shdr, nhdr); } } #endif /* __arm__ */ static int link_elf_link_preload(linker_class_t cls, const char *filename, linker_file_t *result) { Elf_Addr *ctors_addrp; Elf_Size *ctors_sizep; caddr_t modptr, baseptr, sizeptr, dynptr; char *type; elf_file_t ef; linker_file_t lf; int error; vm_offset_t dp; /* Look to see if we have the file preloaded */ modptr = preload_search_by_name(filename); if (modptr == NULL) return (ENOENT); type = (char *)preload_search_info(modptr, MODINFO_TYPE); baseptr = preload_search_info(modptr, MODINFO_ADDR); sizeptr = preload_search_info(modptr, MODINFO_SIZE); dynptr = preload_search_info(modptr, MODINFO_METADATA | MODINFOMD_DYNAMIC); if (type == NULL || (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 && strcmp(type, "elf module") != 0)) return (EFTYPE); if (baseptr == NULL || sizeptr == NULL || dynptr == NULL) return (EINVAL); lf = linker_make_file(filename, &link_elf_class); if (lf == NULL) return (ENOMEM); ef = (elf_file_t) lf; ef->preloaded = 1; ef->modptr = modptr; ef->address = *(caddr_t *)baseptr; #ifdef SPARSE_MAPPING ef->object = NULL; #endif dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr; ef->dynamic = (Elf_Dyn *)dp; lf->address = ef->address; lf->size = *(size_t *)sizeptr; ctors_addrp = (Elf_Addr *)preload_search_info(modptr, MODINFO_METADATA | MODINFOMD_CTORS_ADDR); ctors_sizep = (Elf_Size *)preload_search_info(modptr, MODINFO_METADATA | MODINFOMD_CTORS_SIZE); if (ctors_addrp != NULL && ctors_sizep != NULL) { lf->ctors_addr = ef->address + *ctors_addrp; lf->ctors_size = *ctors_sizep; } #ifdef __arm__ link_elf_locate_exidx_preload(lf, modptr); #endif error = parse_dynamic(ef); if (error == 0) error = parse_dpcpu(ef); #ifdef VIMAGE if (error == 0) error = parse_vnet(ef); #endif if (error == 0) error = preload_protect(ef, VM_PROT_ALL); if (error != 0) { linker_file_unload(lf, LINKER_UNLOAD_FORCE); return (error); } link_elf_reloc_local(lf); *result = lf; return (0); } static int link_elf_link_preload_finish(linker_file_t lf) { elf_file_t ef; int error; ef = (elf_file_t) lf; error = relocate_file(ef); if (error == 0) error = preload_protect(ef, VM_PROT_NONE); if (error != 0) return (error); (void)link_elf_preload_parse_symbols(ef); return (link_elf_link_common_finish(lf)); } static int link_elf_load_file(linker_class_t cls, const char* filename, linker_file_t* result) { struct nameidata nd; struct thread* td = curthread; /* XXX */ Elf_Ehdr *hdr; caddr_t firstpage, segbase; int nbytes, i; Elf_Phdr *phdr; Elf_Phdr *phlimit; Elf_Phdr *segs[MAXSEGS]; int nsegs; Elf_Phdr *phdyn; caddr_t mapbase; size_t mapsize; Elf_Addr base_vaddr; Elf_Addr base_vlimit; int error = 0; ssize_t resid; int flags; elf_file_t ef; linker_file_t lf; Elf_Shdr *shdr; int symtabindex; int symstrindex; int shstrindex; int symcnt; int strcnt; char *shstrs; shdr = NULL; lf = NULL; shstrs = NULL; NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td); flags = FREAD; error = vn_open(&nd, &flags, 0, NULL); if (error != 0) return (error); NDFREE(&nd, NDF_ONLY_PNBUF); if (nd.ni_vp->v_type != VREG) { error = ENOEXEC; firstpage = NULL; goto out; } #ifdef MAC error = mac_kld_check_load(curthread->td_ucred, nd.ni_vp); if (error != 0) { firstpage = NULL; goto out; } #endif /* * Read the elf header from the file. */ firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK); hdr = (Elf_Ehdr *)firstpage; error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); nbytes = PAGE_SIZE - resid; if (error != 0) goto out; if (!IS_ELF(*hdr)) { error = ENOEXEC; goto out; } if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { link_elf_error(filename, "Unsupported file layout"); error = ENOEXEC; goto out; } if (hdr->e_ident[EI_VERSION] != EV_CURRENT || hdr->e_version != EV_CURRENT) { link_elf_error(filename, "Unsupported file version"); error = ENOEXEC; goto out; } if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) { error = ENOSYS; goto out; } if (hdr->e_machine != ELF_TARG_MACH) { link_elf_error(filename, "Unsupported machine"); error = ENOEXEC; goto out; } /* * We rely on the program header being in the first page. * This is not strictly required by the ABI specification, but * it seems to always true in practice. And, it simplifies * things considerably. */ if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) && (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) && (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes))) link_elf_error(filename, "Unreadable program headers"); /* * Scan the program header entries, and save key information. * * We rely on there being exactly two load segments, text and data, * in that order. */ phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff); phlimit = phdr + hdr->e_phnum; nsegs = 0; phdyn = NULL; while (phdr < phlimit) { switch (phdr->p_type) { case PT_LOAD: if (nsegs == MAXSEGS) { link_elf_error(filename, "Too many sections"); error = ENOEXEC; goto out; } /* * XXX: We just trust they come in right order ?? */ segs[nsegs] = phdr; ++nsegs; break; case PT_DYNAMIC: phdyn = phdr; break; case PT_INTERP: error = ENOSYS; goto out; } ++phdr; } if (phdyn == NULL) { link_elf_error(filename, "Object is not dynamically-linked"); error = ENOEXEC; goto out; } if (nsegs == 0) { link_elf_error(filename, "No sections"); error = ENOEXEC; goto out; } /* * Allocate the entire address space of the object, to stake * out our contiguous region, and to establish the base * address for relocation. */ base_vaddr = trunc_page(segs[0]->p_vaddr); base_vlimit = round_page(segs[nsegs - 1]->p_vaddr + segs[nsegs - 1]->p_memsz); mapsize = base_vlimit - base_vaddr; lf = linker_make_file(filename, &link_elf_class); if (lf == NULL) { error = ENOMEM; goto out; } ef = (elf_file_t) lf; #ifdef SPARSE_MAPPING ef->object = vm_pager_allocate(OBJT_PHYS, NULL, mapsize, VM_PROT_ALL, 0, thread0.td_ucred); if (ef->object == NULL) { error = ENOMEM; goto out; } #ifdef __amd64__ mapbase = (caddr_t)KERNBASE; #else mapbase = (caddr_t)vm_map_min(kernel_map); #endif /* * Mapping protections are downgraded after relocation processing. */ error = vm_map_find(kernel_map, ef->object, 0, (vm_offset_t *)&mapbase, mapsize, 0, VMFS_OPTIMAL_SPACE, VM_PROT_ALL, VM_PROT_ALL, 0); if (error != 0) { vm_object_deallocate(ef->object); ef->object = NULL; goto out; } #else - mapbase = malloc(mapsize, M_LINKER, M_EXEC | M_WAITOK); + mapbase = malloc_exec(mapsize, M_LINKER, M_WAITOK); #endif ef->address = mapbase; /* * Read the text and data sections and zero the bss. */ for (i = 0; i < nsegs; i++) { segbase = mapbase + segs[i]->p_vaddr - base_vaddr; #ifdef SPARSE_MAPPING /* * Consecutive segments may have different mapping permissions, * so be strict and verify that their mappings do not overlap. */ if (((vm_offset_t)segbase & PAGE_MASK) != 0) { error = EINVAL; goto out; } error = vm_map_wire(kernel_map, (vm_offset_t)segbase, (vm_offset_t)segbase + round_page(segs[i]->p_memsz), VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); if (error != KERN_SUCCESS) { error = ENOMEM; goto out; } #endif error = vn_rdwr(UIO_READ, nd.ni_vp, segbase, segs[i]->p_filesz, segs[i]->p_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error != 0) goto out; bzero(segbase + segs[i]->p_filesz, segs[i]->p_memsz - segs[i]->p_filesz); } #ifdef GPROF /* Update profiling information with the new text segment. */ mtx_lock(&Giant); kmupetext((uintfptr_t)(mapbase + segs[0]->p_vaddr - base_vaddr + segs[0]->p_memsz)); mtx_unlock(&Giant); #endif ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr); lf->address = ef->address; lf->size = mapsize; error = parse_dynamic(ef); if (error != 0) goto out; error = parse_dpcpu(ef); if (error != 0) goto out; #ifdef VIMAGE error = parse_vnet(ef); if (error != 0) goto out; #endif link_elf_reloc_local(lf); VOP_UNLOCK(nd.ni_vp); error = linker_load_dependencies(lf); vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); if (error != 0) goto out; error = relocate_file(ef); if (error != 0) goto out; #ifdef SPARSE_MAPPING /* * Downgrade permissions on text segment mappings now that relocation * processing is complete. Restrict permissions on read-only segments. */ for (i = 0; i < nsegs; i++) { vm_prot_t prot; if (segs[i]->p_type != PT_LOAD) continue; prot = VM_PROT_READ; if ((segs[i]->p_flags & PF_W) != 0) prot |= VM_PROT_WRITE; if ((segs[i]->p_flags & PF_X) != 0) prot |= VM_PROT_EXECUTE; segbase = mapbase + segs[i]->p_vaddr - base_vaddr; error = vm_map_protect(kernel_map, (vm_offset_t)segbase, (vm_offset_t)segbase + round_page(segs[i]->p_memsz), prot, FALSE); if (error != KERN_SUCCESS) { error = ENOMEM; goto out; } } #endif /* * Try and load the symbol table if it's present. (you can * strip it!) */ nbytes = hdr->e_shnum * hdr->e_shentsize; if (nbytes == 0 || hdr->e_shoff == 0) goto nosyms; shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shdr, nbytes, hdr->e_shoff, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error != 0) goto out; /* Read section string table */ shstrindex = hdr->e_shstrndx; if (shstrindex != 0 && shdr[shstrindex].sh_type == SHT_STRTAB && shdr[shstrindex].sh_size != 0) { nbytes = shdr[shstrindex].sh_size; shstrs = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shstrs, nbytes, shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; } symtabindex = -1; symstrindex = -1; for (i = 0; i < hdr->e_shnum; i++) { if (shdr[i].sh_type == SHT_SYMTAB) { symtabindex = i; symstrindex = shdr[i].sh_link; } else if (shstrs != NULL && shdr[i].sh_name != 0 && strcmp(shstrs + shdr[i].sh_name, ".ctors") == 0) { /* Record relocated address and size of .ctors. */ lf->ctors_addr = mapbase + shdr[i].sh_addr - base_vaddr; lf->ctors_size = shdr[i].sh_size; } } if (symtabindex < 0 || symstrindex < 0) goto nosyms; symcnt = shdr[symtabindex].sh_size; ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK); strcnt = shdr[symstrindex].sh_size; ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, nd.ni_vp, ef->symbase, symcnt, shdr[symtabindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error != 0) goto out; error = vn_rdwr(UIO_READ, nd.ni_vp, ef->strbase, strcnt, shdr[symstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error != 0) goto out; ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); ef->ddbsymtab = (const Elf_Sym *)ef->symbase; ef->ddbstrcnt = strcnt; ef->ddbstrtab = ef->strbase; nosyms: #ifdef __arm__ link_elf_locate_exidx(lf, shdr, hdr->e_shnum); #endif error = link_elf_link_common_finish(lf); if (error != 0) goto out; *result = lf; out: VOP_UNLOCK(nd.ni_vp); vn_close(nd.ni_vp, FREAD, td->td_ucred, td); if (error != 0 && lf != NULL) linker_file_unload(lf, LINKER_UNLOAD_FORCE); free(shdr, M_LINKER); free(firstpage, M_LINKER); free(shstrs, M_LINKER); return (error); } Elf_Addr elf_relocaddr(linker_file_t lf, Elf_Addr x) { elf_file_t ef; KASSERT(lf->ops->cls == (kobj_class_t)&link_elf_class, ("elf_relocaddr: unexpected linker file %p", lf)); ef = (elf_file_t)lf; if (x >= ef->pcpu_start && x < ef->pcpu_stop) return ((x - ef->pcpu_start) + ef->pcpu_base); #ifdef VIMAGE if (x >= ef->vnet_start && x < ef->vnet_stop) return ((x - ef->vnet_start) + ef->vnet_base); #endif return (x); } static void link_elf_unload_file(linker_file_t file) { elf_file_t ef = (elf_file_t) file; if (ef->pcpu_base != 0) { dpcpu_free((void *)ef->pcpu_base, ef->pcpu_stop - ef->pcpu_start); elf_set_delete(&set_pcpu_list, ef->pcpu_start); } #ifdef VIMAGE if (ef->vnet_base != 0) { vnet_data_free((void *)ef->vnet_base, ef->vnet_stop - ef->vnet_start); elf_set_delete(&set_vnet_list, ef->vnet_start); } #endif #ifdef GDB if (ef->gdb.l_ld != NULL) { GDB_STATE(RT_DELETE); free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER); link_elf_delete_gdb(&ef->gdb); GDB_STATE(RT_CONSISTENT); } #endif /* Notify MD code that a module is being unloaded. */ elf_cpu_unload_file(file); if (ef->preloaded) { link_elf_unload_preload(file); return; } #ifdef SPARSE_MAPPING if (ef->object != NULL) { vm_map_remove(kernel_map, (vm_offset_t) ef->address, (vm_offset_t) ef->address + (ef->object->size << PAGE_SHIFT)); } #else free(ef->address, M_LINKER); #endif free(ef->symbase, M_LINKER); free(ef->strbase, M_LINKER); free(ef->ctftab, M_LINKER); free(ef->ctfoff, M_LINKER); free(ef->typoff, M_LINKER); } static void link_elf_unload_preload(linker_file_t file) { if (file->pathname != NULL) preload_delete_name(file->pathname); } static const char * symbol_name(elf_file_t ef, Elf_Size r_info) { const Elf_Sym *ref; if (ELF_R_SYM(r_info)) { ref = ef->symtab + ELF_R_SYM(r_info); return (ef->strtab + ref->st_name); } return (NULL); } static int symbol_type(elf_file_t ef, Elf_Size r_info) { const Elf_Sym *ref; if (ELF_R_SYM(r_info)) { ref = ef->symtab + ELF_R_SYM(r_info); return (ELF_ST_TYPE(ref->st_info)); } return (STT_NOTYPE); } static int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, bool ifuncs) { const Elf_Rel *rel; const Elf_Rela *rela; const char *symname; #define APPLY_RELOCS(iter, tbl, tblsize, type) do { \ for ((iter) = (tbl); (iter) != NULL && \ (iter) < (tbl) + (tblsize) / sizeof(*(iter)); (iter)++) { \ if ((symbol_type(ef, (iter)->r_info) == \ STT_GNU_IFUNC || \ elf_is_ifunc_reloc((iter)->r_info)) != ifuncs) \ continue; \ if (reloc(&ef->lf, (Elf_Addr)ef->address, \ (iter), (type), lookup)) { \ symname = symbol_name(ef, (iter)->r_info); \ printf("link_elf: symbol %s undefined\n", \ symname); \ return (ENOENT); \ } \ } \ } while (0) APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL); APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA); APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL); APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA); #undef APPLY_RELOCS return (0); } static int relocate_file(elf_file_t ef) { int error; error = relocate_file1(ef, elf_lookup, elf_reloc, false); if (error == 0) error = relocate_file1(ef, elf_lookup, elf_reloc, true); return (error); } /* * Hash function for symbol table lookup. Don't even think about changing * this. It is specified by the System V ABI. */ static unsigned long elf_hash(const char *name) { const unsigned char *p = (const unsigned char *) name; unsigned long h = 0; unsigned long g; while (*p != '\0') { h = (h << 4) + *p++; if ((g = h & 0xf0000000) != 0) h ^= g >> 24; h &= ~g; } return (h); } static int link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym) { elf_file_t ef = (elf_file_t) lf; unsigned long symnum; const Elf_Sym* symp; const char *strp; unsigned long hash; int i; /* If we don't have a hash, bail. */ if (ef->buckets == NULL || ef->nbuckets == 0) { printf("link_elf_lookup_symbol: missing symbol hash table\n"); return (ENOENT); } /* First, search hashed global symbols */ hash = elf_hash(name); symnum = ef->buckets[hash % ef->nbuckets]; while (symnum != STN_UNDEF) { if (symnum >= ef->nchains) { printf("%s: corrupt symbol table\n", __func__); return (ENOENT); } symp = ef->symtab + symnum; if (symp->st_name == 0) { printf("%s: corrupt symbol table\n", __func__); return (ENOENT); } strp = ef->strtab + symp->st_name; if (strcmp(name, strp) == 0) { if (symp->st_shndx != SHN_UNDEF || (symp->st_value != 0 && (ELF_ST_TYPE(symp->st_info) == STT_FUNC || ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { *sym = (c_linker_sym_t) symp; return (0); } return (ENOENT); } symnum = ef->chains[symnum]; } /* If we have not found it, look at the full table (if loaded) */ if (ef->symtab == ef->ddbsymtab) return (ENOENT); /* Exhaustive search */ for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { strp = ef->ddbstrtab + symp->st_name; if (strcmp(name, strp) == 0) { if (symp->st_shndx != SHN_UNDEF || (symp->st_value != 0 && (ELF_ST_TYPE(symp->st_info) == STT_FUNC || ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { *sym = (c_linker_sym_t) symp; return (0); } return (ENOENT); } } return (ENOENT); } static int link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, linker_symval_t *symval) { elf_file_t ef; const Elf_Sym *es; caddr_t val; ef = (elf_file_t)lf; es = (const Elf_Sym *)sym; if (es >= ef->symtab && es < (ef->symtab + ef->nchains)) { symval->name = ef->strtab + es->st_name; val = (caddr_t)ef->address + es->st_value; if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) val = ((caddr_t (*)(void))val)(); symval->value = val; symval->size = es->st_size; return (0); } if (ef->symtab == ef->ddbsymtab) return (ENOENT); if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { symval->name = ef->ddbstrtab + es->st_name; val = (caddr_t)ef->address + es->st_value; if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) val = ((caddr_t (*)(void))val)(); symval->value = val; symval->size = es->st_size; return (0); } return (ENOENT); } static int link_elf_search_symbol(linker_file_t lf, caddr_t value, c_linker_sym_t *sym, long *diffp) { elf_file_t ef = (elf_file_t) lf; u_long off = (uintptr_t) (void *) value; u_long diff = off; u_long st_value; const Elf_Sym* es; const Elf_Sym* best = NULL; int i; for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) { if (es->st_name == 0) continue; st_value = es->st_value + (uintptr_t) (void *) ef->address; if (off >= st_value) { if (off - st_value < diff) { diff = off - st_value; best = es; if (diff == 0) break; } else if (off - st_value == diff) { best = es; } } } if (best == NULL) *diffp = off; else *diffp = diff; *sym = (c_linker_sym_t) best; return (0); } /* * Look up a linker set on an ELF system. */ static int link_elf_lookup_set(linker_file_t lf, const char *name, void ***startp, void ***stopp, int *countp) { c_linker_sym_t sym; linker_symval_t symval; char *setsym; void **start, **stop; int len, error = 0, count; len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */ setsym = malloc(len, M_LINKER, M_WAITOK); /* get address of first entry */ snprintf(setsym, len, "%s%s", "__start_set_", name); error = link_elf_lookup_symbol(lf, setsym, &sym); if (error != 0) goto out; link_elf_symbol_values(lf, sym, &symval); if (symval.value == 0) { error = ESRCH; goto out; } start = (void **)symval.value; /* get address of last entry */ snprintf(setsym, len, "%s%s", "__stop_set_", name); error = link_elf_lookup_symbol(lf, setsym, &sym); if (error != 0) goto out; link_elf_symbol_values(lf, sym, &symval); if (symval.value == 0) { error = ESRCH; goto out; } stop = (void **)symval.value; /* and the number of entries */ count = stop - start; /* and copy out */ if (startp != NULL) *startp = start; if (stopp != NULL) *stopp = stop; if (countp != NULL) *countp = count; out: free(setsym, M_LINKER); return (error); } static int link_elf_each_function_name(linker_file_t file, int (*callback)(const char *, void *), void *opaque) { elf_file_t ef = (elf_file_t)file; const Elf_Sym *symp; int i, error; /* Exhaustive search */ for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { if (symp->st_value != 0 && (ELF_ST_TYPE(symp->st_info) == STT_FUNC || ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { error = callback(ef->ddbstrtab + symp->st_name, opaque); if (error != 0) return (error); } } return (0); } static int link_elf_each_function_nameval(linker_file_t file, linker_function_nameval_callback_t callback, void *opaque) { linker_symval_t symval; elf_file_t ef = (elf_file_t)file; const Elf_Sym* symp; int i, error; /* Exhaustive search */ for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { if (symp->st_value != 0 && (ELF_ST_TYPE(symp->st_info) == STT_FUNC || ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { error = link_elf_symbol_values(file, (c_linker_sym_t) symp, &symval); if (error != 0) return (error); error = callback(file, i, &symval, opaque); if (error != 0) return (error); } } return (0); } const Elf_Sym * elf_get_sym(linker_file_t lf, Elf_Size symidx) { elf_file_t ef = (elf_file_t)lf; if (symidx >= ef->nchains) return (NULL); return (ef->symtab + symidx); } const char * elf_get_symname(linker_file_t lf, Elf_Size symidx) { elf_file_t ef = (elf_file_t)lf; const Elf_Sym *sym; if (symidx >= ef->nchains) return (NULL); sym = ef->symtab + symidx; return (ef->strtab + sym->st_name); } /* * Symbol lookup function that can be used when the symbol index is known (ie * in relocations). It uses the symbol index instead of doing a fully fledged * hash table based lookup when such is valid. For example for local symbols. * This is not only more efficient, it's also more correct. It's not always * the case that the symbol can be found through the hash table. */ static int elf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res) { elf_file_t ef = (elf_file_t)lf; const Elf_Sym *sym; const char *symbol; Elf_Addr addr, start, base; /* Don't even try to lookup the symbol if the index is bogus. */ if (symidx >= ef->nchains) { *res = 0; return (EINVAL); } sym = ef->symtab + symidx; /* * Don't do a full lookup when the symbol is local. It may even * fail because it may not be found through the hash table. */ if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) { /* Force lookup failure when we have an insanity. */ if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0) { *res = 0; return (EINVAL); } *res = ((Elf_Addr)ef->address + sym->st_value); return (0); } /* * XXX we can avoid doing a hash table based lookup for global * symbols as well. This however is not always valid, so we'll * just do it the hard way for now. Performance tweaks can * always be added. */ symbol = ef->strtab + sym->st_name; /* Force a lookup failure if the symbol name is bogus. */ if (*symbol == 0) { *res = 0; return (EINVAL); } addr = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps)); if (addr == 0 && ELF_ST_BIND(sym->st_info) != STB_WEAK) { *res = 0; return (EINVAL); } if (elf_set_find(&set_pcpu_list, addr, &start, &base)) addr = addr - start + base; #ifdef VIMAGE else if (elf_set_find(&set_vnet_list, addr, &start, &base)) addr = addr - start + base; #endif *res = addr; return (0); } static void link_elf_reloc_local(linker_file_t lf) { const Elf_Rel *rellim; const Elf_Rel *rel; const Elf_Rela *relalim; const Elf_Rela *rela; elf_file_t ef = (elf_file_t)lf; /* Perform relocations without addend if there are any: */ if ((rel = ef->rel) != NULL) { rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize); while (rel < rellim) { elf_reloc_local(lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL, elf_lookup); rel++; } } /* Perform relocations with addend if there are any: */ if ((rela = ef->rela) != NULL) { relalim = (const Elf_Rela *) ((const char *)ef->rela + ef->relasize); while (rela < relalim) { elf_reloc_local(lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA, elf_lookup); rela++; } } } static long link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab) { elf_file_t ef = (elf_file_t)lf; *symtab = ef->ddbsymtab; if (*symtab == NULL) return (0); return (ef->ddbsymcnt); } static long link_elf_strtab_get(linker_file_t lf, caddr_t *strtab) { elf_file_t ef = (elf_file_t)lf; *strtab = ef->ddbstrtab; if (*strtab == NULL) return (0); return (ef->ddbstrcnt); } #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) || defined(__powerpc__) /* * Use this lookup routine when performing relocations early during boot. * The generic lookup routine depends on kobj, which is not initialized * at that point. */ static int elf_lookup_ifunc(linker_file_t lf, Elf_Size symidx, int deps __unused, Elf_Addr *res) { elf_file_t ef; const Elf_Sym *symp; caddr_t val; ef = (elf_file_t)lf; symp = ef->symtab + symidx; if (ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC) { val = (caddr_t)ef->address + symp->st_value; *res = ((Elf_Addr (*)(void))val)(); return (0); } return (ENOENT); } void link_elf_ireloc(caddr_t kmdp) { struct elf_file eff; elf_file_t ef; ef = &eff; bzero_early(ef, sizeof(*ef)); ef->modptr = kmdp; ef->dynamic = (Elf_Dyn *)&_DYNAMIC; #ifdef RELOCATABLE_KERNEL ef->address = (caddr_t) (__startkernel - KERNBASE); #else ef->address = 0; #endif parse_dynamic(ef); link_elf_preload_parse_symbols(ef); relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true); } #if defined(__aarch64__) || defined(__amd64__) void link_elf_late_ireloc(void) { elf_file_t ef; KASSERT(linker_kernel_file != NULL, ("link_elf_late_ireloc: No kernel linker file found")); ef = (elf_file_t)linker_kernel_file; relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true); } #endif #endif Index: head/sys/sys/malloc.h =================================================================== --- head/sys/sys/malloc.h (revision 367185) +++ head/sys/sys/malloc.h (revision 367186) @@ -1,301 +1,306 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1987, 1993 * The Regents of the University of California. * Copyright (c) 2005, 2009 Robert N. M. Watson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)malloc.h 8.5 (Berkeley) 5/3/95 * $FreeBSD$ */ #ifndef _SYS_MALLOC_H_ #define _SYS_MALLOC_H_ #ifndef _STANDALONE #include #ifdef _KERNEL #include #endif #include #include #include #include #define MINALLOCSIZE UMA_SMALLEST_UNIT /* * Flags to memory allocation functions. */ #define M_NOWAIT 0x0001 /* do not block */ #define M_WAITOK 0x0002 /* ok to block */ #define M_ZERO 0x0100 /* bzero the allocation */ #define M_NOVM 0x0200 /* don't ask VM for pages */ #define M_USE_RESERVE 0x0400 /* can alloc out of reserve memory */ #define M_NODUMP 0x0800 /* don't dump pages in this allocation */ #define M_FIRSTFIT 0x1000 /* only for vmem, fast fit */ #define M_BESTFIT 0x2000 /* only for vmem, low fragmentation */ #define M_EXEC 0x4000 /* allocate executable space */ #define M_NEXTFIT 0x8000 /* only for vmem, follow cursor */ #define M_MAGIC 877983977 /* time when first defined :-) */ /* * Two malloc type structures are present: malloc_type, which is used by a * type owner to declare the type, and malloc_type_internal, which holds * malloc-owned statistics and other ABI-sensitive fields, such as the set of * malloc statistics indexed by the compile-time MAXCPU constant. * Applications should avoid introducing dependence on the allocator private * data layout and size. * * The malloc_type ks_next field is protected by malloc_mtx. Other fields in * malloc_type are static after initialization so unsynchronized. * * Statistics in malloc_type_stats are written only when holding a critical * section and running on the CPU associated with the index into the stat * array, but read lock-free resulting in possible (minor) races, which the * monitoring app should take into account. */ struct malloc_type_stats { uint64_t mts_memalloced; /* Bytes allocated on CPU. */ uint64_t mts_memfreed; /* Bytes freed on CPU. */ uint64_t mts_numallocs; /* Number of allocates on CPU. */ uint64_t mts_numfrees; /* number of frees on CPU. */ uint64_t mts_size; /* Bitmask of sizes allocated on CPU. */ uint64_t _mts_reserved1; /* Reserved field. */ uint64_t _mts_reserved2; /* Reserved field. */ uint64_t _mts_reserved3; /* Reserved field. */ }; /* * Index definitions for the mti_probes[] array. */ #define DTMALLOC_PROBE_MALLOC 0 #define DTMALLOC_PROBE_FREE 1 #define DTMALLOC_PROBE_MAX 2 struct malloc_type_internal { uint32_t mti_probes[DTMALLOC_PROBE_MAX]; /* DTrace probe ID array. */ u_char mti_zone; struct malloc_type_stats *mti_stats; }; /* * Public data structure describing a malloc type. Private data is hung off * of ks_handle to avoid encoding internal malloc(9) data structures in * modules, which will statically allocate struct malloc_type. */ struct malloc_type { struct malloc_type *ks_next; /* Next in global chain. */ u_long ks_magic; /* Detect programmer error. */ const char *ks_shortdesc; /* Printable type name. */ void *ks_handle; /* Priv. data, was lo_class. */ }; /* * Statistics structure headers for user space. The kern.malloc sysctl * exposes a structure stream consisting of a stream header, then a series of * malloc type headers and statistics structures (quantity maxcpus). For * convenience, the kernel will provide the current value of maxcpus at the * head of the stream. */ #define MALLOC_TYPE_STREAM_VERSION 0x00000001 struct malloc_type_stream_header { uint32_t mtsh_version; /* Stream format version. */ uint32_t mtsh_maxcpus; /* Value of MAXCPU for stream. */ uint32_t mtsh_count; /* Number of records. */ uint32_t _mtsh_pad; /* Pad/reserved field. */ }; #define MALLOC_MAX_NAME 32 struct malloc_type_header { char mth_name[MALLOC_MAX_NAME]; }; #ifdef _KERNEL #define MALLOC_DEFINE(type, shortdesc, longdesc) \ struct malloc_type type[1] = { \ { NULL, M_MAGIC, shortdesc, NULL } \ }; \ SYSINIT(type##_init, SI_SUB_KMEM, SI_ORDER_THIRD, malloc_init, \ type); \ SYSUNINIT(type##_uninit, SI_SUB_KMEM, SI_ORDER_ANY, \ malloc_uninit, type) #define MALLOC_DECLARE(type) \ extern struct malloc_type type[1] MALLOC_DECLARE(M_CACHE); MALLOC_DECLARE(M_DEVBUF); MALLOC_DECLARE(M_TEMP); /* * XXX this should be declared in , but that tends to fail * because is included in a header before the source file * has a chance to include to get MALLOC_DECLARE() defined. */ MALLOC_DECLARE(M_IOV); struct domainset; extern struct mtx malloc_mtx; /* * Function type used when iterating over the list of malloc types. */ typedef void malloc_type_list_func_t(struct malloc_type *, void *); void contigfree(void *addr, unsigned long size, struct malloc_type *type); void *contigmalloc(unsigned long size, struct malloc_type *type, int flags, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary) __malloc_like __result_use_check __alloc_size(1) __alloc_align(6); void *contigmalloc_domainset(unsigned long size, struct malloc_type *type, struct domainset *ds, int flags, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary) __malloc_like __result_use_check __alloc_size(1) __alloc_align(7); void free(void *addr, struct malloc_type *type); void zfree(void *addr, struct malloc_type *type); void *malloc(size_t size, struct malloc_type *type, int flags) __malloc_like __result_use_check __alloc_size(1); /* * Try to optimize malloc(..., ..., M_ZERO) allocations by doing zeroing in * place if the size is known at compilation time. * * Passing the flag down requires malloc to blindly zero the entire object. * In practice a lot of the zeroing can be avoided if most of the object * gets explicitly initialized after the allocation. Letting the compiler * zero in place gives it the opportunity to take advantage of this state. * * Note that the operation is only applicable if both flags and size are * known at compilation time. If M_ZERO is passed but M_WAITOK is not, the * allocation can fail and a NULL check is needed. However, if M_WAITOK is * passed we know the allocation must succeed and the check can be elided. * * _malloc_item = malloc(_size, type, (flags) &~ M_ZERO); * if (((flags) & M_WAITOK) != 0 || _malloc_item != NULL) * bzero(_malloc_item, _size); * * If the flag is set, the compiler knows the left side is always true, * therefore the entire statement is true and the callsite is: * * _malloc_item = malloc(_size, type, (flags) &~ M_ZERO); * bzero(_malloc_item, _size); * * If the flag is not set, the compiler knows the left size is always false * and the NULL check is needed, therefore the callsite is: * * _malloc_item = malloc(_size, type, (flags) &~ M_ZERO); * if (_malloc_item != NULL) * bzero(_malloc_item, _size); * * The implementation is a macro because of what appears to be a clang 6 bug: * an inline function variant ended up being compiled to a mere malloc call * regardless of argument. gcc generates expected code (like the above). */ #define malloc(size, type, flags) ({ \ void *_malloc_item; \ size_t _size = (size); \ if (__builtin_constant_p(size) && __builtin_constant_p(flags) &&\ ((flags) & M_ZERO) != 0) { \ _malloc_item = malloc(_size, type, (flags) &~ M_ZERO); \ if (((flags) & M_WAITOK) != 0 || \ __predict_true(_malloc_item != NULL)) \ bzero(_malloc_item, _size); \ } else { \ _malloc_item = malloc(_size, type, flags); \ } \ _malloc_item; \ }) void *malloc_domainset(size_t size, struct malloc_type *type, struct domainset *ds, int flags) __malloc_like __result_use_check __alloc_size(1); void *mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags) __malloc_like __result_use_check __alloc_size2(1, 2); +void *malloc_exec(size_t size, struct malloc_type *type, int flags) __malloc_like + __result_use_check __alloc_size(1); +void *malloc_domainset_exec(size_t size, struct malloc_type *type, + struct domainset *ds, int flags) __malloc_like __result_use_check + __alloc_size(1); void malloc_init(void *); int malloc_last_fail(void); void malloc_type_allocated(struct malloc_type *type, unsigned long size); void malloc_type_freed(struct malloc_type *type, unsigned long size); void malloc_type_list(malloc_type_list_func_t *, void *); void malloc_uninit(void *); size_t malloc_usable_size(const void *); void *realloc(void *addr, size_t size, struct malloc_type *type, int flags) __result_use_check __alloc_size(2); void *reallocf(void *addr, size_t size, struct malloc_type *type, int flags) __result_use_check __alloc_size(2); struct malloc_type *malloc_desc2type(const char *desc); /* * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW */ #define MUL_NO_OVERFLOW (1UL << (sizeof(size_t) * 8 / 2)) static inline bool WOULD_OVERFLOW(size_t nmemb, size_t size) { return ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) && nmemb > 0 && __SIZE_T_MAX / nmemb < size); } #undef MUL_NO_OVERFLOW #endif /* _KERNEL */ #else /* * The native stand malloc / free interface we're mapping to */ extern void Free(void *p, const char *file, int line); extern void *Malloc(size_t bytes, const char *file, int line); /* * Minimal standalone malloc implementation / environment. None of the * flags mean anything and there's no need declare malloc types. * Define the simple alloc / free routines in terms of Malloc and * Free. None of the kernel features that this stuff disables are needed. * * XXX we are setting ourselves up for a potential crash if we can't allocate * memory for a M_WAITOK call. */ #define M_WAITOK 0 #define M_ZERO 0 #define M_NOWAIT 0 #define MALLOC_DECLARE(x) #define kmem_zalloc(size, flags) Malloc((size), __FILE__, __LINE__) #define kmem_free(p, size) Free(p, __FILE__, __LINE__) /* * ZFS mem.h define that's the OpenZFS porting layer way of saying * M_WAITOK. Given the above, it will also be a nop. */ #define KM_SLEEP M_WAITOK #endif /* _STANDALONE */ #endif /* !_SYS_MALLOC_H_ */