diff --git a/stand/common/bootstrap.h b/stand/common/bootstrap.h index b7e1f8553f47..eb4e50203133 100644 --- a/stand/common/bootstrap.h +++ b/stand/common/bootstrap.h @@ -1,404 +1,407 @@ /*- * Copyright (c) 1998 Michael Smith * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _BOOTSTRAP_H_ #define _BOOTSTRAP_H_ #include #include #include #include #include "readin.h" /* Commands and return values; nonzero return sets command_errmsg != NULL */ typedef int (bootblk_cmd_t)(int argc, char *argv[]); #define COMMAND_ERRBUFSZ (256) extern const char *command_errmsg; extern char command_errbuf[COMMAND_ERRBUFSZ]; #define CMD_OK 0 #define CMD_WARN 1 #define CMD_ERROR 2 #define CMD_CRIT 3 #define CMD_FATAL 4 /* interp.c */ void interact(void); void interp_emit_prompt(void); int interp_builtin_cmd(int argc, char *argv[]); /* Called by interp.c for interp_*.c embedded interpreters */ int interp_include(const char *); /* Execute commands from filename */ void interp_init(void); /* Initialize interpreater */ int interp_run(const char *); /* Run a single command */ /* interp_backslash.c */ char *backslash(const char *str); /* interp_parse.c */ int parse(int *argc, char ***argv, const char *str); /* boot.c */ void autoboot_maybe(void); int getrootmount(char *rootdev); /* misc.c */ char *unargv(int argc, char *argv[]); size_t strlenout(vm_offset_t str); char *strdupout(vm_offset_t str); void kern_bzero(vm_offset_t dest, size_t len); int kern_pread(readin_handle_t fd, vm_offset_t dest, size_t len, off_t off); void *alloc_pread(readin_handle_t fd, off_t off, size_t len); /* bcache.c */ void bcache_init(size_t nblks, size_t bsize); void bcache_add_dev(int); void *bcache_allocate(void); void bcache_free(void *); int bcache_strategy(void *devdata, int rw, daddr_t blk, size_t size, char *buf, size_t *rsize); /* * Disk block cache */ struct bcache_devdata { int (*dv_strategy)(void *, int, daddr_t, size_t, char *, size_t *); void *dv_devdata; void *dv_cache; }; /* * Modular console support. */ struct console { const char *c_name; const char *c_desc; int c_flags; #define C_PRESENTIN (1<<0) /* console can provide input */ #define C_PRESENTOUT (1<<1) /* console can provide output */ #define C_ACTIVEIN (1<<2) /* user wants input from console */ #define C_ACTIVEOUT (1<<3) /* user wants output to console */ #define C_WIDEOUT (1<<4) /* c_out routine groks wide chars */ /* set c_flags to match hardware */ void (* c_probe)(struct console *cp); /* reinit XXX may need more args */ int (* c_init)(int arg); /* emit c */ void (* c_out)(int c); /* wait for and return input */ int (* c_in)(void); /* return nonzero if input waiting */ int (* c_ready)(void); }; extern struct console *consoles[]; void cons_probe(void); bool cons_update_mode(bool); void autoload_font(bool); /* * Plug-and-play enumerator/configurator interface. */ struct pnphandler { const char *pp_name; /* handler/bus name */ void (*pp_enumerate)(void); /* enumerate PnP devices, add to chain */ }; struct pnpident { /* ASCII identifier, actual format varies with bus/handler */ char *id_ident; STAILQ_ENTRY(pnpident) id_link; }; struct pnpinfo { char *pi_desc; /* ASCII description, optional */ int pi_revision; /* optional revision (or -1) if not supported */ char *pi_module; /* module/args nominated to handle device */ int pi_argc; /* module arguments */ char **pi_argv; struct pnphandler *pi_handler; /* handler which detected this device */ STAILQ_HEAD(, pnpident) pi_ident; /* list of identifiers */ STAILQ_ENTRY(pnpinfo) pi_link; }; STAILQ_HEAD(pnpinfo_stql, pnpinfo); extern struct pnphandler *pnphandlers[]; /* provided by MD code */ void pnp_addident(struct pnpinfo *pi, char *ident); struct pnpinfo *pnp_allocinfo(void); void pnp_freeinfo(struct pnpinfo *pi); void pnp_addinfo(struct pnpinfo *pi); char *pnp_eisaformat(uint8_t *data); /* * < 0 - No ISA in system * == 0 - Maybe ISA, search for read data port * > 0 - ISA in system, value is read data port address */ extern int isapnp_readport; /* * Version information */ extern char bootprog_info[]; /* * Interpreter information */ extern const char bootprog_interp[]; #define INTERP_DEFINE(interpstr) \ const char bootprog_interp[] = "$Interpreter:" interpstr /* * Preloaded file metadata header. * * Metadata are allocated on our heap, and copied into kernel space * before executing the kernel. */ struct file_metadata { size_t md_size; uint16_t md_type; struct file_metadata *md_next; char md_data[1]; /* data are immediately appended */ }; struct preloaded_file; struct mod_depend; struct kernel_module { char *m_name; /* module name */ int m_version; /* module version */ /* char *m_args; */ /* arguments for the module */ struct preloaded_file *m_fp; struct kernel_module *m_next; }; /* * Preloaded file information. Depending on type, file can contain * additional units called 'modules'. * * At least one file (the kernel) must be loaded in order to boot. * The kernel is always loaded first. * * String fields (m_name, m_type) should be dynamically allocated. */ struct preloaded_file { char *f_name; /* file name */ char *f_type; /* verbose file type, eg 'ELF kernel', 'pnptable', etc. */ char *f_args; /* arguments for the file */ /* metadata that will be placed in the module directory */ struct file_metadata *f_metadata; int f_loader; /* index of the loader that read the file */ vm_offset_t f_addr; /* load address */ size_t f_size; /* file size */ struct kernel_module *f_modules; /* list of modules if any */ struct preloaded_file *f_next; /* next file */ +#ifdef __amd64__ + bool f_kernphys_relocatable; +#endif }; struct file_format { /* * Load function must return EFTYPE if it can't handle * the module supplied */ int (*l_load)(char *, uint64_t, struct preloaded_file **); /* * Only a loader that will load a kernel (first module) * should have an exec handler */ int (*l_exec)(struct preloaded_file *); }; extern struct file_format *file_formats[]; /* supplied by consumer */ extern struct preloaded_file *preloaded_files; int mod_load(char *name, struct mod_depend *verinfo, int argc, char *argv[]); int mod_loadkld(const char *name, int argc, char *argv[]); void unload(void); struct preloaded_file *file_alloc(void); struct preloaded_file *file_findfile(const char *name, const char *type); struct file_metadata *file_findmetadata(struct preloaded_file *fp, int type); struct preloaded_file *file_loadraw(const char *name, char *type, int insert); void file_discard(struct preloaded_file *fp); void file_addmetadata(struct preloaded_file *, int, size_t, void *); int file_addmodule(struct preloaded_file *, char *, int, struct kernel_module **); void file_removemetadata(struct preloaded_file *fp); int file_addbuf(const char *name, const char *type, size_t len, void *buf); int tslog_init(void); int tslog_publish(void); vm_offset_t build_font_module(vm_offset_t); /* MI module loaders */ #ifdef __elfN /* Relocation types. */ #define ELF_RELOC_REL 1 #define ELF_RELOC_RELA 2 /* Relocation offset for some architectures */ extern uint64_t __elfN(relocation_offset); struct elf_file; typedef Elf_Addr (symaddr_fn)(struct elf_file *ef, Elf_Size symidx); int __elfN(loadfile)(char *, uint64_t, struct preloaded_file **); int __elfN(obj_loadfile)(char *, uint64_t, struct preloaded_file **); int __elfN(reloc)(struct elf_file *ef, symaddr_fn *symaddr, const void *reldata, int reltype, Elf_Addr relbase, Elf_Addr dataaddr, void *data, size_t len); int __elfN(loadfile_raw)(char *, uint64_t, struct preloaded_file **, int); int __elfN(load_modmetadata)(struct preloaded_file *, uint64_t); #endif /* * Support for commands */ struct bootblk_command { const char *c_name; const char *c_desc; bootblk_cmd_t *c_fn; }; #define COMMAND_SET(tag, key, desc, func) \ static bootblk_cmd_t func; \ static struct bootblk_command _cmd_ ## tag = { key, desc, func }; \ DATA_SET(Xcommand_set, _cmd_ ## tag) SET_DECLARE(Xcommand_set, struct bootblk_command); /* * The intention of the architecture switch is to provide a convenient * encapsulation of the interface between the bootstrap MI and MD code. * MD code may selectively populate the switch at runtime based on the * actual configuration of the target system. */ struct arch_switch { /* Automatically load modules as required by detected hardware */ int (*arch_autoload)(void); /* Locate the device for (name), return pointer to tail in (*path) */ int (*arch_getdev)(void **dev, const char *name, const char **path); /* * Copy from local address space to module address space, * similar to bcopy() */ ssize_t (*arch_copyin)(const void *, vm_offset_t, const size_t); /* * Copy to local address space from module address space, * similar to bcopy() */ ssize_t (*arch_copyout)(const vm_offset_t, void *, const size_t); /* Read from file to module address space, same semantics as read() */ ssize_t (*arch_readin)(readin_handle_t, vm_offset_t, const size_t); /* Perform ISA byte port I/O (only for systems with ISA) */ int (*arch_isainb)(int port); void (*arch_isaoutb)(int port, int value); /* * Interface to adjust the load address according to the "object" * being loaded. */ uint64_t (*arch_loadaddr)(u_int type, void *data, uint64_t addr); #define LOAD_ELF 1 /* data points to the ELF header. */ #define LOAD_RAW 2 /* data points to the file name. */ /* * Interface to inform MD code about a loaded (ELF) segment. This * can be used to flush caches and/or set up translations. */ #ifdef __elfN void (*arch_loadseg)(Elf_Ehdr *eh, Elf_Phdr *ph, uint64_t delta); #else void (*arch_loadseg)(void *eh, void *ph, uint64_t delta); #endif /* Probe ZFS pool(s), if needed. */ void (*arch_zfs_probe)(void); /* Return the hypervisor name/type or NULL if not virtualized. */ const char *(*arch_hypervisor)(void); /* For kexec-type loaders, get ksegment structure */ void (*arch_kexec_kseg_get)(int *nseg, void **kseg); }; extern struct arch_switch archsw; /* This must be provided by the MD code, but should it be in the archsw? */ void delay(int delay); void dev_cleanup(void); /* * nvstore API. */ typedef int (nvstore_getter_cb_t)(void *, const char *, void **); typedef int (nvstore_setter_cb_t)(void *, int, const char *, const void *, size_t); typedef int (nvstore_setter_str_cb_t)(void *, const char *, const char *, const char *); typedef int (nvstore_unset_cb_t)(void *, const char *); typedef int (nvstore_print_cb_t)(void *, void *); typedef int (nvstore_iterate_cb_t)(void *, int (*)(void *, void *)); typedef struct nvs_callbacks { nvstore_getter_cb_t *nvs_getter; nvstore_setter_cb_t *nvs_setter; nvstore_setter_str_cb_t *nvs_setter_str; nvstore_unset_cb_t *nvs_unset; nvstore_print_cb_t *nvs_print; nvstore_iterate_cb_t *nvs_iterate; } nvs_callbacks_t; int nvstore_init(const char *, nvs_callbacks_t *, void *); int nvstore_fini(const char *); void *nvstore_get_store(const char *); int nvstore_print(void *); int nvstore_get_var(void *, const char *, void **); int nvstore_set_var(void *, int, const char *, void *, size_t); int nvstore_set_var_from_string(void *, const char *, const char *, const char *); int nvstore_unset_var(void *, const char *); #ifndef CTASSERT #define CTASSERT(x) _Static_assert(x, "compile-time assertion failed") #endif #endif /* !_BOOTSTRAP_H_ */ diff --git a/stand/common/load_elf.c b/stand/common/load_elf.c index f1a9ff8e0c22..9ae91036dbb4 100644 --- a/stand/common/load_elf.c +++ b/stand/common/load_elf.c @@ -1,1322 +1,1337 @@ /*- * Copyright (c) 1998 Michael Smith * Copyright (c) 1998 Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #define FREEBSD_ELF #include #include #include "bootstrap.h" #define COPYOUT(s,d,l) archsw.arch_copyout((vm_offset_t)(s), d, l) #if defined(__i386__) && __ELF_WORD_SIZE == 64 #undef ELF_TARG_CLASS #undef ELF_TARG_MACH #define ELF_TARG_CLASS ELFCLASS64 #define ELF_TARG_MACH EM_X86_64 #endif typedef struct elf_file { Elf_Phdr *ph; Elf_Ehdr *ehdr; Elf_Sym *symtab; Elf_Hashelt *hashtab; Elf_Hashelt nbuckets; Elf_Hashelt nchains; Elf_Hashelt *buckets; Elf_Hashelt *chains; Elf_Rel *rel; size_t relsz; Elf_Rela *rela; size_t relasz; char *strtab; size_t strsz; int fd; caddr_t firstpage; size_t firstlen; int kernel; uint64_t off; #ifdef LOADER_VERIEXEC_VECTX struct vectx *vctx; #endif } *elf_file_t; #ifdef LOADER_VERIEXEC_VECTX #define VECTX_HANDLE(ef) (ef)->vctx #else #define VECTX_HANDLE(ef) (ef)->fd #endif static int __elfN(loadimage)(struct preloaded_file *mp, elf_file_t ef, uint64_t loadaddr); static int __elfN(lookup_symbol)(elf_file_t ef, const char* name, Elf_Sym *sym, unsigned char type); static int __elfN(reloc_ptr)(struct preloaded_file *mp, elf_file_t ef, Elf_Addr p, void *val, size_t len); static int __elfN(parse_modmetadata)(struct preloaded_file *mp, elf_file_t ef, Elf_Addr p_start, Elf_Addr p_end); static bool __elfN(parse_vt_drv_set)(struct preloaded_file *mp, elf_file_t ef, Elf_Addr p_start, Elf_Addr p_end); static symaddr_fn __elfN(symaddr); static char *fake_modname(const char *name); const char *__elfN(kerneltype) = "elf kernel"; const char *__elfN(moduletype) = "elf module"; uint64_t __elfN(relocation_offset) = 0; extern void elf_wrong_field_size(void); #define CONVERT_FIELD(b, f, e) \ switch (sizeof((b)->f)) { \ case 2: \ (b)->f = e ## 16toh((b)->f); \ break; \ case 4: \ (b)->f = e ## 32toh((b)->f); \ break; \ case 8: \ (b)->f = e ## 64toh((b)->f); \ break; \ default: \ /* Force a link time error. */ \ elf_wrong_field_size(); \ break; \ } #define CONVERT_SWITCH(h, d, f) \ switch ((h)->e_ident[EI_DATA]) { \ case ELFDATA2MSB: \ f(d, be); \ break; \ case ELFDATA2LSB: \ f(d, le); \ break; \ default: \ return (EINVAL); \ } static int elf_header_convert(Elf_Ehdr *ehdr) { /* * Fixup ELF header endianness. * * The Xhdr structure was loaded using block read call to optimize file * accesses. It might happen, that the endianness of the system memory * is different that endianness of the ELF header. Swap fields here to * guarantee that Xhdr always contain valid data regardless of * architecture. */ #define HEADER_FIELDS(b, e) \ CONVERT_FIELD(b, e_type, e); \ CONVERT_FIELD(b, e_machine, e); \ CONVERT_FIELD(b, e_version, e); \ CONVERT_FIELD(b, e_entry, e); \ CONVERT_FIELD(b, e_phoff, e); \ CONVERT_FIELD(b, e_shoff, e); \ CONVERT_FIELD(b, e_flags, e); \ CONVERT_FIELD(b, e_ehsize, e); \ CONVERT_FIELD(b, e_phentsize, e); \ CONVERT_FIELD(b, e_phnum, e); \ CONVERT_FIELD(b, e_shentsize, e); \ CONVERT_FIELD(b, e_shnum, e); \ CONVERT_FIELD(b, e_shstrndx, e) CONVERT_SWITCH(ehdr, ehdr, HEADER_FIELDS); #undef HEADER_FIELDS return (0); } static int elf_program_header_convert(const Elf_Ehdr *ehdr, Elf_Phdr *phdr) { #define PROGRAM_HEADER_FIELDS(b, e) \ CONVERT_FIELD(b, p_type, e); \ CONVERT_FIELD(b, p_flags, e); \ CONVERT_FIELD(b, p_offset, e); \ CONVERT_FIELD(b, p_vaddr, e); \ CONVERT_FIELD(b, p_paddr, e); \ CONVERT_FIELD(b, p_filesz, e); \ CONVERT_FIELD(b, p_memsz, e); \ CONVERT_FIELD(b, p_align, e) CONVERT_SWITCH(ehdr, phdr, PROGRAM_HEADER_FIELDS); #undef PROGRAM_HEADER_FIELDS return (0); } static int elf_section_header_convert(const Elf_Ehdr *ehdr, Elf_Shdr *shdr) { #define SECTION_HEADER_FIELDS(b, e) \ CONVERT_FIELD(b, sh_name, e); \ CONVERT_FIELD(b, sh_type, e); \ CONVERT_FIELD(b, sh_link, e); \ CONVERT_FIELD(b, sh_info, e); \ CONVERT_FIELD(b, sh_flags, e); \ CONVERT_FIELD(b, sh_addr, e); \ CONVERT_FIELD(b, sh_offset, e); \ CONVERT_FIELD(b, sh_size, e); \ CONVERT_FIELD(b, sh_addralign, e); \ CONVERT_FIELD(b, sh_entsize, e) CONVERT_SWITCH(ehdr, shdr, SECTION_HEADER_FIELDS); #undef SECTION_HEADER_FIELDS return (0); } #undef CONVERT_SWITCH #undef CONVERT_FIELD + +#ifdef __amd64__ +static bool +is_kernphys_relocatable(elf_file_t ef) +{ + Elf_Sym sym; + + return (__elfN(lookup_symbol)(ef, "kernphys", &sym, STT_OBJECT) == 0 && + sym.st_size == 8); +} +#endif + static int __elfN(load_elf_header)(char *filename, elf_file_t ef) { ssize_t bytes_read; Elf_Ehdr *ehdr; int err; /* * Open the image, read and validate the ELF header */ if (filename == NULL) /* can't handle nameless */ return (EFTYPE); if ((ef->fd = open(filename, O_RDONLY)) == -1) return (errno); ef->firstpage = malloc(PAGE_SIZE); if (ef->firstpage == NULL) { close(ef->fd); return (ENOMEM); } #ifdef LOADER_VERIEXEC_VECTX { int verror; ef->vctx = vectx_open(ef->fd, filename, 0L, NULL, &verror, __func__); if (verror) { printf("Unverified %s: %s\n", filename, ve_error_get()); close(ef->fd); free(ef->vctx); return (EAUTH); } } #endif bytes_read = VECTX_READ(VECTX_HANDLE(ef), ef->firstpage, PAGE_SIZE); ef->firstlen = (size_t)bytes_read; if (bytes_read < 0 || ef->firstlen <= sizeof(Elf_Ehdr)) { err = EFTYPE; /* could be EIO, but may be small file */ goto error; } ehdr = ef->ehdr = (Elf_Ehdr *)ef->firstpage; /* Is it ELF? */ if (!IS_ELF(*ehdr)) { err = EFTYPE; goto error; } if (ehdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || /* Layout ? */ ehdr->e_ident[EI_DATA] != ELF_TARG_DATA || ehdr->e_ident[EI_VERSION] != EV_CURRENT) /* Version ? */ { err = EFTYPE; goto error; } err = elf_header_convert(ehdr); if (err) goto error; if (ehdr->e_version != EV_CURRENT || ehdr->e_machine != ELF_TARG_MACH) { /* Machine ? */ err = EFTYPE; goto error; } #if defined(LOADER_VERIEXEC) && !defined(LOADER_VERIEXEC_VECTX) if (verify_file(ef->fd, filename, bytes_read, VE_MUST, __func__) < 0) { err = EAUTH; goto error; } #endif return (0); error: if (ef->firstpage != NULL) { free(ef->firstpage); ef->firstpage = NULL; } if (ef->fd != -1) { #ifdef LOADER_VERIEXEC_VECTX free(ef->vctx); #endif close(ef->fd); ef->fd = -1; } return (err); } /* * Attempt to load the file (file) as an ELF module. It will be stored at * (dest), and a pointer to a module structure describing the loaded object * will be saved in (result). */ int __elfN(loadfile)(char *filename, uint64_t dest, struct preloaded_file **result) { return (__elfN(loadfile_raw)(filename, dest, result, 0)); } int __elfN(loadfile_raw)(char *filename, uint64_t dest, struct preloaded_file **result, int multiboot) { struct preloaded_file *fp, *kfp; struct elf_file ef; Elf_Ehdr *ehdr; int err; fp = NULL; bzero(&ef, sizeof(struct elf_file)); ef.fd = -1; err = __elfN(load_elf_header)(filename, &ef); if (err != 0) return (err); ehdr = ef.ehdr; /* * Check to see what sort of module we are. */ kfp = file_findfile(NULL, __elfN(kerneltype)); #ifdef __powerpc__ /* * Kernels can be ET_DYN, so just assume the first loaded object is the * kernel. This assumption will be checked later. */ if (kfp == NULL) ef.kernel = 1; #endif if (ef.kernel || ehdr->e_type == ET_EXEC) { /* Looks like a kernel */ if (kfp != NULL) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: kernel already loaded\n"); err = EPERM; goto oerr; } /* * Calculate destination address based on kernel entrypoint. * * For ARM, the destination address is independent of any values * in the elf header (an ARM kernel can be loaded at any 2MB * boundary), so we leave dest set to the value calculated by * archsw.arch_loadaddr() and passed in to this function. */ #ifndef __arm__ if (ehdr->e_type == ET_EXEC) dest = (ehdr->e_entry & ~PAGE_MASK); #endif if ((ehdr->e_entry & ~PAGE_MASK) == 0) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: not a kernel (maybe static binary?)\n"); err = EPERM; goto oerr; } ef.kernel = 1; } else if (ehdr->e_type == ET_DYN) { /* Looks like a kld module */ if (multiboot != 0) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: can't load module as multiboot\n"); err = EPERM; goto oerr; } if (kfp == NULL) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: can't load module before kernel\n"); err = EPERM; goto oerr; } if (strcmp(__elfN(kerneltype), kfp->f_type)) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: can't load module with kernel type '%s'\n", kfp->f_type); err = EPERM; goto oerr; } /* Looks OK, got ahead */ ef.kernel = 0; } else { err = EFTYPE; goto oerr; } if (archsw.arch_loadaddr != NULL) dest = archsw.arch_loadaddr(LOAD_ELF, ehdr, dest); else dest = roundup(dest, PAGE_SIZE); /* * Ok, we think we should handle this. */ fp = file_alloc(); if (fp == NULL) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadfile: cannot allocate module info\n"); err = EPERM; goto out; } if (ef.kernel == 1 && multiboot == 0) setenv("kernelname", filename, 1); fp->f_name = strdup(filename); if (multiboot == 0) fp->f_type = strdup(ef.kernel ? __elfN(kerneltype) : __elfN(moduletype)); else fp->f_type = strdup("elf multiboot kernel"); #ifdef ELF_VERBOSE if (ef.kernel) printf("%s entry at 0x%jx\n", filename, (uintmax_t)ehdr->e_entry); #else printf("%s ", filename); #endif fp->f_size = __elfN(loadimage)(fp, &ef, dest); if (fp->f_size == 0 || fp->f_addr == 0) goto ioerr; /* save exec header as metadata */ file_addmetadata(fp, MODINFOMD_ELFHDR, sizeof(*ehdr), ehdr); /* Load OK, return module pointer */ *result = (struct preloaded_file *)fp; err = 0; +#ifdef __amd64__ + fp->f_kernphys_relocatable = is_kernphys_relocatable(&ef); +#endif goto out; ioerr: err = EIO; oerr: file_discard(fp); out: if (ef.firstpage) free(ef.firstpage); if (ef.fd != -1) { #ifdef LOADER_VERIEXEC_VECTX if (!err && ef.vctx) { int verror; verror = vectx_close(ef.vctx, VE_MUST, __func__); if (verror) { err = EAUTH; file_discard(fp); } } #endif close(ef.fd); } return (err); } /* * With the file (fd) open on the image, and (ehdr) containing * the Elf header, load the image at (off) */ static int __elfN(loadimage)(struct preloaded_file *fp, elf_file_t ef, uint64_t off) { int i; u_int j; Elf_Ehdr *ehdr; Elf_Phdr *phdr, *php; Elf_Shdr *shdr; char *shstr; int ret; vm_offset_t firstaddr; vm_offset_t lastaddr; size_t chunk; ssize_t result; Elf_Addr ssym, esym; Elf_Dyn *dp; Elf_Addr adp; Elf_Addr ctors; int ndp; int symstrindex; int symtabindex; Elf_Size size; u_int fpcopy; Elf_Sym sym; Elf_Addr p_start, p_end; dp = NULL; shdr = NULL; ret = 0; firstaddr = lastaddr = 0; ehdr = ef->ehdr; #ifdef __powerpc__ if (ef->kernel) { #else if (ehdr->e_type == ET_EXEC) { #endif #if defined(__i386__) || defined(__amd64__) #if __ELF_WORD_SIZE == 64 /* x86_64 relocates after locore */ off = - (off & 0xffffffffff000000ull); #else /* i386 relocates after locore */ off = - (off & 0xff000000u); #endif #elif defined(__powerpc__) /* * On the purely virtual memory machines like e500, the kernel * is linked against its final VA range, which is most often * not available at the loader stage, but only after kernel * initializes and completes its VM settings. In such cases we * cannot use p_vaddr field directly to load ELF segments, but * put them at some 'load-time' locations. */ if (off & 0xf0000000u) { off = -(off & 0xf0000000u); /* * XXX the physical load address should not be * hardcoded. Note that the Book-E kernel assumes that * it's loaded at a 16MB boundary for now... */ off += 0x01000000; } ehdr->e_entry += off; #ifdef ELF_VERBOSE printf("Converted entry 0x%jx\n", (uintmax_t)ehdr->e_entry); #endif #elif defined(__arm__) && !defined(EFI) /* * The elf headers in arm kernels specify virtual addresses in * all header fields, even the ones that should be physical * addresses. We assume the entry point is in the first page, * and masking the page offset will leave us with the virtual * address the kernel was linked at. We subtract that from the * load offset, making 'off' into the value which, when added * to a virtual address in an elf header, translates it to a * physical address. We do the va->pa conversion on the entry * point address in the header now, so that later we can launch * the kernel by just jumping to that address. * * When booting from UEFI the copyin and copyout functions * handle adjusting the location relative to the first virtual * address. Because of this there is no need to adjust the * offset or entry point address as these will both be handled * by the efi code. */ off -= ehdr->e_entry & ~PAGE_MASK; ehdr->e_entry += off; #ifdef ELF_VERBOSE printf("ehdr->e_entry 0x%jx, va<->pa off %llx\n", (uintmax_t)ehdr->e_entry, off); #endif #else off = 0; /* other archs use direct mapped kernels */ #endif } ef->off = off; if (ef->kernel) __elfN(relocation_offset) = off; if ((ehdr->e_phoff + ehdr->e_phnum * sizeof(*phdr)) > ef->firstlen) { printf("elf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: program header not within first page\n"); goto out; } phdr = (Elf_Phdr *)(ef->firstpage + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { if (elf_program_header_convert(ehdr, phdr)) continue; /* We want to load PT_LOAD segments only.. */ if (phdr[i].p_type != PT_LOAD) continue; #ifdef ELF_VERBOSE printf("Segment: 0x%lx@0x%lx -> 0x%lx-0x%lx", (long)phdr[i].p_filesz, (long)phdr[i].p_offset, (long)(phdr[i].p_vaddr + off), (long)(phdr[i].p_vaddr + off + phdr[i].p_memsz - 1)); #else if ((phdr[i].p_flags & PF_W) == 0) { printf("text=0x%lx ", (long)phdr[i].p_filesz); } else { printf("data=0x%lx", (long)phdr[i].p_filesz); if (phdr[i].p_filesz < phdr[i].p_memsz) printf("+0x%lx", (long)(phdr[i].p_memsz - phdr[i].p_filesz)); printf(" "); } #endif fpcopy = 0; if (ef->firstlen > phdr[i].p_offset) { fpcopy = ef->firstlen - phdr[i].p_offset; archsw.arch_copyin(ef->firstpage + phdr[i].p_offset, phdr[i].p_vaddr + off, fpcopy); } if (phdr[i].p_filesz > fpcopy) { if (kern_pread(VECTX_HANDLE(ef), phdr[i].p_vaddr + off + fpcopy, phdr[i].p_filesz - fpcopy, phdr[i].p_offset + fpcopy) != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: read failed\n"); goto out; } } /* clear space from oversized segments; eg: bss */ if (phdr[i].p_filesz < phdr[i].p_memsz) { #ifdef ELF_VERBOSE printf(" (bss: 0x%lx-0x%lx)", (long)(phdr[i].p_vaddr + off + phdr[i].p_filesz), (long)(phdr[i].p_vaddr + off + phdr[i].p_memsz -1)); #endif kern_bzero(phdr[i].p_vaddr + off + phdr[i].p_filesz, phdr[i].p_memsz - phdr[i].p_filesz); } #ifdef ELF_VERBOSE printf("\n"); #endif if (archsw.arch_loadseg != NULL) archsw.arch_loadseg(ehdr, phdr + i, off); if (firstaddr == 0 || firstaddr > (phdr[i].p_vaddr + off)) firstaddr = phdr[i].p_vaddr + off; if (lastaddr == 0 || lastaddr < (phdr[i].p_vaddr + off + phdr[i].p_memsz)) lastaddr = phdr[i].p_vaddr + off + phdr[i].p_memsz; } lastaddr = roundup(lastaddr, sizeof(long)); /* * Get the section headers. We need this for finding the .ctors * section as well as for loading any symbols. Both may be hard * to do if reading from a .gz file as it involves seeking. I * think the rule is going to have to be that you must strip a * file to remove symbols before gzipping it. */ chunk = (size_t)ehdr->e_shnum * (size_t)ehdr->e_shentsize; if (chunk == 0 || ehdr->e_shoff == 0) goto nosyms; shdr = alloc_pread(VECTX_HANDLE(ef), ehdr->e_shoff, chunk); if (shdr == NULL) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: failed to read section headers"); goto nosyms; } for (i = 0; i < ehdr->e_shnum; i++) elf_section_header_convert(ehdr, &shdr[i]); file_addmetadata(fp, MODINFOMD_SHDR, chunk, shdr); /* * Read the section string table and look for the .ctors section. * We need to tell the kernel where it is so that it can call the * ctors. */ chunk = shdr[ehdr->e_shstrndx].sh_size; if (chunk) { shstr = alloc_pread(VECTX_HANDLE(ef), shdr[ehdr->e_shstrndx].sh_offset, chunk); if (shstr) { for (i = 0; i < ehdr->e_shnum; i++) { if (strcmp(shstr + shdr[i].sh_name, ".ctors") != 0) continue; ctors = shdr[i].sh_addr; file_addmetadata(fp, MODINFOMD_CTORS_ADDR, sizeof(ctors), &ctors); size = shdr[i].sh_size; file_addmetadata(fp, MODINFOMD_CTORS_SIZE, sizeof(size), &size); break; } free(shstr); } } /* * Now load any symbols. */ symtabindex = -1; symstrindex = -1; for (i = 0; i < ehdr->e_shnum; i++) { if (shdr[i].sh_type != SHT_SYMTAB) continue; for (j = 0; j < ehdr->e_phnum; j++) { if (phdr[j].p_type != PT_LOAD) continue; if (shdr[i].sh_offset >= phdr[j].p_offset && (shdr[i].sh_offset + shdr[i].sh_size <= phdr[j].p_offset + phdr[j].p_filesz)) { shdr[i].sh_offset = 0; shdr[i].sh_size = 0; break; } } if (shdr[i].sh_offset == 0 || shdr[i].sh_size == 0) continue; /* alread loaded in a PT_LOAD above */ /* Save it for loading below */ symtabindex = i; symstrindex = shdr[i].sh_link; } if (symtabindex < 0 || symstrindex < 0) goto nosyms; /* Ok, committed to a load. */ #ifndef ELF_VERBOSE printf("syms=["); #endif ssym = lastaddr; for (i = symtabindex; i >= 0; i = symstrindex) { #ifdef ELF_VERBOSE char *secname; switch(shdr[i].sh_type) { case SHT_SYMTAB: /* Symbol table */ secname = "symtab"; break; case SHT_STRTAB: /* String table */ secname = "strtab"; break; default: secname = "WHOA!!"; break; } #endif size = shdr[i].sh_size; #if defined(__powerpc__) #if __ELF_WORD_SIZE == 64 size = htobe64(size); #else size = htobe32(size); #endif #endif archsw.arch_copyin(&size, lastaddr, sizeof(size)); lastaddr += sizeof(size); #ifdef ELF_VERBOSE printf("\n%s: 0x%jx@0x%jx -> 0x%jx-0x%jx", secname, (uintmax_t)shdr[i].sh_size, (uintmax_t)shdr[i].sh_offset, (uintmax_t)lastaddr, (uintmax_t)(lastaddr + shdr[i].sh_size)); #else if (i == symstrindex) printf("+"); printf("0x%lx+0x%lx", (long)sizeof(size), (long)size); #endif if (VECTX_LSEEK(VECTX_HANDLE(ef), (off_t)shdr[i].sh_offset, SEEK_SET) == -1) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: could not seek for symbols - skipped!"); lastaddr = ssym; ssym = 0; goto nosyms; } result = archsw.arch_readin(VECTX_HANDLE(ef), lastaddr, shdr[i].sh_size); if (result < 0 || (size_t)result != shdr[i].sh_size) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "_loadimage: could not read symbols - skipped! " "(%ju != %ju)", (uintmax_t)result, (uintmax_t)shdr[i].sh_size); lastaddr = ssym; ssym = 0; goto nosyms; } /* Reset offsets relative to ssym */ lastaddr += shdr[i].sh_size; lastaddr = roundup(lastaddr, sizeof(size)); if (i == symtabindex) symtabindex = -1; else if (i == symstrindex) symstrindex = -1; } esym = lastaddr; #ifndef ELF_VERBOSE printf("]"); #endif #if defined(__powerpc__) /* On PowerPC we always need to provide BE data to the kernel */ #if __ELF_WORD_SIZE == 64 ssym = htobe64((uint64_t)ssym); esym = htobe64((uint64_t)esym); #else ssym = htobe32((uint32_t)ssym); esym = htobe32((uint32_t)esym); #endif #endif file_addmetadata(fp, MODINFOMD_SSYM, sizeof(ssym), &ssym); file_addmetadata(fp, MODINFOMD_ESYM, sizeof(esym), &esym); nosyms: printf("\n"); ret = lastaddr - firstaddr; fp->f_addr = firstaddr; php = NULL; for (i = 0; i < ehdr->e_phnum; i++) { if (phdr[i].p_type == PT_DYNAMIC) { php = phdr + i; adp = php->p_vaddr; file_addmetadata(fp, MODINFOMD_DYNAMIC, sizeof(adp), &adp); break; } } if (php == NULL) /* this is bad, we cannot get to symbols or _DYNAMIC */ goto out; ndp = php->p_filesz / sizeof(Elf_Dyn); if (ndp == 0) goto out; dp = malloc(php->p_filesz); if (dp == NULL) goto out; archsw.arch_copyout(php->p_vaddr + off, dp, php->p_filesz); ef->strsz = 0; for (i = 0; i < ndp; i++) { if (dp[i].d_tag == 0) break; switch (dp[i].d_tag) { case DT_HASH: ef->hashtab = (Elf_Hashelt*)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_STRTAB: ef->strtab = (char *)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_STRSZ: ef->strsz = dp[i].d_un.d_val; break; case DT_SYMTAB: ef->symtab = (Elf_Sym *)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_REL: ef->rel = (Elf_Rel *)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_RELSZ: ef->relsz = dp[i].d_un.d_val; break; case DT_RELA: ef->rela = (Elf_Rela *)(uintptr_t)(dp[i].d_un.d_ptr + off); break; case DT_RELASZ: ef->relasz = dp[i].d_un.d_val; break; default: break; } } if (ef->hashtab == NULL || ef->symtab == NULL || ef->strtab == NULL || ef->strsz == 0) goto out; COPYOUT(ef->hashtab, &ef->nbuckets, sizeof(ef->nbuckets)); COPYOUT(ef->hashtab + 1, &ef->nchains, sizeof(ef->nchains)); ef->buckets = ef->hashtab + 2; ef->chains = ef->buckets + ef->nbuckets; if (!gfx_state.tg_kernel_supported && __elfN(lookup_symbol)(ef, "__start_set_vt_drv_set", &sym, STT_NOTYPE) == 0) { p_start = sym.st_value + ef->off; if (__elfN(lookup_symbol)(ef, "__stop_set_vt_drv_set", &sym, STT_NOTYPE) == 0) { p_end = sym.st_value + ef->off; gfx_state.tg_kernel_supported = __elfN(parse_vt_drv_set)(fp, ef, p_start, p_end); } } if (__elfN(lookup_symbol)(ef, "__start_set_modmetadata_set", &sym, STT_NOTYPE) != 0) return 0; p_start = sym.st_value + ef->off; if (__elfN(lookup_symbol)(ef, "__stop_set_modmetadata_set", &sym, STT_NOTYPE) != 0) return 0; p_end = sym.st_value + ef->off; if (__elfN(parse_modmetadata)(fp, ef, p_start, p_end) == 0) goto out; if (ef->kernel) /* kernel must not depend on anything */ goto out; out: if (dp) free(dp); if (shdr) free(shdr); return ret; } static char invalid_name[] = "bad"; char * fake_modname(const char *name) { const char *sp, *ep; char *fp; size_t len; sp = strrchr(name, '/'); if (sp) sp++; else sp = name; ep = strrchr(sp, '.'); if (ep == NULL) { ep = sp + strlen(sp); } if (ep == sp) { sp = invalid_name; ep = invalid_name + sizeof(invalid_name) - 1; } len = ep - sp; fp = malloc(len + 1); if (fp == NULL) return NULL; memcpy(fp, sp, len); fp[len] = '\0'; return fp; } #if (defined(__i386__) || defined(__powerpc__)) && __ELF_WORD_SIZE == 64 struct mod_metadata64 { int md_version; /* structure version MDTV_* */ int md_type; /* type of entry MDT_* */ uint64_t md_data; /* specific data */ uint64_t md_cval; /* common string label */ }; #endif #if defined(__amd64__) && __ELF_WORD_SIZE == 32 struct mod_metadata32 { int md_version; /* structure version MDTV_* */ int md_type; /* type of entry MDT_* */ uint32_t md_data; /* specific data */ uint32_t md_cval; /* common string label */ }; #endif int __elfN(load_modmetadata)(struct preloaded_file *fp, uint64_t dest) { struct elf_file ef; int err, i, j; Elf_Shdr *sh_meta, *shdr = NULL; Elf_Shdr *sh_data[2]; char *shstrtab = NULL; size_t size; Elf_Addr p_start, p_end; bzero(&ef, sizeof(struct elf_file)); ef.fd = -1; err = __elfN(load_elf_header)(fp->f_name, &ef); if (err != 0) goto out; if (ef.kernel == 1 || ef.ehdr->e_type == ET_EXEC) { ef.kernel = 1; } else if (ef.ehdr->e_type != ET_DYN) { err = EFTYPE; goto out; } size = (size_t)ef.ehdr->e_shnum * (size_t)ef.ehdr->e_shentsize; shdr = alloc_pread(VECTX_HANDLE(&ef), ef.ehdr->e_shoff, size); if (shdr == NULL) { err = ENOMEM; goto out; } /* Load shstrtab. */ shstrtab = alloc_pread(VECTX_HANDLE(&ef), shdr[ef.ehdr->e_shstrndx].sh_offset, shdr[ef.ehdr->e_shstrndx].sh_size); if (shstrtab == NULL) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load shstrtab\n"); err = EFTYPE; goto out; } /* Find set_modmetadata_set and data sections. */ sh_data[0] = sh_data[1] = sh_meta = NULL; for (i = 0, j = 0; i < ef.ehdr->e_shnum; i++) { if (strcmp(&shstrtab[shdr[i].sh_name], "set_modmetadata_set") == 0) { sh_meta = &shdr[i]; } if ((strcmp(&shstrtab[shdr[i].sh_name], ".data") == 0) || (strcmp(&shstrtab[shdr[i].sh_name], ".rodata") == 0)) { sh_data[j++] = &shdr[i]; } } if (sh_meta == NULL || sh_data[0] == NULL || sh_data[1] == NULL) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to find set_modmetadata_set or data sections\n"); err = EFTYPE; goto out; } /* Load set_modmetadata_set into memory */ err = kern_pread(VECTX_HANDLE(&ef), dest, sh_meta->sh_size, sh_meta->sh_offset); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load set_modmetadata_set: %d\n", err); goto out; } p_start = dest; p_end = dest + sh_meta->sh_size; dest += sh_meta->sh_size; /* Load data sections into memory. */ err = kern_pread(VECTX_HANDLE(&ef), dest, sh_data[0]->sh_size, sh_data[0]->sh_offset); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load data: %d\n", err); goto out; } /* * We have to increment the dest, so that the offset is the same into * both the .rodata and .data sections. */ ef.off = -(sh_data[0]->sh_addr - dest); dest += (sh_data[1]->sh_addr - sh_data[0]->sh_addr); err = kern_pread(VECTX_HANDLE(&ef), dest, sh_data[1]->sh_size, sh_data[1]->sh_offset); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to load data: %d\n", err); goto out; } err = __elfN(parse_modmetadata)(fp, &ef, p_start, p_end); if (err != 0) { printf("\nelf" __XSTRING(__ELF_WORD_SIZE) "load_modmetadata: unable to parse metadata: %d\n", err); goto out; } out: if (shstrtab != NULL) free(shstrtab); if (shdr != NULL) free(shdr); if (ef.firstpage != NULL) free(ef.firstpage); if (ef.fd != -1) { #ifdef LOADER_VERIEXEC_VECTX if (!err && ef.vctx) { int verror; verror = vectx_close(ef.vctx, VE_MUST, __func__); if (verror) { err = EAUTH; file_discard(fp); } } #endif close(ef.fd); } return (err); } /* * Walk through vt_drv_set, each vt driver structure starts with * static 16 chars for driver name. If we have "vbefb", return true. */ static bool __elfN(parse_vt_drv_set)(struct preloaded_file *fp, elf_file_t ef, Elf_Addr p_start, Elf_Addr p_end) { Elf_Addr v, p; char vd_name[16]; int error; p = p_start; while (p < p_end) { COPYOUT(p, &v, sizeof(v)); error = __elfN(reloc_ptr)(fp, ef, p, &v, sizeof(v)); if (error == EOPNOTSUPP) v += ef->off; else if (error != 0) return (false); COPYOUT(v, &vd_name, sizeof(vd_name)); if (strncmp(vd_name, "vbefb", sizeof(vd_name)) == 0) return (true); p += sizeof(Elf_Addr); } return (false); } int __elfN(parse_modmetadata)(struct preloaded_file *fp, elf_file_t ef, Elf_Addr p_start, Elf_Addr p_end) { struct mod_metadata md; #if (defined(__i386__) || defined(__powerpc__)) && __ELF_WORD_SIZE == 64 struct mod_metadata64 md64; #elif defined(__amd64__) && __ELF_WORD_SIZE == 32 struct mod_metadata32 md32; #endif struct mod_depend *mdepend; struct mod_version mver; char *s; int error, modcnt, minfolen; Elf_Addr v, p; modcnt = 0; p = p_start; while (p < p_end) { COPYOUT(p, &v, sizeof(v)); error = __elfN(reloc_ptr)(fp, ef, p, &v, sizeof(v)); if (error == EOPNOTSUPP) v += ef->off; else if (error != 0) return (error); #if (defined(__i386__) || defined(__powerpc__)) && __ELF_WORD_SIZE == 64 COPYOUT(v, &md64, sizeof(md64)); error = __elfN(reloc_ptr)(fp, ef, v, &md64, sizeof(md64)); if (error == EOPNOTSUPP) { md64.md_cval += ef->off; md64.md_data += ef->off; } else if (error != 0) return (error); md.md_version = md64.md_version; md.md_type = md64.md_type; md.md_cval = (const char *)(uintptr_t)md64.md_cval; md.md_data = (void *)(uintptr_t)md64.md_data; #elif defined(__amd64__) && __ELF_WORD_SIZE == 32 COPYOUT(v, &md32, sizeof(md32)); error = __elfN(reloc_ptr)(fp, ef, v, &md32, sizeof(md32)); if (error == EOPNOTSUPP) { md32.md_cval += ef->off; md32.md_data += ef->off; } else if (error != 0) return (error); md.md_version = md32.md_version; md.md_type = md32.md_type; md.md_cval = (const char *)(uintptr_t)md32.md_cval; md.md_data = (void *)(uintptr_t)md32.md_data; #else COPYOUT(v, &md, sizeof(md)); error = __elfN(reloc_ptr)(fp, ef, v, &md, sizeof(md)); if (error == EOPNOTSUPP) { md.md_cval += ef->off; md.md_data = (void *)((uintptr_t)md.md_data + (uintptr_t)ef->off); } else if (error != 0) return (error); #endif p += sizeof(Elf_Addr); switch(md.md_type) { case MDT_DEPEND: if (ef->kernel) /* kernel must not depend on anything */ break; s = strdupout((vm_offset_t)md.md_cval); minfolen = sizeof(*mdepend) + strlen(s) + 1; mdepend = malloc(minfolen); if (mdepend == NULL) return ENOMEM; COPYOUT((vm_offset_t)md.md_data, mdepend, sizeof(*mdepend)); strcpy((char*)(mdepend + 1), s); free(s); file_addmetadata(fp, MODINFOMD_DEPLIST, minfolen, mdepend); free(mdepend); break; case MDT_VERSION: s = strdupout((vm_offset_t)md.md_cval); COPYOUT((vm_offset_t)md.md_data, &mver, sizeof(mver)); file_addmodule(fp, s, mver.mv_version, NULL); free(s); modcnt++; break; } } if (modcnt == 0) { s = fake_modname(fp->f_name); file_addmodule(fp, s, 1, NULL); free(s); } return 0; } static unsigned long elf_hash(const char *name) { const unsigned char *p = (const unsigned char *) name; unsigned long h = 0; unsigned long g; while (*p != '\0') { h = (h << 4) + *p++; if ((g = h & 0xf0000000) != 0) h ^= g >> 24; h &= ~g; } return h; } static const char __elfN(bad_symtable)[] = "elf" __XSTRING(__ELF_WORD_SIZE) "_lookup_symbol: corrupt symbol table\n"; int __elfN(lookup_symbol)(elf_file_t ef, const char* name, Elf_Sym *symp, unsigned char type) { Elf_Hashelt symnum; Elf_Sym sym; char *strp; unsigned long hash; hash = elf_hash(name); COPYOUT(&ef->buckets[hash % ef->nbuckets], &symnum, sizeof(symnum)); while (symnum != STN_UNDEF) { if (symnum >= ef->nchains) { printf(__elfN(bad_symtable)); return ENOENT; } COPYOUT(ef->symtab + symnum, &sym, sizeof(sym)); if (sym.st_name == 0) { printf(__elfN(bad_symtable)); return ENOENT; } strp = strdupout((vm_offset_t)(ef->strtab + sym.st_name)); if (strcmp(name, strp) == 0) { free(strp); if (sym.st_shndx != SHN_UNDEF || (sym.st_value != 0 && ELF_ST_TYPE(sym.st_info) == type)) { *symp = sym; return 0; } return ENOENT; } free(strp); COPYOUT(&ef->chains[symnum], &symnum, sizeof(symnum)); } return ENOENT; } /* * Apply any intra-module relocations to the value. p is the load address * of the value and val/len is the value to be modified. This does NOT modify * the image in-place, because this is done by kern_linker later on. * * Returns EOPNOTSUPP if no relocation method is supplied. */ static int __elfN(reloc_ptr)(struct preloaded_file *mp, elf_file_t ef, Elf_Addr p, void *val, size_t len) { size_t n; Elf_Rela a; Elf_Rel r; int error; /* * The kernel is already relocated, but we still want to apply * offset adjustments. */ if (ef->kernel) return (EOPNOTSUPP); for (n = 0; n < ef->relsz / sizeof(r); n++) { COPYOUT(ef->rel + n, &r, sizeof(r)); error = __elfN(reloc)(ef, __elfN(symaddr), &r, ELF_RELOC_REL, ef->off, p, val, len); if (error != 0) return (error); } for (n = 0; n < ef->relasz / sizeof(a); n++) { COPYOUT(ef->rela + n, &a, sizeof(a)); error = __elfN(reloc)(ef, __elfN(symaddr), &a, ELF_RELOC_RELA, ef->off, p, val, len); if (error != 0) return (error); } return (0); } static Elf_Addr __elfN(symaddr)(struct elf_file *ef, Elf_Size symidx) { /* Symbol lookup by index not required here. */ return (0); } diff --git a/stand/efi/loader/arch/amd64/elf64_freebsd.c b/stand/efi/loader/arch/amd64/elf64_freebsd.c index a950ca55e843..d0c8ef96eeea 100644 --- a/stand/efi/loader/arch/amd64/elf64_freebsd.c +++ b/stand/efi/loader/arch/amd64/elf64_freebsd.c @@ -1,214 +1,300 @@ /*- * Copyright (c) 1998 Michael Smith * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #define __ELF_WORD_SIZE 64 #include #include #include #include #include #include #include #include #include #include #include "bootstrap.h" #include "platform/acfreebsd.h" #include "acconfig.h" #define ACPI_SYSTEM_XFACE #include "actypes.h" #include "actbl.h" #include "loader_efi.h" static EFI_GUID acpi_guid = ACPI_TABLE_GUID; static EFI_GUID acpi20_guid = ACPI_20_TABLE_GUID; extern int bi_load(char *args, vm_offset_t *modulep, vm_offset_t *kernendp, bool exit_bs); static int elf64_exec(struct preloaded_file *amp); static int elf64_obj_exec(struct preloaded_file *amp); static struct file_format amd64_elf = { .l_load = elf64_loadfile, .l_exec = elf64_exec, }; static struct file_format amd64_elf_obj = { .l_load = elf64_obj_loadfile, .l_exec = elf64_obj_exec, }; extern struct file_format multiboot2; extern struct file_format multiboot2_obj; struct file_format *file_formats[] = { &multiboot2, &multiboot2_obj, &amd64_elf, &amd64_elf_obj, NULL }; static pml4_entry_t *PT4; static pdp_entry_t *PT3; +static pdp_entry_t *PT3_l, *PT3_u; static pd_entry_t *PT2; +static pd_entry_t *PT2_l0, *PT2_l1, *PT2_l2, *PT2_l3, *PT2_u0, *PT2_u1; + +extern EFI_PHYSICAL_ADDRESS staging; static void (*trampoline)(uint64_t stack, void *copy_finish, uint64_t kernend, uint64_t modulep, pml4_entry_t *pagetable, uint64_t entry); extern uintptr_t amd64_tramp; extern uint32_t amd64_tramp_size; /* * There is an ELF kernel and one or more ELF modules loaded. * We wish to start executing the kernel image, so make such * preparations as are required, and do so. */ static int elf64_exec(struct preloaded_file *fp) { struct file_metadata *md; Elf_Ehdr *ehdr; vm_offset_t modulep, kernend, trampcode, trampstack; int err, i; ACPI_TABLE_RSDP *rsdp; char buf[24]; int revision; + bool copy_auto; + + copy_auto = copy_staging == COPY_STAGING_AUTO; + if (copy_auto) + copy_staging = fp->f_kernphys_relocatable ? + COPY_STAGING_DISABLE : COPY_STAGING_ENABLE; /* * Report the RSDP to the kernel. While this can be found with * a BIOS boot, the RSDP may be elsewhere when booted from UEFI. * The old code used the 'hints' method to communite this to * the kernel. However, while convenient, the 'hints' method * is fragile and does not work when static hints are compiled * into the kernel. Instead, move to setting different tunables * that start with acpi. The old 'hints' can be removed before * we branch for FreeBSD 12. */ rsdp = efi_get_table(&acpi20_guid); if (rsdp == NULL) { rsdp = efi_get_table(&acpi_guid); } if (rsdp != NULL) { sprintf(buf, "0x%016llx", (unsigned long long)rsdp); setenv("hint.acpi.0.rsdp", buf, 1); setenv("acpi.rsdp", buf, 1); revision = rsdp->Revision; if (revision == 0) revision = 1; sprintf(buf, "%d", revision); setenv("hint.acpi.0.revision", buf, 1); setenv("acpi.revision", buf, 1); strncpy(buf, rsdp->OemId, sizeof(rsdp->OemId)); buf[sizeof(rsdp->OemId)] = '\0'; setenv("hint.acpi.0.oem", buf, 1); setenv("acpi.oem", buf, 1); sprintf(buf, "0x%016x", rsdp->RsdtPhysicalAddress); setenv("hint.acpi.0.rsdt", buf, 1); setenv("acpi.rsdt", buf, 1); if (revision >= 2) { /* XXX extended checksum? */ sprintf(buf, "0x%016llx", (unsigned long long)rsdp->XsdtPhysicalAddress); setenv("hint.acpi.0.xsdt", buf, 1); setenv("acpi.xsdt", buf, 1); sprintf(buf, "%d", rsdp->Length); setenv("hint.acpi.0.xsdt_length", buf, 1); setenv("acpi.xsdt_length", buf, 1); } } if ((md = file_findmetadata(fp, MODINFOMD_ELFHDR)) == NULL) - return(EFTYPE); + return (EFTYPE); ehdr = (Elf_Ehdr *)&(md->md_data); - trampcode = (vm_offset_t)0x0000000040000000; + trampcode = copy_staging == COPY_STAGING_ENABLE ? + (vm_offset_t)0x0000000040000000 /* 1G */ : + (vm_offset_t)0x0000000100000000; /* 4G */; err = BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, 1, (EFI_PHYSICAL_ADDRESS *)&trampcode); + if (EFI_ERROR(err)) { + printf("Unable to allocate trampoline\n"); + if (copy_auto) + copy_staging = COPY_STAGING_AUTO; + return (ENOMEM); + } bzero((void *)trampcode, EFI_PAGE_SIZE); trampstack = trampcode + EFI_PAGE_SIZE - 8; bcopy((void *)&amd64_tramp, (void *)trampcode, amd64_tramp_size); trampoline = (void *)trampcode; - PT4 = (pml4_entry_t *)0x0000000040000000; - err = BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, 3, - (EFI_PHYSICAL_ADDRESS *)&PT4); - bzero(PT4, 3 * EFI_PAGE_SIZE); + if (copy_staging == COPY_STAGING_ENABLE) { + PT4 = (pml4_entry_t *)0x0000000040000000; + err = BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, 3, + (EFI_PHYSICAL_ADDRESS *)&PT4); + if (EFI_ERROR(err)) { + printf("Unable to allocate trampoline page table\n"); + BS->FreePages(trampcode, 1); + if (copy_auto) + copy_staging = COPY_STAGING_AUTO; + return (ENOMEM); + } + bzero(PT4, 3 * EFI_PAGE_SIZE); + PT3 = &PT4[512]; + PT2 = &PT3[512]; + + /* + * This is kinda brutal, but every single 1GB VM + * memory segment points to the same first 1GB of + * physical memory. But it is more than adequate. + */ + for (i = 0; i < NPTEPG; i++) { + /* + * Each slot of the L4 pages points to the + * same L3 page. + */ + PT4[i] = (pml4_entry_t)PT3; + PT4[i] |= PG_V | PG_RW; + + /* + * Each slot of the L3 pages points to the + * same L2 page. + */ + PT3[i] = (pdp_entry_t)PT2; + PT3[i] |= PG_V | PG_RW; + + /* + * The L2 page slots are mapped with 2MB pages for 1GB. + */ + PT2[i] = (pd_entry_t)i * (2 * 1024 * 1024); + PT2[i] |= PG_V | PG_RW | PG_PS; + } + } else { + PT4 = (pml4_entry_t *)0x0000000100000000; /* 4G */ + err = BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, 9, + (EFI_PHYSICAL_ADDRESS *)&PT4); + if (EFI_ERROR(err)) { + printf("Unable to allocate trampoline page table\n"); + BS->FreePages(trampcode, 9); + if (copy_auto) + copy_staging = COPY_STAGING_AUTO; + return (ENOMEM); + } - PT3 = &PT4[512]; - PT2 = &PT3[512]; + bzero(PT4, 9 * EFI_PAGE_SIZE); + + PT3_l = &PT4[NPML4EPG * 1]; + PT3_u = &PT4[NPML4EPG * 2]; + PT2_l0 = &PT4[NPML4EPG * 3]; + PT2_l1 = &PT4[NPML4EPG * 4]; + PT2_l2 = &PT4[NPML4EPG * 5]; + PT2_l3 = &PT4[NPML4EPG * 6]; + PT2_u0 = &PT4[NPML4EPG * 7]; + PT2_u1 = &PT4[NPML4EPG * 8]; + + /* 1:1 mapping of lower 4G */ + PT4[0] = (pml4_entry_t)PT3_l | PG_V | PG_RW; + PT3_l[0] = (pdp_entry_t)PT2_l0 | PG_V | PG_RW; + PT3_l[1] = (pdp_entry_t)PT2_l1 | PG_V | PG_RW; + PT3_l[2] = (pdp_entry_t)PT2_l2 | PG_V | PG_RW; + PT3_l[3] = (pdp_entry_t)PT2_l3 | PG_V | PG_RW; + for (i = 0; i < 4 * NPDEPG; i++) { + PT2_l0[i] = ((pd_entry_t)i << PDRSHIFT) | PG_V | + PG_RW | PG_PS; + } - /* - * This is kinda brutal, but every single 1GB VM memory segment points - * to the same first 1GB of physical memory. But it is more than - * adequate. - */ - for (i = 0; i < 512; i++) { - /* Each slot of the L4 pages points to the same L3 page. */ - PT4[i] = (pml4_entry_t)PT3; - PT4[i] |= PG_V | PG_RW; - - /* Each slot of the L3 pages points to the same L2 page. */ - PT3[i] = (pdp_entry_t)PT2; - PT3[i] |= PG_V | PG_RW; - - /* The L2 page slots are mapped with 2MB pages for 1GB. */ - PT2[i] = i * (2 * 1024 * 1024); - PT2[i] |= PG_V | PG_RW | PG_PS; + /* mapping of kernel 2G below top */ + PT4[NPML4EPG - 1] = (pml4_entry_t)PT3_u | PG_V | PG_RW; + PT3_u[NPDPEPG - 2] = (pdp_entry_t)PT2_u0 | PG_V | PG_RW; + PT3_u[NPDPEPG - 1] = (pdp_entry_t)PT2_u1 | PG_V | PG_RW; + /* compat mapping of phys @0 */ + PT2_u0[0] = PG_PS | PG_V | PG_RW; + /* this maps past staging area */ + for (i = 1; i < 2 * NPDEPG; i++) { + PT2_u0[i] = ((pd_entry_t)staging + + ((pd_entry_t)i - 1) * NBPDR) | + PG_V | PG_RW | PG_PS; + } } + printf("staging %#lx (%scoping) tramp %p PT4 %p\n", + staging, copy_staging == COPY_STAGING_ENABLE ? "" : "not ", + trampoline, PT4); printf("Start @ 0x%lx ...\n", ehdr->e_entry); efi_time_fini(); err = bi_load(fp->f_args, &modulep, &kernend, true); if (err != 0) { efi_time_init(); - return(err); + if (copy_auto) + copy_staging = COPY_STAGING_AUTO; + return (err); } dev_cleanup(); - trampoline(trampstack, efi_copy_finish, kernend, modulep, PT4, - ehdr->e_entry); + trampoline(trampstack, copy_staging == COPY_STAGING_ENABLE ? + efi_copy_finish : efi_copy_finish_nop, kernend, modulep, + PT4, ehdr->e_entry); panic("exec returned"); } static int elf64_obj_exec(struct preloaded_file *fp) { return (EFTYPE); } diff --git a/stand/efi/loader/bootinfo.c b/stand/efi/loader/bootinfo.c index 9924901d29e6..f4501f18f14c 100644 --- a/stand/efi/loader/bootinfo.c +++ b/stand/efi/loader/bootinfo.c @@ -1,572 +1,576 @@ /*- * Copyright (c) 1998 Michael Smith * Copyright (c) 2004, 2006 Marcel Moolenaar * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "bootstrap.h" #include "loader_efi.h" #if defined(__amd64__) #include #endif #include "gfx_fb.h" #if defined(LOADER_FDT_SUPPORT) #include #endif #ifdef LOADER_GELI_SUPPORT #include "geliboot.h" #endif int bi_load(char *args, vm_offset_t *modulep, vm_offset_t *kernendp, bool exit_bs); extern EFI_SYSTEM_TABLE *ST; +int boot_services_gone; + static int bi_getboothowto(char *kargs) { const char *sw, *tmp; char *opts; char *console; int howto, speed, port; char buf[50]; howto = boot_parse_cmdline(kargs); howto |= boot_env_to_howto(); console = getenv("console"); if (console != NULL) { if (strcmp(console, "comconsole") == 0) howto |= RB_SERIAL; if (strcmp(console, "nullconsole") == 0) howto |= RB_MUTE; #if defined(__i386__) || defined(__amd64__) if (strcmp(console, "efi") == 0 && getenv("efi_8250_uid") != NULL && getenv("hw.uart.console") == NULL) { /* * If we found a 8250 com port and com speed, we need to * tell the kernel where the serial port is, and how * fast. Ideally, we'd get the port from ACPI, but that * isn't running in the loader. Do the next best thing * by allowing it to be set by a loader.conf variable, * either a EFI specific one, or the compatible * comconsole_port if not. PCI support is needed, but * for that we'd ideally refactor the * libi386/comconsole.c code to have identical behavior. * We only try to set the port for cases where we saw * the Serial(x) node when parsing, otherwise * specialized hardware that has Uart nodes will have a * bogus address set. * But if someone specifically setup hw.uart.console, * don't override that. */ speed = -1; port = -1; tmp = getenv("efi_com_speed"); if (tmp != NULL) speed = strtol(tmp, NULL, 0); tmp = getenv("efi_com_port"); if (tmp == NULL) tmp = getenv("comconsole_port"); if (tmp != NULL) port = strtol(tmp, NULL, 0); if (speed != -1 && port != -1) { snprintf(buf, sizeof(buf), "io:%d,br:%d", port, speed); env_setenv("hw.uart.console", EV_VOLATILE, buf, NULL, NULL); } } #endif } return (howto); } /* * Copy the environment into the load area starting at (addr). * Each variable is formatted as =, with a single nul * separating each variable, and a double nul terminating the environment. */ static vm_offset_t bi_copyenv(vm_offset_t start) { struct env_var *ep; vm_offset_t addr, last; size_t len; addr = last = start; /* Traverse the environment. */ for (ep = environ; ep != NULL; ep = ep->ev_next) { len = strlen(ep->ev_name); if ((size_t)archsw.arch_copyin(ep->ev_name, addr, len) != len) break; addr += len; if (archsw.arch_copyin("=", addr, 1) != 1) break; addr++; if (ep->ev_value != NULL) { len = strlen(ep->ev_value); if ((size_t)archsw.arch_copyin(ep->ev_value, addr, len) != len) break; addr += len; } if (archsw.arch_copyin("", addr, 1) != 1) break; last = ++addr; } if (archsw.arch_copyin("", last++, 1) != 1) last = start; return(last); } /* * Copy module-related data into the load area, where it can be * used as a directory for loaded modules. * * Module data is presented in a self-describing format. Each datum * is preceded by a 32-bit identifier and a 32-bit size field. * * Currently, the following data are saved: * * MOD_NAME (variable) module name (string) * MOD_TYPE (variable) module type (string) * MOD_ARGS (variable) module parameters (string) * MOD_ADDR sizeof(vm_offset_t) module load address * MOD_SIZE sizeof(size_t) module size * MOD_METADATA (variable) type-specific metadata */ #define COPY32(v, a, c) { \ uint32_t x = (v); \ if (c) \ archsw.arch_copyin(&x, a, sizeof(x)); \ a += sizeof(x); \ } #define MOD_STR(t, a, s, c) { \ COPY32(t, a, c); \ COPY32(strlen(s) + 1, a, c); \ if (c) \ archsw.arch_copyin(s, a, strlen(s) + 1); \ a += roundup(strlen(s) + 1, sizeof(u_long)); \ } #define MOD_NAME(a, s, c) MOD_STR(MODINFO_NAME, a, s, c) #define MOD_TYPE(a, s, c) MOD_STR(MODINFO_TYPE, a, s, c) #define MOD_ARGS(a, s, c) MOD_STR(MODINFO_ARGS, a, s, c) #define MOD_VAR(t, a, s, c) { \ COPY32(t, a, c); \ COPY32(sizeof(s), a, c); \ if (c) \ archsw.arch_copyin(&s, a, sizeof(s)); \ a += roundup(sizeof(s), sizeof(u_long)); \ } #define MOD_ADDR(a, s, c) MOD_VAR(MODINFO_ADDR, a, s, c) #define MOD_SIZE(a, s, c) MOD_VAR(MODINFO_SIZE, a, s, c) #define MOD_METADATA(a, mm, c) { \ COPY32(MODINFO_METADATA | mm->md_type, a, c); \ COPY32(mm->md_size, a, c); \ if (c) \ archsw.arch_copyin(mm->md_data, a, mm->md_size); \ a += roundup(mm->md_size, sizeof(u_long)); \ } #define MOD_END(a, c) { \ COPY32(MODINFO_END, a, c); \ COPY32(0, a, c); \ } static vm_offset_t bi_copymodules(vm_offset_t addr) { struct preloaded_file *fp; struct file_metadata *md; int c; uint64_t v; c = addr != 0; /* Start with the first module on the list, should be the kernel. */ for (fp = file_findfile(NULL, NULL); fp != NULL; fp = fp->f_next) { MOD_NAME(addr, fp->f_name, c); /* This must come first. */ MOD_TYPE(addr, fp->f_type, c); if (fp->f_args) MOD_ARGS(addr, fp->f_args, c); v = fp->f_addr; #if defined(__arm__) v -= __elfN(relocation_offset); #endif MOD_ADDR(addr, v, c); v = fp->f_size; MOD_SIZE(addr, v, c); for (md = fp->f_metadata; md != NULL; md = md->md_next) if (!(md->md_type & MODINFOMD_NOCOPY)) MOD_METADATA(addr, md, c); } MOD_END(addr, c); return(addr); } static EFI_STATUS efi_do_vmap(EFI_MEMORY_DESCRIPTOR *mm, UINTN sz, UINTN mmsz, UINT32 mmver) { EFI_MEMORY_DESCRIPTOR *desc, *viter, *vmap; EFI_STATUS ret; int curr, ndesc, nset; nset = 0; desc = mm; ndesc = sz / mmsz; vmap = malloc(sz); if (vmap == NULL) /* This isn't really an EFI error case, but pretend it is */ return (EFI_OUT_OF_RESOURCES); viter = vmap; for (curr = 0; curr < ndesc; curr++, desc = NextMemoryDescriptor(desc, mmsz)) { if ((desc->Attribute & EFI_MEMORY_RUNTIME) != 0) { ++nset; desc->VirtualStart = desc->PhysicalStart; *viter = *desc; viter = NextMemoryDescriptor(viter, mmsz); } } ret = RS->SetVirtualAddressMap(nset * mmsz, mmsz, mmver, vmap); free(vmap); return (ret); } static int bi_load_efi_data(struct preloaded_file *kfp, bool exit_bs) { EFI_MEMORY_DESCRIPTOR *mm; EFI_PHYSICAL_ADDRESS addr = 0; EFI_STATUS status; const char *efi_novmap; size_t efisz; UINTN efi_mapkey; UINTN dsz, pages, retry, sz; UINT32 mmver; struct efi_map_header *efihdr; bool do_vmap; #if defined(__amd64__) || defined(__aarch64__) struct efi_fb efifb; efifb.fb_addr = gfx_state.tg_fb.fb_addr; efifb.fb_size = gfx_state.tg_fb.fb_size; efifb.fb_height = gfx_state.tg_fb.fb_height; efifb.fb_width = gfx_state.tg_fb.fb_width; efifb.fb_stride = gfx_state.tg_fb.fb_stride; efifb.fb_mask_red = gfx_state.tg_fb.fb_mask_red; efifb.fb_mask_green = gfx_state.tg_fb.fb_mask_green; efifb.fb_mask_blue = gfx_state.tg_fb.fb_mask_blue; efifb.fb_mask_reserved = gfx_state.tg_fb.fb_mask_reserved; printf("EFI framebuffer information:\n"); printf("addr, size 0x%jx, 0x%jx\n", efifb.fb_addr, efifb.fb_size); printf("dimensions %d x %d\n", efifb.fb_width, efifb.fb_height); printf("stride %d\n", efifb.fb_stride); printf("masks 0x%08x, 0x%08x, 0x%08x, 0x%08x\n", efifb.fb_mask_red, efifb.fb_mask_green, efifb.fb_mask_blue, efifb.fb_mask_reserved); if (efifb.fb_addr != 0) file_addmetadata(kfp, MODINFOMD_EFI_FB, sizeof(efifb), &efifb); #endif do_vmap = true; efi_novmap = getenv("efi_disable_vmap"); if (efi_novmap != NULL) do_vmap = strcasecmp(efi_novmap, "YES") != 0; efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; /* * Assign size of EFI_MEMORY_DESCRIPTOR to keep compatible with * u-boot which doesn't fill this value when buffer for memory * descriptors is too small (eg. 0 to obtain memory map size) */ dsz = sizeof(EFI_MEMORY_DESCRIPTOR); /* * Allocate enough pages to hold the bootinfo block and the * memory map EFI will return to us. The memory map has an * unknown size, so we have to determine that first. Note that * the AllocatePages call can itself modify the memory map, so * we have to take that into account as well. The changes to * the memory map are caused by splitting a range of free * memory into two, so that one is marked as being loader * data. */ sz = 0; /* * Matthew Garrett has observed at least one system changing the * memory map when calling ExitBootServices, causing it to return an * error, probably because callbacks are allocating memory. * So we need to retry calling it at least once. */ for (retry = 2; retry > 0; retry--) { for (;;) { status = BS->GetMemoryMap(&sz, mm, &efi_mapkey, &dsz, &mmver); if (!EFI_ERROR(status)) break; if (status != EFI_BUFFER_TOO_SMALL) { printf("%s: GetMemoryMap error %lu\n", __func__, EFI_ERROR_CODE(status)); return (EINVAL); } if (addr != 0) BS->FreePages(addr, pages); /* Add 10 descriptors to the size to allow for * fragmentation caused by calling AllocatePages */ sz += (10 * dsz); pages = EFI_SIZE_TO_PAGES(sz + efisz); status = BS->AllocatePages(AllocateAnyPages, EfiLoaderData, pages, &addr); if (EFI_ERROR(status)) { printf("%s: AllocatePages error %lu\n", __func__, EFI_ERROR_CODE(status)); return (ENOMEM); } /* * Read the memory map and stash it after bootinfo. Align the * memory map on a 16-byte boundary (the bootinfo block is page * aligned). */ efihdr = (struct efi_map_header *)(uintptr_t)addr; mm = (void *)((uint8_t *)efihdr + efisz); sz = (EFI_PAGE_SIZE * pages) - efisz; } if (!exit_bs) break; status = BS->ExitBootServices(IH, efi_mapkey); - if (!EFI_ERROR(status)) + if (!EFI_ERROR(status)) { + boot_services_gone = 1; break; + } } if (retry == 0) { BS->FreePages(addr, pages); printf("ExitBootServices error %lu\n", EFI_ERROR_CODE(status)); return (EINVAL); } /* * This may be disabled by setting efi_disable_vmap in * loader.conf(5). By default we will setup the virtual * map entries. */ if (do_vmap) efi_do_vmap(mm, sz, dsz, mmver); efihdr->memory_size = sz; efihdr->descriptor_size = dsz; efihdr->descriptor_version = mmver; file_addmetadata(kfp, MODINFOMD_EFI_MAP, efisz + sz, efihdr); return (0); } /* * Load the information expected by an amd64 kernel. * * - The 'boothowto' argument is constructed. * - The 'bootdev' argument is constructed. * - The 'bootinfo' struct is constructed, and copied into the kernel space. * - The kernel environment is copied into kernel space. * - Module metadata are formatted and placed in kernel space. */ int bi_load(char *args, vm_offset_t *modulep, vm_offset_t *kernendp, bool exit_bs) { struct preloaded_file *xp, *kfp; struct devdesc *rootdev; struct file_metadata *md; vm_offset_t addr; uint64_t kernend, module; uint64_t envp; vm_offset_t size; char *rootdevname; int howto; #if defined(LOADER_FDT_SUPPORT) vm_offset_t dtbp; int dtb_size; #endif #if defined(__arm__) vm_offset_t vaddr; size_t i; /* * These metadata addreses must be converted for kernel after * relocation. */ uint32_t mdt[] = { MODINFOMD_SSYM, MODINFOMD_ESYM, MODINFOMD_KERNEND, MODINFOMD_ENVP, MODINFOMD_FONT, #if defined(LOADER_FDT_SUPPORT) MODINFOMD_DTBP #endif }; #endif howto = bi_getboothowto(args); /* * Allow the environment variable 'rootdev' to override the supplied * device. This should perhaps go to MI code and/or have $rootdev * tested/set by MI code before launching the kernel. */ rootdevname = getenv("rootdev"); archsw.arch_getdev((void**)(&rootdev), rootdevname, NULL); if (rootdev == NULL) { printf("Can't determine root device.\n"); return(EINVAL); } /* Try reading the /etc/fstab file to select the root device */ getrootmount(efi_fmtdev((void *)rootdev)); addr = 0; for (xp = file_findfile(NULL, NULL); xp != NULL; xp = xp->f_next) { if (addr < (xp->f_addr + xp->f_size)) addr = xp->f_addr + xp->f_size; } /* Pad to a page boundary. */ addr = roundup(addr, PAGE_SIZE); addr = build_font_module(addr); /* Pad to a page boundary. */ addr = roundup(addr, PAGE_SIZE); /* Copy our environment. */ envp = addr; addr = bi_copyenv(addr); /* Pad to a page boundary. */ addr = roundup(addr, PAGE_SIZE); #if defined(LOADER_FDT_SUPPORT) /* Handle device tree blob */ dtbp = addr; dtb_size = fdt_copy(addr); /* Pad to a page boundary */ if (dtb_size) addr += roundup(dtb_size, PAGE_SIZE); #endif kfp = file_findfile(NULL, "elf kernel"); if (kfp == NULL) kfp = file_findfile(NULL, "elf64 kernel"); if (kfp == NULL) panic("can't find kernel file"); kernend = 0; /* fill it in later */ /* Figure out the size and location of the metadata. */ module = *modulep = addr; file_addmetadata(kfp, MODINFOMD_HOWTO, sizeof(howto), &howto); file_addmetadata(kfp, MODINFOMD_ENVP, sizeof(envp), &envp); #if defined(LOADER_FDT_SUPPORT) if (dtb_size) file_addmetadata(kfp, MODINFOMD_DTBP, sizeof(dtbp), &dtbp); else printf("WARNING! Trying to fire up the kernel, but no " "device tree blob found!\n"); #endif file_addmetadata(kfp, MODINFOMD_KERNEND, sizeof(kernend), &kernend); #ifdef MODINFOMD_MODULEP file_addmetadata(kfp, MODINFOMD_MODULEP, sizeof(module), &module); #endif file_addmetadata(kfp, MODINFOMD_FW_HANDLE, sizeof(ST), &ST); #ifdef LOADER_GELI_SUPPORT geli_export_key_metadata(kfp); #endif bi_load_efi_data(kfp, exit_bs); size = bi_copymodules(0); kernend = roundup(addr + size, PAGE_SIZE); *kernendp = kernend; /* patch MODINFOMD_KERNEND */ md = file_findmetadata(kfp, MODINFOMD_KERNEND); bcopy(&kernend, md->md_data, sizeof kernend); #if defined(__arm__) *modulep -= __elfN(relocation_offset); /* Do relocation fixup on metadata of each module. */ for (xp = file_findfile(NULL, NULL); xp != NULL; xp = xp->f_next) { for (i = 0; i < nitems(mdt); i++) { md = file_findmetadata(xp, mdt[i]); if (md) { bcopy(md->md_data, &vaddr, sizeof vaddr); vaddr -= __elfN(relocation_offset); bcopy(&vaddr, md->md_data, sizeof vaddr); } } } #endif /* Copy module list and metadata. */ (void)bi_copymodules(addr); return (0); } diff --git a/stand/efi/loader/copy.c b/stand/efi/loader/copy.c index e723b61e3bca..b8ed4c8e027e 100644 --- a/stand/efi/loader/copy.c +++ b/stand/efi/loader/copy.c @@ -1,366 +1,536 @@ /*- * Copyright (c) 2013 The FreeBSD Foundation * All rights reserved. * * This software was developed by Benno Rice under sponsorship from * the FreeBSD Foundation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "loader_efi.h" +#define M(x) ((x) * 1024 * 1024) +#define G(x) (1UL * (x) * 1024 * 1024 * 1024) + +extern int boot_services_gone; + #if defined(__i386__) || defined(__amd64__) #include #include #include /* * The code is excerpted from sys/x86/x86/identcpu.c: identify_cpu(), * identify_hypervisor(), and dev/hyperv/vmbus/hyperv.c: hyperv_identify(). */ #define CPUID_LEAF_HV_MAXLEAF 0x40000000 #define CPUID_LEAF_HV_INTERFACE 0x40000001 #define CPUID_LEAF_HV_FEATURES 0x40000003 #define CPUID_LEAF_HV_LIMITS 0x40000005 #define CPUID_HV_IFACE_HYPERV 0x31237648 /* HV#1 */ #define CPUID_HV_MSR_TIME_REFCNT 0x0002 /* MSR_HV_TIME_REF_COUNT */ #define CPUID_HV_MSR_HYPERCALL 0x0020 static int running_on_hyperv(void) { char hv_vendor[16]; uint32_t regs[4]; do_cpuid(1, regs); if ((regs[2] & CPUID2_HV) == 0) return (0); do_cpuid(CPUID_LEAF_HV_MAXLEAF, regs); if (regs[0] < CPUID_LEAF_HV_LIMITS) return (0); ((uint32_t *)&hv_vendor)[0] = regs[1]; ((uint32_t *)&hv_vendor)[1] = regs[2]; ((uint32_t *)&hv_vendor)[2] = regs[3]; hv_vendor[12] = '\0'; if (strcmp(hv_vendor, "Microsoft Hv") != 0) return (0); do_cpuid(CPUID_LEAF_HV_INTERFACE, regs); if (regs[0] != CPUID_HV_IFACE_HYPERV) return (0); do_cpuid(CPUID_LEAF_HV_FEATURES, regs); if ((regs[0] & CPUID_HV_MSR_HYPERCALL) == 0) return (0); if ((regs[0] & CPUID_HV_MSR_TIME_REFCNT) == 0) return (0); return (1); } static void efi_verify_staging_size(unsigned long *nr_pages) { UINTN sz; EFI_MEMORY_DESCRIPTOR *map = NULL, *p; EFI_PHYSICAL_ADDRESS start, end; UINTN key, dsz; UINT32 dver; EFI_STATUS status; int i, ndesc; unsigned long available_pages = 0; sz = 0; for (;;) { status = BS->GetMemoryMap(&sz, map, &key, &dsz, &dver); if (!EFI_ERROR(status)) break; if (status != EFI_BUFFER_TOO_SMALL) { printf("Can't read memory map: %lu\n", EFI_ERROR_CODE(status)); goto out; } free(map); /* Allocate 10 descriptors more than the size reported, * to allow for any fragmentation caused by calling * malloc */ map = malloc(sz + (10 * dsz)); if (map == NULL) { printf("Unable to allocate memory\n"); goto out; } } ndesc = sz / dsz; for (i = 0, p = map; i < ndesc; i++, p = NextMemoryDescriptor(p, dsz)) { start = p->PhysicalStart; end = start + p->NumberOfPages * EFI_PAGE_SIZE; if (KERNLOAD < start || KERNLOAD >= end) continue; available_pages = p->NumberOfPages - ((KERNLOAD - start) >> EFI_PAGE_SHIFT); break; } if (available_pages == 0) { printf("Can't find valid memory map for staging area!\n"); goto out; } i++; p = NextMemoryDescriptor(p, dsz); for ( ; i < ndesc; i++, p = NextMemoryDescriptor(p, dsz)) { if (p->Type != EfiConventionalMemory && p->Type != EfiLoaderData) break; if (p->PhysicalStart != end) break; end = p->PhysicalStart + p->NumberOfPages * EFI_PAGE_SIZE; available_pages += p->NumberOfPages; } if (*nr_pages > available_pages) { printf("Staging area's size is reduced: %ld -> %ld!\n", *nr_pages, available_pages); *nr_pages = available_pages; } out: free(map); } #endif /* __i386__ || __amd64__ */ #ifndef EFI_STAGING_SIZE #if defined(__arm__) -#define EFI_STAGING_SIZE 32 +#define EFI_STAGING_SIZE M(32) +#else +#define EFI_STAGING_SIZE M(64) +#endif +#endif + +#if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ + defined(__riscv) +#define EFI_STAGING_2M_ALIGN 1 #else -#define EFI_STAGING_SIZE 64 +#define EFI_STAGING_2M_ALIGN 0 #endif + +#if defined(__amd64__) +#define EFI_STAGING_SLOP M(8) +#else +#define EFI_STAGING_SLOP 0 #endif +static u_long staging_slop = EFI_STAGING_SLOP; + EFI_PHYSICAL_ADDRESS staging, staging_end, staging_base; int stage_offset_set = 0; ssize_t stage_offset; +static void +efi_copy_free(void) +{ + BS->FreePages(staging_base, (staging_end - staging_base) / + EFI_PAGE_SIZE); + stage_offset_set = 0; + stage_offset = 0; +} + +#ifdef __amd64__ +int copy_staging = COPY_STAGING_ENABLE; + +static int +command_copy_staging(int argc, char *argv[]) +{ + static const char *const mode[3] = { + [COPY_STAGING_ENABLE] = "enable", + [COPY_STAGING_DISABLE] = "disable", + [COPY_STAGING_AUTO] = "auto", + }; + int prev, res; + + res = CMD_OK; + if (argc > 2) { + res = CMD_ERROR; + } else if (argc == 2) { + prev = copy_staging; + if (strcmp(argv[1], "enable") == 0) + copy_staging = COPY_STAGING_ENABLE; + else if (strcmp(argv[1], "disable") == 0) + copy_staging = COPY_STAGING_DISABLE; + else if (strcmp(argv[1], "auto") == 0) + copy_staging = COPY_STAGING_AUTO; + else { + printf("usage: copy_staging enable|disable|auto\n"); + res = CMD_ERROR; + } + if (res == CMD_OK && prev != copy_staging) { + printf("changed copy_staging, unloading kernel\n"); + unload(); + efi_copy_free(); + efi_copy_init(); + } + } else { + printf("copy staging: %s\n", mode[copy_staging]); + } + return (res); +} +COMMAND_SET(copy_staging, "copy_staging", "copy staging", command_copy_staging); +#endif + +static int +command_staging_slop(int argc, char *argv[]) +{ + char *endp; + u_long new, prev; + int res; + + res = CMD_OK; + if (argc > 2) { + res = CMD_ERROR; + } else if (argc == 2) { + new = strtoul(argv[1], &endp, 0); + if (*endp != '\0') { + printf("invalid slop value\n"); + res = CMD_ERROR; + } + if (res == CMD_OK && staging_slop != new) { + printf("changed slop, unloading kernel\n"); + unload(); + efi_copy_free(); + efi_copy_init(); + } + } else { + printf("staging slop %#lx\n", staging_slop); + } + return (res); +} +COMMAND_SET(staging_slop, "staging_slop", "set staging slop", + command_staging_slop); + +#if defined(__i386__) || defined(__amd64__) +/* + * The staging area must reside in the the first 1GB or 4GB physical + * memory: see elf64_exec() in + * boot/efi/loader/arch/amd64/elf64_freebsd.c. + */ +static EFI_PHYSICAL_ADDRESS +get_staging_max(void) +{ + EFI_PHYSICAL_ADDRESS res; + +#if defined(__i386__) + res = G(1); +#elif defined(__amd64__) + res = copy_staging == COPY_STAGING_ENABLE ? G(1) : G(4); +#endif + return (res); +} +#define EFI_ALLOC_METHOD AllocateMaxAddress +#else +#define EFI_ALLOC_METHOD AllocateAnyPages +#endif + int efi_copy_init(void) { EFI_STATUS status; - unsigned long nr_pages; - nr_pages = EFI_SIZE_TO_PAGES((EFI_STAGING_SIZE) * 1024 * 1024); + nr_pages = EFI_SIZE_TO_PAGES((EFI_STAGING_SIZE)); #if defined(__i386__) || defined(__amd64__) /* * We'll decrease nr_pages, if it's too big. Currently we only * apply this to FreeBSD VM running on Hyper-V. Why? Please see * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=211746#c28 */ if (running_on_hyperv()) efi_verify_staging_size(&nr_pages); - /* - * The staging area must reside in the the first 1GB physical - * memory: see elf64_exec() in - * boot/efi/loader/arch/amd64/elf64_freebsd.c. - */ - staging = 1024*1024*1024; - status = BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, - nr_pages, &staging); -#else - status = BS->AllocatePages(AllocateAnyPages, EfiLoaderData, - nr_pages, &staging); + staging = get_staging_max(); #endif + status = BS->AllocatePages(EFI_ALLOC_METHOD, EfiLoaderData, + nr_pages, &staging); if (EFI_ERROR(status)) { printf("failed to allocate staging area: %lu\n", EFI_ERROR_CODE(status)); return (status); } staging_base = staging; staging_end = staging + nr_pages * EFI_PAGE_SIZE; -#if defined(__aarch64__) || defined(__arm__) || defined(__riscv) +#if EFI_STAGING_2M_ALIGN /* * Round the kernel load address to a 2MiB value. This is needed * because the kernel builds a page table based on where it has * been loaded in physical address space. As the kernel will use * either a 1MiB or 2MiB page for this we need to make sure it * is correctly aligned for both cases. */ - staging = roundup2(staging, 2 * 1024 * 1024); + staging = roundup2(staging, M(2)); #endif return (0); } static bool efi_check_space(vm_offset_t end) { - EFI_PHYSICAL_ADDRESS addr; + EFI_PHYSICAL_ADDRESS addr, new_base, new_staging; EFI_STATUS status; unsigned long nr_pages; + end = roundup2(end, EFI_PAGE_SIZE); + /* There is already enough space */ - if (end <= staging_end) + if (end + staging_slop <= staging_end) return (true); - end = roundup2(end, EFI_PAGE_SIZE); - nr_pages = EFI_SIZE_TO_PAGES(end - staging_end); + if (boot_services_gone) { + if (end <= staging_end) + return (true); + panic("efi_check_space: cannot expand staging area " + "after boot services were exited\n"); + } + + /* + * Add slop at the end: + * 1. amd64 kernel expects to do some very early allocations + * by carving out memory after kernend. Slop guarantees + * that it does not ovewrite anything useful. + * 2. It seems that initial calculation of the staging size + * could be somewhat smaller than actually copying in after + * boot services are exited. Slop avoids calling + * BS->AllocatePages() when it cannot work. + */ + end += staging_slop; + nr_pages = EFI_SIZE_TO_PAGES(end - staging_end); #if defined(__i386__) || defined(__amd64__) - /* X86 needs all memory to be allocated under the 1G boundary */ - if (end > 1024*1024*1024) + /* + * i386 needs all memory to be allocated under the 1G boundary. + * amd64 needs all memory to be allocated under the 1G or 4G boundary. + */ + if (end > get_staging_max()) goto before_staging; #endif /* Try to allocate more space after the previous allocation */ addr = staging_end; status = BS->AllocatePages(AllocateAddress, EfiLoaderData, nr_pages, &addr); if (!EFI_ERROR(status)) { staging_end = staging_end + nr_pages * EFI_PAGE_SIZE; return (true); } before_staging: /* Try allocating space before the previous allocation */ - if (staging < nr_pages * EFI_PAGE_SIZE) { - printf("Not enough space before allocation\n"); - return (false); - } + if (staging < nr_pages * EFI_PAGE_SIZE) + goto expand; addr = staging - nr_pages * EFI_PAGE_SIZE; -#if defined(__aarch64__) || defined(__arm__) || defined(__riscv) +#if EFI_STAGING_2M_ALIGN /* See efi_copy_init for why this is needed */ - addr = rounddown2(addr, 2 * 1024 * 1024); + addr = rounddown2(addr, M(2)); #endif nr_pages = EFI_SIZE_TO_PAGES(staging_base - addr); status = BS->AllocatePages(AllocateAddress, EfiLoaderData, nr_pages, &addr); if (!EFI_ERROR(status)) { /* * Move the old allocation and update the state so * translation still works. */ staging_base = addr; memmove((void *)(uintptr_t)staging_base, (void *)(uintptr_t)staging, staging_end - staging); - stage_offset -= (staging - staging_base); + stage_offset -= staging - staging_base; staging = staging_base; return (true); } +expand: + nr_pages = EFI_SIZE_TO_PAGES(end - (vm_offset_t)staging); +#if EFI_STAGING_2M_ALIGN + nr_pages += M(2) / EFI_PAGE_SIZE; +#endif +#if defined(__i386__) || defined(__amd64__) + new_base = get_staging_max(); +#endif + status = BS->AllocatePages(EFI_ALLOC_METHOD, EfiLoaderData, + nr_pages, &new_base); + if (!EFI_ERROR(status)) { +#if EFI_STAGING_2M_ALIGN + new_staging = roundup2(new_base, M(2)); +#else + new_staging = new_base; +#endif + /* + * Move the old allocation and update the state so + * translation still works. + */ + memcpy((void *)(uintptr_t)new_staging, + (void *)(uintptr_t)staging, staging_end - staging); + BS->FreePages(staging_base, (staging_end - staging_base) / + EFI_PAGE_SIZE); + stage_offset -= staging - new_staging; + staging = new_staging; + staging_end = new_base + nr_pages * EFI_PAGE_SIZE; + staging_base = new_base; + return (true); + } + printf("efi_check_space: Unable to expand staging area\n"); return (false); } void * efi_translate(vm_offset_t ptr) { return ((void *)(ptr + stage_offset)); } ssize_t efi_copyin(const void *src, vm_offset_t dest, const size_t len) { if (!stage_offset_set) { stage_offset = (vm_offset_t)staging - dest; stage_offset_set = 1; } /* XXX: Callers do not check for failure. */ if (!efi_check_space(dest + stage_offset + len)) { errno = ENOMEM; return (-1); } bcopy(src, (void *)(dest + stage_offset), len); return (len); } ssize_t efi_copyout(const vm_offset_t src, void *dest, const size_t len) { /* XXX: Callers do not check for failure. */ if (src + stage_offset + len > staging_end) { errno = ENOMEM; return (-1); } bcopy((void *)(src + stage_offset), dest, len); return (len); } - ssize_t efi_readin(readin_handle_t fd, vm_offset_t dest, const size_t len) { if (!stage_offset_set) { stage_offset = (vm_offset_t)staging - dest; stage_offset_set = 1; } if (!efi_check_space(dest + stage_offset + len)) { errno = ENOMEM; return (-1); } return (VECTX_READ(fd, (void *)(dest + stage_offset), len)); } void efi_copy_finish(void) { uint64_t *src, *dst, *last; src = (uint64_t *)(uintptr_t)staging; dst = (uint64_t *)(uintptr_t)(staging - stage_offset); last = (uint64_t *)(uintptr_t)staging_end; while (src < last) *dst++ = *src++; } + +void +efi_copy_finish_nop(void) +{ +} diff --git a/stand/efi/loader/loader_efi.h b/stand/efi/loader/loader_efi.h index 4d077514e423..8254d16b1592 100644 --- a/stand/efi/loader/loader_efi.h +++ b/stand/efi/loader/loader_efi.h @@ -1,48 +1,58 @@ /*- * Copyright (c) 2013 The FreeBSD Foundation * All rights reserved. * * This software was developed by Benno Rice under sponsorship from * the FreeBSD Foundation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _LOADER_EFI_COPY_H_ #define _LOADER_EFI_COPY_H_ #include #include +#ifdef __amd64__ +enum { + COPY_STAGING_ENABLE, + COPY_STAGING_DISABLE, + COPY_STAGING_AUTO, +}; +extern int copy_staging; +#endif + int efi_autoload(void); int efi_copy_init(void); ssize_t efi_copyin(const void *src, vm_offset_t dest, const size_t len); ssize_t efi_copyout(const vm_offset_t src, void *dest, const size_t len); ssize_t efi_readin(readin_handle_t fd, vm_offset_t dest, const size_t len); void * efi_translate(vm_offset_t ptr); void efi_copy_finish(void); +void efi_copy_finish_nop(void); #endif /* _LOADER_EFI_COPY_H_ */