Changeset View
Changeset View
Standalone View
Standalone View
stand/efi/loader/copy.c
Show All 33 Lines | |||||||||||
#include <stand.h> | #include <stand.h> | ||||||||||
#include <bootstrap.h> | #include <bootstrap.h> | ||||||||||
#include <efi.h> | #include <efi.h> | ||||||||||
#include <efilib.h> | #include <efilib.h> | ||||||||||
#include "loader_efi.h" | #include "loader_efi.h" | ||||||||||
#define M(x) ((x) * 1024 * 1024) | |||||||||||
#define G(x) (1UL * (x) * 1024 * 1024 * 1024) | |||||||||||
extern int boot_services_gone; | |||||||||||
#if defined(__i386__) || defined(__amd64__) | #if defined(__i386__) || defined(__amd64__) | ||||||||||
#include <machine/cpufunc.h> | #include <machine/cpufunc.h> | ||||||||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||||||||
#include <machine/vmparam.h> | #include <machine/vmparam.h> | ||||||||||
/* | /* | ||||||||||
* The code is excerpted from sys/x86/x86/identcpu.c: identify_cpu(), | * The code is excerpted from sys/x86/x86/identcpu.c: identify_cpu(), | ||||||||||
* identify_hypervisor(), and dev/hyperv/vmbus/hyperv.c: hyperv_identify(). | * identify_hypervisor(), and dev/hyperv/vmbus/hyperv.c: hyperv_identify(). | ||||||||||
▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 Lines | efi_verify_staging_size(unsigned long *nr_pages) | ||||||||||
} | } | ||||||||||
out: | out: | ||||||||||
free(map); | free(map); | ||||||||||
} | } | ||||||||||
#endif /* __i386__ || __amd64__ */ | #endif /* __i386__ || __amd64__ */ | ||||||||||
#ifndef EFI_STAGING_SIZE | #ifndef EFI_STAGING_SIZE | ||||||||||
#if defined(__arm__) | #if defined(__arm__) | ||||||||||
#define EFI_STAGING_SIZE 32 | #define EFI_STAGING_SIZE M(32) | ||||||||||
#else | #else | ||||||||||
#define EFI_STAGING_SIZE 64 | #define EFI_STAGING_SIZE M(64) | ||||||||||
#endif | #endif | ||||||||||
#endif | #endif | ||||||||||
#if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ | |||||||||||
defined(__riscv) | |||||||||||
#define EFI_STAGING_2M_ALIGN 1 | |||||||||||
#else | |||||||||||
#define EFI_STAGING_2M_ALIGN 0 | |||||||||||
#endif | |||||||||||
#if defined(__amd64__) | |||||||||||
#define EFI_STAGING_SLOP M(8) | |||||||||||
#else | |||||||||||
#define EFI_STAGING_SLOP 0 | |||||||||||
#endif | |||||||||||
static u_long staging_slop = EFI_STAGING_SLOP; | |||||||||||
EFI_PHYSICAL_ADDRESS staging, staging_end, staging_base; | EFI_PHYSICAL_ADDRESS staging, staging_end, staging_base; | ||||||||||
int stage_offset_set = 0; | int stage_offset_set = 0; | ||||||||||
ssize_t stage_offset; | ssize_t stage_offset; | ||||||||||
static void | |||||||||||
efi_copy_free(void) | |||||||||||
{ | |||||||||||
BS->FreePages(staging_base, (staging_end - staging_base) / | |||||||||||
EFI_PAGE_SIZE); | |||||||||||
stage_offset_set = 0; | |||||||||||
stage_offset = 0; | |||||||||||
} | |||||||||||
#ifdef __amd64__ | |||||||||||
int copy_staging = COPY_STAGING_AUTO; | |||||||||||
markj: Strange indentation here. | |||||||||||
static int | |||||||||||
command_copy_staging(int argc, char *argv[]) | |||||||||||
{ | |||||||||||
static const char *const mode[3] = { | |||||||||||
[COPY_STAGING_ENABLE] = "enable", | |||||||||||
[COPY_STAGING_DISABLE] = "disable", | |||||||||||
[COPY_STAGING_AUTO] = "auto", | |||||||||||
}; | |||||||||||
int prev, res; | |||||||||||
res = CMD_OK; | |||||||||||
if (argc > 2) { | |||||||||||
res = CMD_ERROR; | |||||||||||
} else if (argc == 2) { | |||||||||||
prev = copy_staging; | |||||||||||
if (strcmp(argv[1], "enable") == 0) | |||||||||||
copy_staging = COPY_STAGING_ENABLE; | |||||||||||
else if (strcmp(argv[1], "disable") == 0) | |||||||||||
copy_staging = COPY_STAGING_DISABLE; | |||||||||||
else if (strcmp(argv[1], "auto") == 0) | |||||||||||
Done Inline ActionsI think some enum for these values would be worthwhile. markj: I think some enum for these values would be worthwhile. | |||||||||||
copy_staging = COPY_STAGING_AUTO; | |||||||||||
else { | |||||||||||
Done Inline Actions
markj: | |||||||||||
printf("usage: copy_staging enable|disable|auto\n"); | |||||||||||
res = CMD_ERROR; | |||||||||||
} | |||||||||||
if (res == CMD_OK && prev != copy_staging) { | |||||||||||
printf("changed copy_staging, unloading kernel\n"); | |||||||||||
unload(); | |||||||||||
efi_copy_free(); | |||||||||||
efi_copy_init(); | |||||||||||
} | |||||||||||
} else { | |||||||||||
printf("copy staging: %s\n", mode[copy_staging]); | |||||||||||
} | |||||||||||
return (res); | |||||||||||
} | |||||||||||
COMMAND_SET(copy_staging, "copy_staging", "copy staging", command_copy_staging); | |||||||||||
#endif | |||||||||||
static int | |||||||||||
command_staging_slop(int argc, char *argv[]) | |||||||||||
{ | |||||||||||
char *endp; | |||||||||||
u_long new, prev; | |||||||||||
int res; | |||||||||||
res = CMD_OK; | |||||||||||
if (argc > 2) { | |||||||||||
res = CMD_ERROR; | |||||||||||
} else if (argc == 2) { | |||||||||||
new = strtoul(argv[1], &endp, 0); | |||||||||||
if (*endp != '\0') { | |||||||||||
printf("invalid slop value\n"); | |||||||||||
res = CMD_ERROR; | |||||||||||
} | |||||||||||
if (res == CMD_OK && staging_slop != new) { | |||||||||||
printf("changed slop, unloading kernel\n"); | |||||||||||
unload(); | |||||||||||
efi_copy_free(); | |||||||||||
efi_copy_init(); | |||||||||||
} | |||||||||||
} else { | |||||||||||
printf("staging slop %#lx\n", staging_slop); | |||||||||||
} | |||||||||||
return (res); | |||||||||||
} | |||||||||||
COMMAND_SET(staging_slop, "staging_slop", "set staging slop", | |||||||||||
command_staging_slop); | |||||||||||
#if defined(__i386__) || defined(__amd64__) | |||||||||||
/* | |||||||||||
* The staging area must reside in the the first 1GB or 4GB physical | |||||||||||
* memory: see elf64_exec() in | |||||||||||
* boot/efi/loader/arch/amd64/elf64_freebsd.c. | |||||||||||
*/ | |||||||||||
static EFI_PHYSICAL_ADDRESS | |||||||||||
get_staging_max(void) | |||||||||||
{ | |||||||||||
EFI_PHYSICAL_ADDRESS res; | |||||||||||
#if defined(__i386__) | |||||||||||
res = G(1); | |||||||||||
#elif defined(__amd64__) | |||||||||||
res = copy_staging == COPY_STAGING_ENABLE ? G(1) : G(4); | |||||||||||
#endif | |||||||||||
return (res); | |||||||||||
} | |||||||||||
#define EFI_ALLOC_METHOD AllocateMaxAddress | |||||||||||
#else | |||||||||||
#define EFI_ALLOC_METHOD AllocateAnyPages | |||||||||||
#endif | |||||||||||
int | int | ||||||||||
efi_copy_init(void) | efi_copy_init(void) | ||||||||||
{ | { | ||||||||||
EFI_STATUS status; | EFI_STATUS status; | ||||||||||
unsigned long nr_pages; | unsigned long nr_pages; | ||||||||||
nr_pages = EFI_SIZE_TO_PAGES((EFI_STAGING_SIZE) * 1024 * 1024); | nr_pages = EFI_SIZE_TO_PAGES((EFI_STAGING_SIZE)); | ||||||||||
#if defined(__i386__) || defined(__amd64__) | #if defined(__i386__) || defined(__amd64__) | ||||||||||
/* | /* | ||||||||||
* We'll decrease nr_pages, if it's too big. Currently we only | * We'll decrease nr_pages, if it's too big. Currently we only | ||||||||||
* apply this to FreeBSD VM running on Hyper-V. Why? Please see | * apply this to FreeBSD VM running on Hyper-V. Why? Please see | ||||||||||
* https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=211746#c28 | * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=211746#c28 | ||||||||||
*/ | */ | ||||||||||
if (running_on_hyperv()) | if (running_on_hyperv()) | ||||||||||
efi_verify_staging_size(&nr_pages); | efi_verify_staging_size(&nr_pages); | ||||||||||
/* | staging = get_staging_max(); | ||||||||||
* The staging area must reside in the the first 1GB physical | |||||||||||
* memory: see elf64_exec() in | |||||||||||
* boot/efi/loader/arch/amd64/elf64_freebsd.c. | |||||||||||
*/ | |||||||||||
staging = 1024*1024*1024; | |||||||||||
status = BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, | |||||||||||
nr_pages, &staging); | |||||||||||
#else | |||||||||||
status = BS->AllocatePages(AllocateAnyPages, EfiLoaderData, | |||||||||||
nr_pages, &staging); | |||||||||||
#endif | #endif | ||||||||||
status = BS->AllocatePages(EFI_ALLOC_METHOD, EfiLoaderData, | |||||||||||
nr_pages, &staging); | |||||||||||
if (EFI_ERROR(status)) { | if (EFI_ERROR(status)) { | ||||||||||
printf("failed to allocate staging area: %lu\n", | printf("failed to allocate staging area: %lu\n", | ||||||||||
EFI_ERROR_CODE(status)); | EFI_ERROR_CODE(status)); | ||||||||||
return (status); | return (status); | ||||||||||
} | } | ||||||||||
staging_base = staging; | staging_base = staging; | ||||||||||
staging_end = staging + nr_pages * EFI_PAGE_SIZE; | staging_end = staging + nr_pages * EFI_PAGE_SIZE; | ||||||||||
#if defined(__aarch64__) || defined(__arm__) || defined(__riscv) | #if EFI_STAGING_2M_ALIGN | ||||||||||
/* | /* | ||||||||||
* Round the kernel load address to a 2MiB value. This is needed | * Round the kernel load address to a 2MiB value. This is needed | ||||||||||
* because the kernel builds a page table based on where it has | * because the kernel builds a page table based on where it has | ||||||||||
* been loaded in physical address space. As the kernel will use | * been loaded in physical address space. As the kernel will use | ||||||||||
* either a 1MiB or 2MiB page for this we need to make sure it | * either a 1MiB or 2MiB page for this we need to make sure it | ||||||||||
* is correctly aligned for both cases. | * is correctly aligned for both cases. | ||||||||||
*/ | */ | ||||||||||
staging = roundup2(staging, 2 * 1024 * 1024); | staging = roundup2(staging, M(2)); | ||||||||||
#endif | #endif | ||||||||||
return (0); | return (0); | ||||||||||
} | } | ||||||||||
static bool | static bool | ||||||||||
efi_check_space(vm_offset_t end) | efi_check_space(vm_offset_t end) | ||||||||||
{ | { | ||||||||||
EFI_PHYSICAL_ADDRESS addr; | EFI_PHYSICAL_ADDRESS addr, new_base, new_staging; | ||||||||||
EFI_STATUS status; | EFI_STATUS status; | ||||||||||
unsigned long nr_pages; | unsigned long nr_pages; | ||||||||||
end = roundup2(end, EFI_PAGE_SIZE); | |||||||||||
/* There is already enough space */ | /* There is already enough space */ | ||||||||||
if (end + staging_slop <= staging_end) | |||||||||||
return (true); | |||||||||||
if (boot_services_gone) { | |||||||||||
if (end <= staging_end) | if (end <= staging_end) | ||||||||||
return (true); | return (true); | ||||||||||
panic("efi_check_space: cannot expand staging area " | |||||||||||
"after boot services were exited\n"); | |||||||||||
Done Inline ActionsI don't think it is possible to print messages on the screen after ExitBootServices() has been called. dasebek_gmail.com: I don't think it is possible to print messages on the screen after ExitBootServices() has been… | |||||||||||
Done Inline ActionsOn screen perhaps not, on the directly driven serial port we can (which is e.g. my test configuration). There not much can be done anyway at this point. kib: On screen perhaps not, on the directly driven serial port we can (which is e.g. my test… | |||||||||||
} | |||||||||||
end = roundup2(end, EFI_PAGE_SIZE); | /* | ||||||||||
nr_pages = EFI_SIZE_TO_PAGES(end - staging_end); | * Add slop at the end: | ||||||||||
Done Inline Actions
markj: | |||||||||||
* 1. amd64 kernel expects to do some very early allocations | |||||||||||
* by carving out memory after kernend. Slop guarantees | |||||||||||
* that it does not ovewrite anything useful. | |||||||||||
Done Inline Actions
markj: | |||||||||||
* 2. It seems that initial calculation of the staging size | |||||||||||
* could be somewhat smaller than actually copying in after | |||||||||||
* boot services are exited. Slop avoids calling | |||||||||||
* BS->AllocatePages() when it cannot work. | |||||||||||
*/ | |||||||||||
end += staging_slop; | |||||||||||
Done Inline ActionsI wonder if this should be larger. We allocate at least 1 page per CPU, for example, for pcpu. Doesn't that come from this region? markj: I wonder if this should be larger. We allocate at least 1 page per CPU, for example, for pcpu. | |||||||||||
nr_pages = EFI_SIZE_TO_PAGES(end - staging_end); | |||||||||||
#if defined(__i386__) || defined(__amd64__) | #if defined(__i386__) || defined(__amd64__) | ||||||||||
/* X86 needs all memory to be allocated under the 1G boundary */ | /* | ||||||||||
if (end > 1024*1024*1024) | * i386 needs all memory to be allocated under the 1G boundary. | ||||||||||
Done Inline Actions
markj: | |||||||||||
* amd64 needs all memory to be allocated under the 1G or 4G boundary. | |||||||||||
Done Inline Actions
markj: | |||||||||||
*/ | |||||||||||
if (end > get_staging_max()) | |||||||||||
goto before_staging; | goto before_staging; | ||||||||||
#endif | #endif | ||||||||||
/* Try to allocate more space after the previous allocation */ | /* Try to allocate more space after the previous allocation */ | ||||||||||
addr = staging_end; | addr = staging_end; | ||||||||||
status = BS->AllocatePages(AllocateAddress, EfiLoaderData, nr_pages, | status = BS->AllocatePages(AllocateAddress, EfiLoaderData, nr_pages, | ||||||||||
&addr); | &addr); | ||||||||||
if (!EFI_ERROR(status)) { | if (!EFI_ERROR(status)) { | ||||||||||
staging_end = staging_end + nr_pages * EFI_PAGE_SIZE; | staging_end = staging_end + nr_pages * EFI_PAGE_SIZE; | ||||||||||
return (true); | return (true); | ||||||||||
} | } | ||||||||||
before_staging: | before_staging: | ||||||||||
/* Try allocating space before the previous allocation */ | /* Try allocating space before the previous allocation */ | ||||||||||
if (staging < nr_pages * EFI_PAGE_SIZE) { | if (staging < nr_pages * EFI_PAGE_SIZE) | ||||||||||
printf("Not enough space before allocation\n"); | goto expand; | ||||||||||
return (false); | |||||||||||
} | |||||||||||
addr = staging - nr_pages * EFI_PAGE_SIZE; | addr = staging - nr_pages * EFI_PAGE_SIZE; | ||||||||||
#if defined(__aarch64__) || defined(__arm__) || defined(__riscv) | #if EFI_STAGING_2M_ALIGN | ||||||||||
/* See efi_copy_init for why this is needed */ | /* See efi_copy_init for why this is needed */ | ||||||||||
addr = rounddown2(addr, 2 * 1024 * 1024); | addr = rounddown2(addr, M(2)); | ||||||||||
#endif | #endif | ||||||||||
nr_pages = EFI_SIZE_TO_PAGES(staging_base - addr); | nr_pages = EFI_SIZE_TO_PAGES(staging_base - addr); | ||||||||||
status = BS->AllocatePages(AllocateAddress, EfiLoaderData, nr_pages, | status = BS->AllocatePages(AllocateAddress, EfiLoaderData, nr_pages, | ||||||||||
&addr); | &addr); | ||||||||||
if (!EFI_ERROR(status)) { | if (!EFI_ERROR(status)) { | ||||||||||
/* | /* | ||||||||||
* Move the old allocation and update the state so | * Move the old allocation and update the state so | ||||||||||
* translation still works. | * translation still works. | ||||||||||
*/ | */ | ||||||||||
staging_base = addr; | staging_base = addr; | ||||||||||
memmove((void *)(uintptr_t)staging_base, | memmove((void *)(uintptr_t)staging_base, | ||||||||||
(void *)(uintptr_t)staging, staging_end - staging); | (void *)(uintptr_t)staging, staging_end - staging); | ||||||||||
stage_offset -= (staging - staging_base); | stage_offset -= staging - staging_base; | ||||||||||
staging = staging_base; | staging = staging_base; | ||||||||||
return (true); | return (true); | ||||||||||
} | } | ||||||||||
expand: | |||||||||||
nr_pages = EFI_SIZE_TO_PAGES(end - (vm_offset_t)staging); | |||||||||||
#if EFI_STAGING_2M_ALIGN | |||||||||||
nr_pages += M(2) / EFI_PAGE_SIZE; | |||||||||||
#endif | |||||||||||
#if defined(__i386__) || defined(__amd64__) | |||||||||||
new_base = get_staging_max(); | |||||||||||
#endif | |||||||||||
status = BS->AllocatePages(EFI_ALLOC_METHOD, EfiLoaderData, | |||||||||||
nr_pages, &new_base); | |||||||||||
if (!EFI_ERROR(status)) { | |||||||||||
#if EFI_STAGING_2M_ALIGN | |||||||||||
new_staging = roundup2(new_base, M(2)); | |||||||||||
#else | |||||||||||
new_staging = new_base; | |||||||||||
#endif | |||||||||||
/* | |||||||||||
* Move the old allocation and update the state so | |||||||||||
* translation still works. | |||||||||||
*/ | |||||||||||
memcpy((void *)(uintptr_t)new_staging, | |||||||||||
(void *)(uintptr_t)staging, staging_end - staging); | |||||||||||
BS->FreePages(staging_base, (staging_end - staging_base) / | |||||||||||
EFI_PAGE_SIZE); | |||||||||||
stage_offset -= staging - new_staging; | |||||||||||
staging = new_staging; | |||||||||||
staging_end = new_base + nr_pages * EFI_PAGE_SIZE; | |||||||||||
staging_base = new_base; | |||||||||||
return (true); | |||||||||||
} | |||||||||||
printf("efi_check_space: Unable to expand staging area\n"); | printf("efi_check_space: Unable to expand staging area\n"); | ||||||||||
return (false); | return (false); | ||||||||||
} | } | ||||||||||
void * | void * | ||||||||||
efi_translate(vm_offset_t ptr) | efi_translate(vm_offset_t ptr) | ||||||||||
{ | { | ||||||||||
Show All 26 Lines | efi_copyout(const vm_offset_t src, void *dest, const size_t len) | ||||||||||
if (src + stage_offset + len > staging_end) { | if (src + stage_offset + len > staging_end) { | ||||||||||
errno = ENOMEM; | errno = ENOMEM; | ||||||||||
return (-1); | return (-1); | ||||||||||
} | } | ||||||||||
bcopy((void *)(src + stage_offset), dest, len); | bcopy((void *)(src + stage_offset), dest, len); | ||||||||||
return (len); | return (len); | ||||||||||
} | } | ||||||||||
ssize_t | ssize_t | ||||||||||
efi_readin(readin_handle_t fd, vm_offset_t dest, const size_t len) | efi_readin(readin_handle_t fd, vm_offset_t dest, const size_t len) | ||||||||||
{ | { | ||||||||||
if (!stage_offset_set) { | if (!stage_offset_set) { | ||||||||||
stage_offset = (vm_offset_t)staging - dest; | stage_offset = (vm_offset_t)staging - dest; | ||||||||||
stage_offset_set = 1; | stage_offset_set = 1; | ||||||||||
} | } | ||||||||||
Show All 11 Lines | efi_copy_finish(void) | ||||||||||
uint64_t *src, *dst, *last; | uint64_t *src, *dst, *last; | ||||||||||
src = (uint64_t *)(uintptr_t)staging; | src = (uint64_t *)(uintptr_t)staging; | ||||||||||
dst = (uint64_t *)(uintptr_t)(staging - stage_offset); | dst = (uint64_t *)(uintptr_t)(staging - stage_offset); | ||||||||||
last = (uint64_t *)(uintptr_t)staging_end; | last = (uint64_t *)(uintptr_t)staging_end; | ||||||||||
while (src < last) | while (src < last) | ||||||||||
*dst++ = *src++; | *dst++ = *src++; | ||||||||||
} | |||||||||||
void | |||||||||||
efi_copy_finish_nop(void) | |||||||||||
{ | |||||||||||
} | } |
Strange indentation here.