Index: tests/sys/posixshm/posixshm_test.c =================================================================== --- tests/sys/posixshm/posixshm_test.c +++ tests/sys/posixshm/posixshm_test.c @@ -28,6 +28,7 @@ __FBSDID("$FreeBSD$"); #include +#include #include #include #include @@ -743,7 +744,6 @@ static void ignoreit(int sig __unused) { - ; } ATF_TC_WITHOUT_HEAD(shm_functionality_across_fork); @@ -956,6 +956,521 @@ close(fd); } +static int +shm_open_large(int psind, int policy, size_t sz) +{ + struct shm_largepage_conf lpc; + int error, fd; + + fd = shm_open2(SHM_ANON, O_CREAT | O_RDWR, 0, SHM_LARGEPAGE); + ATF_REQUIRE_MSG(fd >= 0, "shm_open2 failed; errno=%d", errno); + + lpc.psind = psind; + lpc.alloc_policy = policy; + + error = ioctl(fd, FIOSHMLPGCNF, &lpc); + ATF_REQUIRE_MSG(error == 0, "ioctl(FIOSHMLPGCNF) failed; errno=%d", + errno); + + error = ftruncate(fd, sz); + if (error != 0 && errno == ENOMEM) + /* XXX depends on alloc policy */ + atf_tc_skip("failed to allocate %zu-byte superpage", sz); + ATF_REQUIRE_MSG(error == 0, "ftruncate failed; errno=%d", errno); + + return (fd); +} + +static int +pagesizes(size_t ps[MAXPAGESIZES]) +{ + int pscnt; + + pscnt = getpagesizes(ps, MAXPAGESIZES); + ATF_REQUIRE_MSG(pscnt != -1, "getpagesizes failed; errno=%d", errno); + ATF_REQUIRE_MSG(ps[0] == PAGE_SIZE, "psind 0 is %zu", ps[0]); + if (pscnt == 1) + atf_tc_skip("no large page support"); + + return (pscnt); +} + +ATF_TC_WITHOUT_HEAD(largepage_basic); +ATF_TC_BODY(largepage_basic, tc) +{ + char zeroes[PAGE_SIZE]; + char *addr, *vec; + size_t ps[MAXPAGESIZES]; + int error, fd, pscnt; + + memset(zeroes, 0, PAGE_SIZE); + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; errno=%d", ps[i], errno); + ATF_REQUIRE_MSG(((uintptr_t)addr & (ps[i] - 1)) == 0, + "mmap(%zu bytes) returned unaligned mapping; addr=%p", + ps[i], addr); + + /* Force a page fault. */ + *addr = 0; + + vec = malloc(ps[i] / PAGE_SIZE); + ATF_REQUIRE(vec != NULL); + error = mincore(addr, ps[i], vec); + ATF_REQUIRE_MSG(error == 0, "mincore failed; errno=%d", errno); + + /* Verify that all pages in the run are mapped. */ + for (size_t p = 0; p < ps[i] / PAGE_SIZE; p++) { + ATF_REQUIRE_MSG((vec[p] & MINCORE_INCORE) != 0, + "page %zu is not mapped", p); + ATF_REQUIRE_MSG((vec[p] & MINCORE_SUPERIDX(i)) != 0, + "page %zu is not in a %zu-byte superpage", + p, ps[i]); + } + + /* Validate zeroing. */ + for (size_t p = 0; p < ps[i] / PAGE_SIZE; p++) { + ATF_REQUIRE_MSG(memcmp(addr + p * PAGE_SIZE, zeroes, + PAGE_SIZE) == 0, "page %zu miscompare", p); + } + + free(vec); + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + +ATF_TC_WITHOUT_HEAD(largepage_config); +ATF_TC_BODY(largepage_config, tc) +{ + struct shm_largepage_conf lpc; + char *addr, *buf; + size_t ps[MAXPAGESIZES]; + int error, fd, pscnt; + + pscnt = pagesizes(ps); + + fd = shm_open(SHM_ANON, O_CREAT | O_RDWR, 0); + ATF_REQUIRE_MSG(fd >= 0, "shm_open failed; error=%d", errno); + + /* + * Configure a large page policy for an object created without + * SHM_LARGEPAGE. + */ + lpc.psind = 1; + lpc.alloc_policy = SHM_LARGEPAGE_ALLOC_DEFAULT; + error = ioctl(fd, FIOSHMLPGCNF, &lpc); + ATF_REQUIRE(error != 0); + ATF_REQUIRE_MSG(errno == ENOTTY, "ioctl(FIOSHMLPGCNF) returned %d", + errno); + ATF_REQUIRE(close(fd) == 0); + + /* + * Create a largepage object and try to use it without actually + * configuring anything. + */ + fd = shm_open2(SHM_ANON, O_CREAT | O_RDWR, 0, SHM_LARGEPAGE); + ATF_REQUIRE_MSG(fd >= 0, "shm_open2 failed; error=%d", errno); + + error = ftruncate(fd, ps[1]); + ATF_REQUIRE(error != 0); + ATF_REQUIRE_MSG(errno == EINVAL, "ftruncate returned %d", errno); + + addr = mmap(NULL, ps[1], PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + ATF_REQUIRE(addr == MAP_FAILED); + ATF_REQUIRE_MSG(errno == EINVAL, "mmap returned %d", errno); + addr = mmap(NULL, 0, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + ATF_REQUIRE(addr == MAP_FAILED); + ATF_REQUIRE_MSG(errno == EINVAL, "mmap returned %d", errno); + + buf = calloc(1, PAGE_SIZE); + ATF_REQUIRE(buf != NULL); + ATF_REQUIRE(write(fd, buf, PAGE_SIZE) == -1); + ATF_REQUIRE_MSG(errno == EINVAL, "write returned %d", errno); + free(buf); + buf = calloc(1, ps[1]); + ATF_REQUIRE(buf != NULL); + ATF_REQUIRE(write(fd, buf, ps[1]) == -1); + ATF_REQUIRE_MSG(errno == EINVAL, "write returned %d", errno); + free(buf); + + error = posix_fallocate(fd, 0, PAGE_SIZE); + ATF_REQUIRE_MSG(error == EINVAL, "posix_fallocate returned %d", error); + + ATF_REQUIRE(close(fd) == 0); +} + +ATF_TC_WITHOUT_HEAD(largepage_map); +ATF_TC_BODY(largepage_map, tc) +{ + char *addr, *addr1, *vec; + size_t ps[MAXPAGESIZES]; + int fd, pscnt; + char ch; + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + + /* + * Wrong mapping size. + */ + addr = mmap(NULL, ps[i - 1], PROT_READ | PROT_WRITE, MAP_SHARED, + fd, 0); + ATF_REQUIRE_MSG(addr == MAP_FAILED, "mmap(%zu bytes) succeeded", + ps[i - 1]); + ATF_REQUIRE_MSG(errno == EINVAL, "mmap(%zu bytes) error=%d", + ps[i - 1], errno); + + /* + * Fixed mappings. + */ + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; errno=%d", ps[i], errno); + ATF_REQUIRE_MSG(((uintptr_t)addr & (ps[i] - 1)) == 0, + "mmap(%zu bytes) returned unaligned mapping; addr=%p", + ps[i], addr); + + /* Try overwriting a small page with anonymous memory. */ + addr1 = mmap(addr, ps[i - 1], PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ATF_REQUIRE_MSG(addr1 == MAP_FAILED, + "anon mmap(%zu bytes) succeeded", ps[i - 1]); + /* XXX wrong errno */ + ATF_REQUIRE_MSG(errno == ENOMEM, "mmap returned %d", errno); + + /* Check MAP_EXCL when creating a second largepage mapping. */ + addr1 = mmap(addr, ps[i], PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED | MAP_EXCL, fd, 0); + ATF_REQUIRE_MSG(addr1 == MAP_FAILED, + "remap(%zu bytes) succeeded", ps[i]); + /* XXX wrong errno */ + ATF_REQUIRE_MSG(errno == ENOSPC, "mmap returned %d", errno); + + /* Overwrite a largepage mapping with a lagepage mapping. */ + addr1 = mmap(addr, ps[i], PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + ATF_REQUIRE_MSG(addr1 != MAP_FAILED, + "mmap(%zu bytes) failed; errno=%d", ps[i], errno); + ATF_REQUIRE_MSG(addr == addr1, + "mmap(%zu bytes) moved from %p to %p", ps[i], addr, addr1); + + ATF_REQUIRE(munmap(addr, ps[i] == 0)); + + /* + * Copy-on-write mappings. + */ + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_PRIVATE, + fd, 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; errno=%d", ps[i], errno); + ATF_REQUIRE_MSG(((uintptr_t)addr & (ps[i] - 1)) == 0, + "mmap(%zu bytes) returned unaligned mapping; addr=%p", + ps[i], addr); + + vec = malloc(ps[i]); + ATF_REQUIRE(vec != NULL); + + /* Trigger a read fault to populate the mapping. */ + ch = *(volatile char *)addr; + ATF_REQUIRE(mincore(addr, ps[i], vec) == 0); + for (size_t p = 0; p < ps[i] / PAGE_SIZE; p++) { + ATF_REQUIRE_MSG((vec[p] & MINCORE_INCORE) != 0, + "page %zu is not resident", p); + ATF_REQUIRE_MSG((vec[p] & MINCORE_SUPERIDX(i)) != 0, + "page %zu is not resident", p); + } + + /* XXX this result doesn't make sense */ + *(volatile char *)addr = 0; + ATF_REQUIRE(mincore(addr, ps[i], vec) == 0); + for (size_t p = 0; p < ps[i] / PAGE_SIZE; p++) { + ATF_REQUIRE_MSG((vec[p] & MINCORE_INCORE) != 0, + "page %zu is not resident", p); + ATF_REQUIRE_MSG((vec[p] & MINCORE_SUPERIDX(i)) != 0, + "page %zu is not resident", p); + } + + free(vec); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + +ATF_TC_WITHOUT_HEAD(largepage_unmap); +ATF_TC_BODY(largepage_unmap, tc) +{ + char *addr; + size_t ps[MAXPAGESIZES], ps1; + int fd, pscnt; + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + ps1 = ps[i - 1]; + + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; errno=%d", ps[i], errno); + + /* Try several unaligned munmap() requests. */ + ATF_REQUIRE(munmap(addr, ps1) != 0); + ATF_REQUIRE_MSG(errno == EINVAL, + "unexpected error %d from munmap", errno); + ATF_REQUIRE(munmap(addr, ps[i] - ps1)); + ATF_REQUIRE_MSG(errno == EINVAL, + "unexpected error %d from munmap", errno); + ATF_REQUIRE(munmap(addr + ps1, ps1) != 0); + ATF_REQUIRE_MSG(errno == EINVAL, + "unexpected error %d from munmap", errno); + ATF_REQUIRE(munmap(addr, 0)); + ATF_REQUIRE_MSG(errno == EINVAL, + "unexpected error %d from munmap", errno); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + +static void +largepage_madvise(char *addr, size_t sz, int advice, int error) +{ + if (error == 0) { + ATF_REQUIRE_MSG(madvise(addr, sz, advice) == 0, + "madvise(%zu, %d) failed; error=%d", sz, advice, errno); + } else { + ATF_REQUIRE_MSG(madvise(addr, sz, advice) != 0, + "madvise(%zu, %d) succeeded", sz, advice); + ATF_REQUIRE_MSG(errno == error, + "unexpected error %d from madvise(%zu, %d)", + errno, sz, advice); + } +} + +ATF_TC_WITHOUT_HEAD(largepage_madvise); +ATF_TC_BODY(largepage_madvise, tc) +{ + char *addr; + size_t ps[MAXPAGESIZES]; + int fd, pscnt; + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; error=%d", ps[i], errno); + + /* Advice that requires clipping. */ + largepage_madvise(addr, PAGE_SIZE, MADV_NORMAL, EINVAL); + largepage_madvise(addr, ps[i], MADV_NORMAL, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_RANDOM, EINVAL); + largepage_madvise(addr, ps[i], MADV_RANDOM, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_SEQUENTIAL, EINVAL); + largepage_madvise(addr, ps[i], MADV_SEQUENTIAL, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_NOSYNC, EINVAL); + largepage_madvise(addr, ps[i], MADV_NOSYNC, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_AUTOSYNC, EINVAL); + largepage_madvise(addr, ps[i], MADV_AUTOSYNC, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_CORE, EINVAL); + largepage_madvise(addr, ps[i], MADV_CORE, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_NOCORE, EINVAL); + largepage_madvise(addr, ps[i], MADV_NOCORE, 0); + + /* Advice that does not result in clipping. */ + largepage_madvise(addr, PAGE_SIZE, MADV_DONTNEED, 0); + largepage_madvise(addr, ps[i], MADV_DONTNEED, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_WILLNEED, 0); + largepage_madvise(addr, ps[i], MADV_WILLNEED, 0); + largepage_madvise(addr, PAGE_SIZE, MADV_FREE, 0); + largepage_madvise(addr, ps[i], MADV_FREE, 0); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + +ATF_TC(largepage_mlock); +ATF_TC_HEAD(largepage_mlock, tc) +{ + /* Needed to set rlimit. */ + atf_tc_set_md_var(tc, "require.user", "root"); +} +ATF_TC_BODY(largepage_mlock, tc) +{ + struct rlimit rl; + char *addr; + size_t ps[MAXPAGESIZES]; + int fd, pscnt; + + atf_tc_skip("triggers kernel panics"); + + /* XXX max_user_wired also needs to be bumped */ + rl.rlim_cur = rl.rlim_max = RLIM_INFINITY; + ATF_REQUIRE_MSG(setrlimit(RLIMIT_MEMLOCK, &rl) == 0, + "setrlimit failed; error=%d", errno); + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; error=%d", ps[i], errno); + + ATF_REQUIRE(mlock(addr, PAGE_SIZE) != 0); + ATF_REQUIRE_MSG(errno == EINVAL, + "unexpected error %d from mlock(%zu bytes)", errno, ps[i]); + ATF_REQUIRE(mlock(addr, ps[i] - PAGE_SIZE) != 0); + ATF_REQUIRE_MSG(errno == EINVAL, + "unexpected error %d from mlock(%zu bytes)", errno, ps[i]); + + ATF_REQUIRE_MSG(mlock(addr, ps[i]) == 0, + "mlock failed; error=%d", errno); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + + ATF_REQUIRE(mlockall(MCL_FUTURE) == 0); + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; error=%d", ps[i], errno); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + +ATF_TC_WITHOUT_HEAD(largepage_msync); +ATF_TC_BODY(largepage_msync, tc) +{ + char *addr; + size_t ps[MAXPAGESIZES]; + int fd, pscnt; + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; error=%d", ps[i], errno); + + memset(addr, 0, ps[i]); + + /* + * "Sync" requests are no-ops for SHM objects, so small + * PAGE_SIZE-sized requests succeed. + */ + ATF_REQUIRE_MSG(msync(addr, PAGE_SIZE, MS_ASYNC) == 0, + "msync(MS_ASYNC) failed; error=%d", errno); + ATF_REQUIRE_MSG(msync(addr, ps[i], MS_ASYNC) == 0, + "msync(MS_ASYNC) failed; error=%d", errno); + ATF_REQUIRE_MSG(msync(addr, PAGE_SIZE, MS_SYNC) == 0, + "msync(MS_SYNC) failed; error=%d", errno); + ATF_REQUIRE_MSG(msync(addr, ps[i], MS_SYNC) == 0, + "msync(MS_SYNC) failed; error=%d", errno); + + ATF_REQUIRE_MSG(msync(addr, PAGE_SIZE, MS_INVALIDATE) != 0, + "msync(MS_INVALIDATE) succeeded"); + /* XXX wrong errno */ + ATF_REQUIRE_MSG(errno == EBUSY, + "unexpected error %d from msync(MS_INVALIDATE)", errno); + ATF_REQUIRE_MSG(msync(addr, ps[i], MS_INVALIDATE) == 0, + "msync(MS_INVALIDATE) failed; error=%d", errno); + memset(addr, 0, ps[i]); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + +static void +largepage_protect(char *addr, size_t sz, int prot, int error) +{ + if (error == 0) { + ATF_REQUIRE_MSG(mprotect(addr, sz, prot) == 0, + "mprotect(%zu, %x) failed; error=%d", sz, prot, errno); + } else { + ATF_REQUIRE_MSG(mprotect(addr, sz, prot) != 0, + "mprotect(%zu, %x) succeeded", sz, prot); + ATF_REQUIRE_MSG(errno == error, + "unexpected error %d from mprotect(%zu, %x)", + errno, sz, prot); + } +} + +ATF_TC_WITHOUT_HEAD(largepage_protect); +ATF_TC_BODY(largepage_protect, tc) +{ + char *addr, *addr1; + size_t ps[MAXPAGESIZES]; + int fd, pscnt; + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + addr = mmap(NULL, ps[i], PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; error=%d", ps[i], errno); + + /* + * These should be no-ops from the pmap perspective since the + * page is not yet entered into the pmap. + */ + largepage_protect(addr, PAGE_SIZE, PROT_READ, EINVAL); + largepage_protect(addr, ps[i], PROT_READ, 0); + largepage_protect(addr, PAGE_SIZE, PROT_NONE, EINVAL); + largepage_protect(addr, ps[i], PROT_NONE, 0); + largepage_protect(addr, PAGE_SIZE, + PROT_READ | PROT_WRITE | PROT_EXEC, EINVAL); + largepage_protect(addr, ps[i], + PROT_READ | PROT_WRITE | PROT_EXEC, 0); + + /* Trigger creation of a mapping and try again. */ + *(volatile char *)addr = 0; + largepage_protect(addr, PAGE_SIZE, PROT_READ, EINVAL); + largepage_protect(addr, ps[i], PROT_READ, 0); + largepage_protect(addr, PAGE_SIZE, PROT_NONE, EINVAL); + largepage_protect(addr, ps[i], PROT_NONE, 0); + largepage_protect(addr, PAGE_SIZE, + PROT_READ | PROT_WRITE | PROT_EXEC, EINVAL); + largepage_protect(addr, ps[i], + PROT_READ | PROT_WRITE | PROT_EXEC, 0); + + memset(addr, 0, ps[i]); + + /* Map two contiguous large pages and merge map entries. */ + addr1 = mmap(addr + ps[i], ps[i], PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED | MAP_EXCL, fd, 0); + /* XXX can fail if no space exists, use MAP_GUARD */ + ATF_REQUIRE_MSG(addr1 != MAP_FAILED, + "mmap(%zu bytes) failed; error=%d", ps[i], errno); + + largepage_protect(addr1 - PAGE_SIZE, PAGE_SIZE * 2, + PROT_READ | PROT_WRITE, EINVAL); + largepage_protect(addr, ps[i] * 2, PROT_READ | PROT_WRITE, 0); + + memset(addr, 0, ps[i] * 2); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(munmap(addr1, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + ATF_TP_ADD_TCS(tp) { @@ -990,6 +1505,14 @@ ATF_TP_ADD_TC(tp, cloexec); ATF_TP_ADD_TC(tp, mode); ATF_TP_ADD_TC(tp, fallocate); + ATF_TP_ADD_TC(tp, largepage_basic); + ATF_TP_ADD_TC(tp, largepage_config); + ATF_TP_ADD_TC(tp, largepage_map); + ATF_TP_ADD_TC(tp, largepage_unmap); + ATF_TP_ADD_TC(tp, largepage_madvise); + ATF_TP_ADD_TC(tp, largepage_mlock); + ATF_TP_ADD_TC(tp, largepage_msync); + ATF_TP_ADD_TC(tp, largepage_protect); return (atf_no_error()); }