Page MenuHomeFreeBSD
Files F176238622

D59890.id.diff
No OneTemporary
Actions

D59890.id.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
diff --git a/contrib/ofed/include/udma_barrier.h b/contrib/ofed/include/udma_barrier.h
--- a/contrib/ofed/include/udma_barrier.h
+++ b/contrib/ofed/include/udma_barrier.h
@@ -104,6 +104,10 @@
#include <sys/types.h>
#include <machine/atomic.h>
#define udma_to_device_barrier() dmb()
+#elif defined(__loongarch__)
+#include <sys/types.h>
+#include <machine/atomic.h>
+#define udma_to_device_barrier() fence()
#elif defined(__riscv)
#include <sys/types.h>
#include <machine/atomic.h>
@@ -144,6 +148,8 @@
#define udma_from_device_barrier() mips_sync()
#elif defined(__arm__)
#define udma_from_device_barrier() dmb()
+#elif defined(__loongarch__)
+#define udma_from_device_barrier() fence()
#elif defined(__riscv)
#define udma_from_device_barrier() fence()
#else
@@ -214,6 +220,8 @@
#define mmio_flush_writes() mips_sync()
#elif defined(__arm__)
#define mmio_flush_writes() dmb()
+#elif defined(__loongarch__)
+#define mmio_flush_writes() fence()
#elif defined(__riscv)
#define mmio_flush_writes() fence()
#else
diff --git a/sys/contrib/openzfs/include/os/freebsd/spl/sys/isa_defs.h b/sys/contrib/openzfs/include/os/freebsd/spl/sys/isa_defs.h
--- a/sys/contrib/openzfs/include/os/freebsd/spl/sys/isa_defs.h
+++ b/sys/contrib/openzfs/include/os/freebsd/spl/sys/isa_defs.h
@@ -139,6 +139,12 @@
#endif
#define _SUNOS_VTOC_16
+#elif defined(__loongarch__)
+#if !defined(_LP64)
+#define _LP64
+#endif
+#define _SUNOS_VTOC_16
+
#elif defined(__riscv)
/*
diff --git a/sys/dev/hwpmc/hwpmc_loongarch.h b/sys/dev/hwpmc/hwpmc_loongarch.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/hwpmc/hwpmc_loongarch.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * This software was developed by the University of Cambridge Computer
+ * Laboratory with support from ARM Ltd.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DEV_HWPMC_LOONGARCH_H_
+#define _DEV_HWPMC_LOONGARCH_H_
+
+#define LOONGARCH_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \
+ PMC_CAP_SYSTEM | PMC_CAP_EDGE | \
+ PMC_CAP_THRESHOLD | PMC_CAP_READ | \
+ PMC_CAP_WRITE | PMC_CAP_INVERT | \
+ PMC_CAP_QUALIFIER)
+
+#define LOONGARCH_RELOAD_COUNT_TO_PERFCTR_VALUE(R) (-(R))
+#define LOONGARCH_PERFCTR_VALUE_TO_RELOAD_COUNT(P) (-(P))
+#define EVENT_ID_MASK 0xFF
+
+#ifdef _KERNEL
+/* MD extension for 'struct pmc' */
+struct pmc_md_loongarch_pmc {
+ uint32_t pm_loongarch_evsel;
+};
+#endif /* _KERNEL */
+#endif /* _DEV_HWPMC_LOONGARCH_H_ */
diff --git a/sys/dev/sym/sym_hipd.c b/sys/dev/sym/sym_hipd.c
--- a/sys/dev/sym/sym_hipd.c
+++ b/sys/dev/sym/sym_hipd.c
@@ -128,6 +128,8 @@
#define MEMORY_BARRIER() dmb()
#elif defined __aarch64__
#define MEMORY_BARRIER() dmb(sy)
+#elif defined __loongarch__
+#define MEMORY_BARRIER() fence()
#elif defined __riscv
#define MEMORY_BARRIER() fence()
#else
diff --git a/sys/dev/uart/uart_cpu_loongarch.c b/sys/dev/uart/uart_cpu_loongarch.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/uart/uart_cpu_loongarch.c
@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ */
+
+/*
+ * LoongArch uart_cpu implementation, modeled on arm64's uart_cpu_arm64.c.
+ * Provides uart_cpu_getdev (dispatches to ACPI SPCR), uart_cpu_eqres, and
+ * the bus_space tags that the uart driver expects.
+ */
+
+#include <sys/cdefs.h>
+#include "opt_acpi.h"
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+
+#include <dev/uart/uart.h>
+#include <dev/uart/uart_cpu.h>
+#include <dev/uart/uart_bus.h>
+#include <dev/uart/uart_dev_ns8250.h>
+
+#ifdef DEV_ACPI
+#include <dev/uart/uart_cpu_acpi.h>
+#endif
+
+extern struct bus_space memmap_bus;
+
+bus_space_tag_t uart_bus_space_io;
+bus_space_tag_t uart_bus_space_mem = &memmap_bus;
+
+int
+uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2)
+{
+
+ if (pmap_kextract(b1->bsh) == 0)
+ return (0);
+ if (pmap_kextract(b2->bsh) == 0)
+ return (0);
+ return ((pmap_kextract(b1->bsh) == pmap_kextract(b2->bsh)) ? 1 : 0);
+}
+
+int
+uart_cpu_getdev(int devtype, struct uart_devinfo *di)
+{
+ struct uart_class *class;
+ int err;
+
+ class = &uart_ns8250_class;
+ err = uart_getenv(devtype, di, class);
+ if (err == 0)
+ return (0);
+
+#ifdef DEV_ACPI
+ if (uart_cpu_acpi_setup(devtype, di) == 0) {
+ uart_bus_space_mem = di->bas.bst;
+ uart_bus_space_io = NULL;
+ return (0);
+ }
+#endif
+ return (ENXIO);
+}
diff --git a/sys/kern/link_elf_obj.c b/sys/kern/link_elf_obj.c
--- a/sys/kern/link_elf_obj.c
+++ b/sys/kern/link_elf_obj.c
@@ -992,6 +992,17 @@
vm_object_color(ef->object, 0);
#endif
+/*
+ * if platform defines a reserved module KVA window
+ * (KMOD_MIN_ADDRESS, KMOD_MAX_ADDRESS), restrict
+ * module mappings to it.
+ */
+#ifdef KMOD_MAX_ADDRESS
+ mapbase = KMOD_MIN_ADDRESS;
+ error = vm_map_find(kernel_map, ef->object, 0, &mapbase,
+ round_page(mapsize), KMOD_MAX_ADDRESS,
+ VMFS_OPTIMAL_SPACE, VM_PROT_ALL, VM_PROT_ALL, 0);
+#else
/*
* In order to satisfy amd64's architectural requirements on the
* location of code and data in the kernel's address space, request a
@@ -1007,6 +1018,7 @@
error = vm_map_find(kernel_map, ef->object, 0, &mapbase,
round_page(mapsize), 0, VMFS_OPTIMAL_SPACE, VM_PROT_ALL,
VM_PROT_ALL, 0);
+#endif
if (error != KERN_SUCCESS) {
vm_object_deallocate(ef->object);
ef->object = NULL;
diff --git a/sys/kern/subr_devmap.c b/sys/kern/subr_devmap.c
--- a/sys/kern/subr_devmap.c
+++ b/sys/kern/subr_devmap.c
@@ -59,7 +59,7 @@
#endif
static vm_offset_t akva_devmap_vaddr = DEVMAP_MAX_VADDR;
-#if defined(__aarch64__) || defined(__riscv)
+#if defined(__aarch64__) || defined(__loongarch__) || defined(__riscv)
extern int early_boot;
#endif
diff --git a/sys/kern/subr_physmem.c b/sys/kern/subr_physmem.c
--- a/sys/kern/subr_physmem.c
+++ b/sys/kern/subr_physmem.c
@@ -75,7 +75,7 @@
#if defined(__arm__)
#define MAX_PHYS_ADDR 0xFFFFFFFFull
-#elif defined(__aarch64__) || defined(__amd64__) || defined(__riscv)
+#elif defined(__aarch64__) || defined(__amd64__) || defined(__loongarch__) || defined(__riscv)
#define MAX_PHYS_ADDR 0xFFFFFFFFFFFFFFFFull
#endif
diff --git a/sys/loongarch/conf/DEFAULTS b/sys/loongarch/conf/DEFAULTS
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/conf/DEFAULTS
@@ -0,0 +1,20 @@
+#
+# DEFAULTS -- Default kernel configuration file for FreeBSD/LoongArch
+#
+
+machine loongarch loongarch64
+
+# Pseudo devices.
+device mem # Memory and kernel memory devices
+
+# Default partitioning schemes
+options GEOM_PART_BSD
+options GEOM_PART_MBR
+options GEOM_PART_GPT
+
+# Default congestion control algorithm
+options CC_CUBIC # include CUBIC congestion control
+
+options INTRNG
+
+options NETLINK # netlink(4) support
diff --git a/sys/loongarch/conf/GENERIC b/sys/loongarch/conf/GENERIC
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/conf/GENERIC
@@ -0,0 +1,173 @@
+#
+# GENERIC -- Generic kernel configuration file for FreeBSD/LoongArch
+#
+# For more information on this file, please read the config(5) manual page,
+# and/or the handbook section on Kernel Configuration Files:
+#
+# https://docs.freebsd.org/en/books/handbook/kernelconfig/#kernelconfig-config
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files.
+# If you are in doubt as to the purpose or necessity of a line, check first
+# in NOTES.
+#
+
+cpu LoongArch
+ident GENERIC
+
+makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
+makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support
+
+options SCHED_ULE # ULE scheduler
+options PREEMPTION # Enable kernel thread preemption
+options VIMAGE # Subsystem virtualization, e.g. VNET
+options INET # InterNETworking
+options INET6 # IPv6 communications protocols
+options TCP_HHOOK # hhook(9) framework for TCP
+options IPSEC_SUPPORT # Allow kldload of ipsec and tcpmd5
+options TCP_OFFLOAD # TCP offload
+options TCP_BLACKBOX # Enhanced TCP event logging
+options TCP_RFC7413 # TCP Fast Open
+options SCTP_SUPPORT # Allow kldload of SCTP
+options FFS # Berkeley Fast Filesystem
+options SOFTUPDATES # Enable FFS soft updates support
+options UFS_ACL # Support for access control lists
+options UFS_DIRHASH # Improve performance on big directories
+options UFS_GJOURNAL # Enable gjournal-based UFS journaling
+options QUOTA # Enable disk quotas for UFS
+options NFSCL # Network Filesystem Client
+options NFSD # Network Filesystem Server
+options NFSLOCKD # Network Lock Manager
+options NFS_ROOT # NFS usable as /, requires NFSCL
+options MSDOSFS # MSDOS Filesystem
+options CD9660 # ISO 9660 Filesystem
+options PROCFS # Process filesystem (requires PSEUDOFS)
+options PSEUDOFS # Pseudo-filesystem framework
+options TMPFS # Efficient memory filesystem
+options GEOM_RAID # Soft RAID functionality.
+options GEOM_LABEL # Provides labelization
+options COMPAT_FREEBSD12 # Compatible with FreeBSD12
+options COMPAT_FREEBSD13 # Compatible with FreeBSD13
+options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI
+options KTRACE # ktrace(1) support
+options STACK # stack(9) support
+options SYSVSHM # SYSV-style shared memory
+options SYSVMSG # SYSV-style message queues
+options SYSVSEM # SYSV-style semaphores
+options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
+options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed.
+options KBD_INSTALL_CDEV # install a CDEV entry in /dev
+# options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4)
+options AUDIT # Security event auditing
+options CAPABILITY_MODE # Capsicum capability mode
+options CAPABILITIES # Capsicum capabilities
+options MAC # TrustedBSD MAC Framework
+options DDB_CTF # Kernel ELF linker loads CTF data
+options RACCT # Resource accounting framework
+options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default
+options RCTL # Resource limits
+options SMP
+
+
+options FDT
+
+# pseudo devices
+device clk
+device hwreset
+device syscon
+#device syscon_power # disabled: crashes during panic due to TLB invalidation race
+device loongarch_syscon
+
+# Bus drivers
+device pci
+
+# Block devices
+device scbus
+device da
+
+# VirtIO support
+device virtio # Generic VirtIO bus (required)
+device virtio_pci # VirtIO PCI device
+device vtnet # VirtIO Ethernet device
+device virtio_blk # VirtIO Block device
+device virtio_scsi # VirtIO SCSI device
+
+# ATA/SCSI peripherals
+device pass # Passthrough device (direct ATA/SCSI access)
+
+# NVM Express (NVMe) support
+device nvme # base NVMe driver
+options NVME_USE_NVD=0 # prefer the cam(4) based nda(4) driver
+device nvd # expose NVMe namespaces as disks, depends on nvme
+
+# USB support
+options USB_DEBUG # enable debug msgs
+device ohci # OHCI USB interface
+device ehci # EHCI USB interface (USB 2.0)
+device xhci # XHCI USB interface (USB 3.0)
+device usb # USB Bus (required)
+device ukbd # Keyboard
+device ums # Mouse
+device umass # Disks/Mass storage - Requires scbus and da
+
+# HID support
+options HID_DEBUG # enable debug msgs
+device hid # Generic HID support
+
+# DTrace support
+# device dtrace
+# device dtrace_profile
+# device dtrace_sdt
+# device dtrace_fbt
+# device dtrace_systrace
+# device dtrace_prototype
+# device dtraceall
+
+# Serial (COM) ports
+device uart # Generic UART driver
+device uart_ns8250 # ns8250-type UART driver
+
+# Console
+device vt
+device vt_efifb
+device kbdmux
+
+# Power management controllers
+# device da9063_pmic # Dialog Semiconductor DA9063 PMIC
+
+# Uncomment for memory disk
+# options MD_ROOT
+# options MD_ROOT_SIZE=32768 # 32MB ram disk
+# makeoptions MFS_IMAGE=/path/to/img
+# options ROOTDEVNAME=\"ufs:/dev/md0\"
+
+# Uncomment for virtio block device
+# options ROOTDEVNAME=\"ufs:/dev/vtbd0\"
+
+# Debugging support. Always need this:
+options KDB # Enable kernel debugger support.
+options KDB_TRACE # Print a stack trace for a panic.
+
+# Kernel dump features.
+options ZSTDIO # zstd-compressed kernel and user dumps
+
+# Pseudo devices.
+device crypto # core crypto support
+device loop # Network loopback
+device ether # Ethernet support
+device vlan # 802.1Q VLAN support
+device tuntap # Packet tunnel.
+device md # Memory "disks"
+device gif # IPv6 and IPv4 tunneling
+device firmware # firmware assist module
+
+# The `bpf' device enables the Berkeley Packet Filter.
+# Be aware of the administrative consequences of enabling this!
+# Note that 'bpf' is required for DHCP.
+device bpf # Berkeley packet filter
+
diff --git a/sys/loongarch/conf/QEMU b/sys/loongarch/conf/QEMU
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/conf/QEMU
@@ -0,0 +1,16 @@
+
+#NO_UNIVERSE
+
+include "GENERIC"
+
+ident QEMU
+
+options HZ=100
+options ROOTDEVNAME=\"ufs:/dev/vtbd0p2\"
+options BOOTVERBOSE=1
+options KTR
+options INVARIANTS
+options INVARIANT_SUPPORT
+
+#options VERBOSE_SYSINIT=1
+
diff --git a/sys/loongarch/include/_bus.h b/sys/loongarch/include/_bus.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/_bus.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 2005 M. Warner Losh <imp@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions, and the following disclaimer,
+ * without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE__BUS_H_
+#define _MACHINE__BUS_H_
+
+/*
+ * Addresses (in bus space).
+ */
+typedef u_long bus_addr_t;
+typedef u_long bus_size_t;
+
+/*
+ * Access methods for bus space.
+ */
+typedef u_long bus_space_handle_t;
+typedef struct bus_space *bus_space_tag_t;
+
+#endif /* !_MACHINE__BUS_H_ */
diff --git a/sys/loongarch/include/_inttypes.h b/sys/loongarch/include/_inttypes.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/_inttypes.h
@@ -0,0 +1,257 @@
+/*-
+ * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Klaus Klein.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * From: $NetBSD: int_fmtio.h,v 1.4 2008/04/28 20:23:36 martin Exp $
+ */
+
+#ifndef _MACHINE__INTTYPES_H_
+#define _MACHINE__INTTYPES_H_
+
+/*
+ * Macros for format specifiers.
+ */
+
+/* fprintf(3) macros for signed integers. */
+
+#define PRId8 "d" /* int8_t */
+#define PRId16 "d" /* int16_t */
+#define PRId32 "d" /* int32_t */
+#define PRId64 "ld" /* int64_t */
+#define PRIdLEAST8 "d" /* int_least8_t */
+#define PRIdLEAST16 "d" /* int_least16_t */
+#define PRIdLEAST32 "d" /* int_least32_t */
+#define PRIdLEAST64 "ld" /* int_least64_t */
+#define PRIdFAST8 "d" /* int_fast8_t */
+#define PRIdFAST16 "d" /* int_fast16_t */
+#define PRIdFAST32 "d" /* int_fast32_t */
+#define PRIdFAST64 "ld" /* int_fast64_t */
+#define PRIdMAX "jd" /* intmax_t */
+#define PRIdPTR "ld" /* intptr_t */
+
+#define PRIi8 "i" /* int8_t */
+#define PRIi16 "i" /* int16_t */
+#define PRIi32 "i" /* int32_t */
+#define PRIi64 "li" /* int64_t */
+#define PRIiLEAST8 "i" /* int_least8_t */
+#define PRIiLEAST16 "i" /* int_least16_t */
+#define PRIiLEAST32 "i" /* int_least32_t */
+#define PRIiLEAST64 "li" /* int_least64_t */
+#define PRIiFAST8 "i" /* int_fast8_t */
+#define PRIiFAST16 "i" /* int_fast16_t */
+#define PRIiFAST32 "i" /* int_fast32_t */
+#define PRIiFAST64 "li" /* int_fast64_t */
+#define PRIiMAX "ji" /* intmax_t */
+#define PRIiPTR "li" /* intptr_t */
+
+/* fprintf(3) macros for unsigned integers. */
+
+#define PRIo8 "o" /* uint8_t */
+#define PRIo16 "o" /* uint16_t */
+#define PRIo32 "o" /* uint32_t */
+#define PRIo64 "lo" /* uint64_t */
+#define PRIoLEAST8 "o" /* uint_least8_t */
+#define PRIoLEAST16 "o" /* uint_least16_t */
+#define PRIoLEAST32 "o" /* uint_least32_t */
+#define PRIoLEAST64 "lo" /* uint_least64_t */
+#define PRIoFAST8 "o" /* uint_fast8_t */
+#define PRIoFAST16 "o" /* uint_fast16_t */
+#define PRIoFAST32 "o" /* uint_fast32_t */
+#define PRIoFAST64 "lo" /* uint_fast64_t */
+#define PRIoMAX "jo" /* uintmax_t */
+#define PRIoPTR "lo" /* uintptr_t */
+
+#define PRIu8 "u" /* uint8_t */
+#define PRIu16 "u" /* uint16_t */
+#define PRIu32 "u" /* uint32_t */
+#define PRIu64 "lu" /* uint64_t */
+#define PRIuLEAST8 "u" /* uint_least8_t */
+#define PRIuLEAST16 "u" /* uint_least16_t */
+#define PRIuLEAST32 "u" /* uint_least32_t */
+#define PRIuLEAST64 "lu" /* uint_least64_t */
+#define PRIuFAST8 "u" /* uint_fast8_t */
+#define PRIuFAST16 "u" /* uint_fast16_t */
+#define PRIuFAST32 "u" /* uint_fast32_t */
+#define PRIuFAST64 "lu" /* uint_fast64_t */
+#define PRIuMAX "ju" /* uintmax_t */
+#define PRIuPTR "lu" /* uintptr_t */
+
+#define PRIx8 "x" /* uint8_t */
+#define PRIx16 "x" /* uint16_t */
+#define PRIx32 "x" /* uint32_t */
+#define PRIx64 "lx" /* uint64_t */
+#define PRIxLEAST8 "x" /* uint_least8_t */
+#define PRIxLEAST16 "x" /* uint_least16_t */
+#define PRIxLEAST32 "x" /* uint_least32_t */
+#define PRIxLEAST64 "lx" /* uint_least64_t */
+#define PRIxFAST8 "x" /* uint_fast8_t */
+#define PRIxFAST16 "x" /* uint_fast16_t */
+#define PRIxFAST32 "x" /* uint_fast32_t */
+#define PRIxFAST64 "lx" /* uint_fast64_t */
+#define PRIxMAX "jx" /* uintmax_t */
+#define PRIxPTR "lx" /* uintptr_t */
+
+#define PRIX8 "X" /* uint8_t */
+#define PRIX16 "X" /* uint16_t */
+#define PRIX32 "X" /* uint32_t */
+#define PRIX64 "lX" /* uint64_t */
+#define PRIXLEAST8 "X" /* uint_least8_t */
+#define PRIXLEAST16 "X" /* uint_least16_t */
+#define PRIXLEAST32 "X" /* uint_least32_t */
+#define PRIXLEAST64 "lX" /* uint_least64_t */
+#define PRIXFAST8 "X" /* uint_fast8_t */
+#define PRIXFAST16 "X" /* uint_fast16_t */
+#define PRIXFAST32 "X" /* uint_fast32_t */
+#define PRIXFAST64 "lX" /* uint_fast64_t */
+#define PRIXMAX "jX" /* uintmax_t */
+#define PRIXPTR "lX" /* uintptr_t */
+
+#define PRIb8 "b" /* uint8_t */
+#define PRIb16 "b" /* uint16_t */
+#define PRIb32 "b" /* uint32_t */
+#define PRIb64 "lb" /* uint64_t */
+#define PRIbLEAST8 "b" /* uint_least8_t */
+#define PRIbLEAST16 "b" /* uint_least16_t */
+#define PRIbLEAST32 "b" /* uint_least32_t */
+#define PRIbLEAST64 "lb" /* uint_least64_t */
+#define PRIbFAST8 "b" /* uint_fast8_t */
+#define PRIbFAST16 "b" /* uint_fast16_t */
+#define PRIbFAST32 "b" /* uint_fast32_t */
+#define PRIbFAST64 "lb" /* uint_fast64_t */
+#define PRIbMAX "jb" /* uintmax_t */
+#define PRIbPTR "lb" /* uintptr_t */
+
+#define PRIB8 "B" /* uint8_t */
+#define PRIB16 "B" /* uint16_t */
+#define PRIB32 "B" /* uint32_t */
+#define PRIB64 "lB" /* uint64_t */
+#define PRIBLEAST8 "B" /* uint_least8_t */
+#define PRIBLEAST16 "B" /* uint_least16_t */
+#define PRIBLEAST32 "B" /* uint_least32_t */
+#define PRIBLEAST64 "lB" /* uint_least64_t */
+#define PRIBFAST8 "B" /* uint_fast8_t */
+#define PRIBFAST16 "B" /* uint_fast16_t */
+#define PRIBFAST32 "B" /* uint_fast32_t */
+#define PRIBFAST64 "lB" /* uint_fast64_t */
+#define PRIBMAX "jB" /* uintmax_t */
+#define PRIBPTR "lB" /* uintptr_t */
+
+/* fscanf(3) macros for signed integers. */
+
+#define SCNd8 "hhd" /* int8_t */
+#define SCNd16 "hd" /* int16_t */
+#define SCNd32 "d" /* int32_t */
+#define SCNd64 "ld" /* int64_t */
+#define SCNdLEAST8 "hhd" /* int_least8_t */
+#define SCNdLEAST16 "hd" /* int_least16_t */
+#define SCNdLEAST32 "d" /* int_least32_t */
+#define SCNdLEAST64 "ld" /* int_least64_t */
+#define SCNdFAST8 "d" /* int_fast8_t */
+#define SCNdFAST16 "d" /* int_fast16_t */
+#define SCNdFAST32 "d" /* int_fast32_t */
+#define SCNdFAST64 "ld" /* int_fast64_t */
+#define SCNdMAX "jd" /* intmax_t */
+#define SCNdPTR "ld" /* intptr_t */
+
+#define SCNi8 "hhi" /* int8_t */
+#define SCNi16 "hi" /* int16_t */
+#define SCNi32 "i" /* int32_t */
+#define SCNi64 "li" /* int64_t */
+#define SCNiLEAST8 "hhi" /* int_least8_t */
+#define SCNiLEAST16 "hi" /* int_least16_t */
+#define SCNiLEAST32 "i" /* int_least32_t */
+#define SCNiLEAST64 "li" /* int_least64_t */
+#define SCNiFAST8 "i" /* int_fast8_t */
+#define SCNiFAST16 "i" /* int_fast16_t */
+#define SCNiFAST32 "i" /* int_fast32_t */
+#define SCNiFAST64 "li" /* int_fast64_t */
+#define SCNiMAX "ji" /* intmax_t */
+#define SCNiPTR "li" /* intptr_t */
+
+/* fscanf(3) macros for unsigned integers. */
+
+#define SCNo8 "hho" /* uint8_t */
+#define SCNo16 "ho" /* uint16_t */
+#define SCNo32 "o" /* uint32_t */
+#define SCNo64 "lo" /* uint64_t */
+#define SCNoLEAST8 "hho" /* uint_least8_t */
+#define SCNoLEAST16 "ho" /* uint_least16_t */
+#define SCNoLEAST32 "o" /* uint_least32_t */
+#define SCNoLEAST64 "lo" /* uint_least64_t */
+#define SCNoFAST8 "o" /* uint_fast8_t */
+#define SCNoFAST16 "o" /* uint_fast16_t */
+#define SCNoFAST32 "o" /* uint_fast32_t */
+#define SCNoFAST64 "lo" /* uint_fast64_t */
+#define SCNoMAX "jo" /* uintmax_t */
+#define SCNoPTR "lo" /* uintptr_t */
+
+#define SCNu8 "hhu" /* uint8_t */
+#define SCNu16 "hu" /* uint16_t */
+#define SCNu32 "u" /* uint32_t */
+#define SCNu64 "lu" /* uint64_t */
+#define SCNuLEAST8 "hhu" /* uint_least8_t */
+#define SCNuLEAST16 "hu" /* uint_least16_t */
+#define SCNuLEAST32 "u" /* uint_least32_t */
+#define SCNuLEAST64 "lu" /* uint_least64_t */
+#define SCNuFAST8 "u" /* uint_fast8_t */
+#define SCNuFAST16 "u" /* uint_fast16_t */
+#define SCNuFAST32 "u" /* uint_fast32_t */
+#define SCNuFAST64 "lu" /* uint_fast64_t */
+#define SCNuMAX "ju" /* uintmax_t */
+#define SCNuPTR "lu" /* uintptr_t */
+
+#define SCNx8 "hhx" /* uint8_t */
+#define SCNx16 "hx" /* uint16_t */
+#define SCNx32 "x" /* uint32_t */
+#define SCNx64 "lx" /* uint64_t */
+#define SCNxLEAST8 "hhx" /* uint_least8_t */
+#define SCNxLEAST16 "hx" /* uint_least16_t */
+#define SCNxLEAST32 "x" /* uint_least32_t */
+#define SCNxLEAST64 "lx" /* uint_least64_t */
+#define SCNxFAST8 "x" /* uint_fast8_t */
+#define SCNxFAST16 "x" /* uint_fast16_t */
+#define SCNxFAST32 "x" /* uint_fast32_t */
+#define SCNxFAST64 "lx" /* uint_fast64_t */
+#define SCNxMAX "jx" /* uintmax_t */
+#define SCNxPTR "lx" /* uintptr_t */
+
+#define SCNb8 "hhb" /* uint8_t */
+#define SCNb16 "hb" /* uint16_t */
+#define SCNb32 "b" /* uint32_t */
+#define SCNb64 "lb" /* uint64_t */
+#define SCNbLEAST8 "hhb" /* uint_least8_t */
+#define SCNbLEAST16 "hb" /* uint_least16_t */
+#define SCNbLEAST32 "b" /* uint_least32_t */
+#define SCNbLEAST64 "lb" /* uint_least64_t */
+#define SCNbFAST8 "b" /* uint_fast8_t */
+#define SCNbFAST16 "b" /* uint_fast16_t */
+#define SCNbFAST32 "b" /* uint_fast32_t */
+#define SCNbFAST64 "lb" /* uint_fast64_t */
+#define SCNbMAX "jb" /* uintmax_t */
+#define SCNbPTR "lb" /* uintptr_t */
+
+#endif /* !_MACHINE__INTTYPES_H_ */
diff --git a/sys/loongarch/include/_limits.h b/sys/loongarch/include/_limits.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/_limits.h
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 1988, 1993
+ * The Regents of the University of California. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)limits.h 8.3 (Berkeley) 1/4/94
+ */
+
+#ifndef _MACHINE__LIMITS_H_
+#define _MACHINE__LIMITS_H_
+
+/*
+ * According to ANSI (section 2.2.4.2), the values below must be usable by
+ * #if preprocessing directives. Additionally, the expression must have the
+ * same type as would an expression that is an object of the corresponding
+ * type converted according to the integral promotions. The subtraction for
+ * INT_MIN, etc., is so the value is not unsigned; e.g., 0x80000000 is an
+ * unsigned int for 32-bit two's complement ANSI compilers (section 3.1.3.2).
+ */
+
+#define __CHAR_BIT 8 /* number of bits in a char */
+#define __SHRT_BIT 16 /* number of bits in a short */
+#define __INT_BIT 32 /* number of bits in an int */
+#define __LLONG_BIT 64 /* number of bits in a long long */
+
+#define __SCHAR_MAX 0x7f /* max value for a signed char */
+#define __SCHAR_MIN (-0x7f - 1) /* min value for a signed char */
+
+#define __UCHAR_MAX 0xff /* max value for an unsigned char */
+
+#define __USHRT_MAX 0xffff /* max value for an unsigned short */
+#define __SHRT_MAX 0x7fff /* max value for a short */
+#define __SHRT_MIN (-0x7fff - 1) /* min value for a short */
+
+#define __UINT_MAX 0xffffffff /* max value for an unsigned int */
+#define __INT_MAX 0x7fffffff /* max value for an int */
+#define __INT_MIN (-0x7fffffff - 1) /* min value for an int */
+
+#define __ULONG_MAX 0xffffffffffffffffUL /* max for an unsigned long */
+#define __LONG_MAX 0x7fffffffffffffffL /* max for a long */
+#define __LONG_MIN (-0x7fffffffffffffffL - 1) /* min for a long */
+
+/* Long longs have the same size but not the same type as longs. */
+ /* max for an unsigned long long */
+#define __ULLONG_MAX 0xffffffffffffffffULL
+#define __LLONG_MAX 0x7fffffffffffffffLL /* max for a long long */
+#define __LLONG_MIN (-0x7fffffffffffffffLL - 1) /* min for a long long */
+
+#define __SSIZE_MAX __LONG_MAX /* max value for a ssize_t */
+
+#define __SIZE_T_MAX __ULONG_MAX /* max value for a size_t */
+
+#define __OFF_MAX __LONG_MAX /* max value for an off_t */
+#define __OFF_MIN __LONG_MIN /* min value for an off_t */
+
+/* Quads and longs are the same size. Ensure they stay in sync. */
+#define __UQUAD_MAX (__ULONG_MAX) /* max value for a uquad_t */
+#define __QUAD_MAX (__LONG_MAX) /* max value for a quad_t */
+#define __QUAD_MIN (__LONG_MIN) /* min value for a quad_t */
+
+#define __LONG_BIT 64
+#define __WORD_BIT 32
+
+/* Minimum signal stack size. */
+#define __MINSIGSTKSZ (1024 * 4)
+
+#endif /* !_MACHINE__LIMITS_H_ */
diff --git a/sys/loongarch/include/_stdint.h b/sys/loongarch/include/_stdint.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/_stdint.h
@@ -0,0 +1,170 @@
+/*-
+ * Copyright (c) 2001, 2002 Mike Barcroft <mike@FreeBSD.org>
+ * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Klaus Klein.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE__STDINT_H_
+#define _MACHINE__STDINT_H_
+
+#define INT8_C(c) (c)
+#define INT16_C(c) (c)
+#define INT32_C(c) (c)
+#define INT64_C(c) (c ## L)
+
+#define UINT8_C(c) (c)
+#define UINT16_C(c) (c)
+#define UINT32_C(c) (c ## U)
+#define UINT64_C(c) (c ## UL)
+
+#define INTMAX_C(c) INT64_C(c)
+#define UINTMAX_C(c) UINT64_C(c)
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.1 Limits of exact-width integer types
+ */
+/* Minimum values of exact-width signed integer types. */
+#define INT8_MIN (-0x7f-1)
+#define INT16_MIN (-0x7fff-1)
+#define INT32_MIN (-0x7fffffff-1)
+#define INT64_MIN (-0x7fffffffffffffffL-1)
+
+/* Maximum values of exact-width signed integer types. */
+#define INT8_MAX 0x7f
+#define INT16_MAX 0x7fff
+#define INT32_MAX 0x7fffffff
+#define INT64_MAX 0x7fffffffffffffffL
+
+/* Maximum values of exact-width unsigned integer types. */
+#define UINT8_MAX 0xff
+#define UINT16_MAX 0xffff
+#define UINT32_MAX 0xffffffffU
+#define UINT64_MAX 0xffffffffffffffffUL
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.2 Limits of minimum-width integer types
+ */
+/* Minimum values of minimum-width signed integer types. */
+#define INT_LEAST8_MIN INT8_MIN
+#define INT_LEAST16_MIN INT16_MIN
+#define INT_LEAST32_MIN INT32_MIN
+#define INT_LEAST64_MIN INT64_MIN
+
+/* Maximum values of minimum-width signed integer types. */
+#define INT_LEAST8_MAX INT8_MAX
+#define INT_LEAST16_MAX INT16_MAX
+#define INT_LEAST32_MAX INT32_MAX
+#define INT_LEAST64_MAX INT64_MAX
+
+/* Maximum values of minimum-width unsigned integer types. */
+#define UINT_LEAST8_MAX UINT8_MAX
+#define UINT_LEAST16_MAX UINT16_MAX
+#define UINT_LEAST32_MAX UINT32_MAX
+#define UINT_LEAST64_MAX UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.3 Limits of fastest minimum-width integer types
+ */
+/* Minimum values of fastest minimum-width signed integer types. */
+#define INT_FAST8_MIN INT32_MIN
+#define INT_FAST16_MIN INT32_MIN
+#define INT_FAST32_MIN INT32_MIN
+#define INT_FAST64_MIN INT64_MIN
+
+/* Maximum values of fastest minimum-width signed integer types. */
+#define INT_FAST8_MAX INT32_MAX
+#define INT_FAST16_MAX INT32_MAX
+#define INT_FAST32_MAX INT32_MAX
+#define INT_FAST64_MAX INT64_MAX
+
+/* Maximum values of fastest minimum-width unsigned integer types. */
+#define UINT_FAST8_MAX UINT32_MAX
+#define UINT_FAST16_MAX UINT32_MAX
+#define UINT_FAST32_MAX UINT32_MAX
+#define UINT_FAST64_MAX UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.4 Limits of integer types capable of holding object pointers
+ */
+#define INTPTR_MIN INT64_MIN
+#define INTPTR_MAX INT64_MAX
+#define UINTPTR_MAX UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.2.5 Limits of greatest-width integer types
+ */
+#define INTMAX_MIN INT64_MIN
+#define INTMAX_MAX INT64_MAX
+#define UINTMAX_MAX UINT64_MAX
+
+/*
+ * ISO/IEC 9899:1999
+ * 7.18.3 Limits of other integer types
+ */
+/* Limits of ptrdiff_t. */
+#define PTRDIFF_MIN INT64_MIN
+#define PTRDIFF_MAX INT64_MAX
+
+/* Limits of sig_atomic_t. */
+#define SIG_ATOMIC_MIN INT64_MIN
+#define SIG_ATOMIC_MAX INT64_MAX
+
+/* Limit of size_t. */
+#define SIZE_MAX UINT64_MAX
+
+/* Limits of wint_t. */
+#define WINT_MIN INT32_MIN
+#define WINT_MAX INT32_MAX
+
+#if __ISO_C_VISIBLE >= 2023
+/*
+ * ISO/IEC 9899:2023
+ * 7.22.2 Widths of specified-width integer types
+ */
+#define INT_FAST8_WIDTH INT32_WIDTH
+#define INT_FAST16_WIDTH INT32_WIDTH
+#define INT_FAST32_WIDTH INT32_WIDTH
+#define INT_FAST64_WIDTH INT64_WIDTH
+#define INTPTR_WIDTH INT64_WIDTH
+#define INTMAX_WIDTH INT64_WIDTH
+
+/*
+ * ISO/IEC 9899:2023
+ * 7.22.3 Width of other integer types
+ */
+#define PTRDIFF_WIDTH INT64_WIDTH
+#define SIG_ATOMIC_WIDTH INT64_WIDTH
+#define SIZE_WIDTH INT64_WIDTH
+#define WINT_WIDTH INT32_WIDTH
+#endif /* __ISO_C_VISIBLE >= 2023 */
+
+#endif /* !_MACHINE__STDINT_H_ */
diff --git a/sys/loongarch/include/_types.h b/sys/loongarch/include/_types.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/_types.h
@@ -0,0 +1,69 @@
+/*-
+ * Copyright (c) 2002 Mike Barcroft <mike@FreeBSD.org>
+ * Copyright (c) 1990, 1993
+ * The Regents of the University of California. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * From: @(#)ansi.h 8.2 (Berkeley) 1/4/94
+ * From: @(#)types.h 8.3 (Berkeley) 1/5/94
+ */
+
+#ifndef _MACHINE__TYPES_H_
+#define _MACHINE__TYPES_H_
+
+#ifndef _SYS__TYPES_H_
+#error do not include this header, use sys/_types.h
+#endif
+
+/*
+ * Standard type definitions.
+ */
+typedef __int32_t __clock_t; /* clock()... */
+typedef __int64_t __critical_t;
+#ifndef _STANDALONE
+typedef double __double_t;
+typedef float __float_t;
+#endif
+typedef __int32_t __int_fast8_t;
+typedef __int32_t __int_fast16_t;
+typedef __int32_t __int_fast32_t;
+typedef __int64_t __int_fast64_t;
+typedef __int64_t __register_t;
+typedef __int64_t __segsz_t; /* segment size (in pages) */
+typedef __int64_t __time_t; /* time()... */
+#define __SIZEOF_TIME_T __SIZEOF_INT64_T
+typedef __uint32_t __uint_fast8_t;
+typedef __uint32_t __uint_fast16_t;
+typedef __uint32_t __uint_fast32_t;
+typedef __uint64_t __uint_fast64_t;
+typedef __uint64_t __u_register_t;
+typedef __uint64_t __vm_paddr_t;
+typedef int ___wchar_t;
+
+#define __WCHAR_MIN __INT_MIN /* min value for a wchar_t */
+#define __WCHAR_MAX __INT_MAX /* max value for a wchar_t */
+
+#endif /* !_MACHINE__TYPES_H_ */
diff --git a/sys/loongarch/include/asm.h b/sys/loongarch/include/asm.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/asm.h
@@ -0,0 +1,61 @@
+/*-
+ * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_ASM_H_
+#define _MACHINE_ASM_H_
+
+#undef __FBSDID
+#if !defined(lint) && !defined(STRIP_FBSDID)
+#define __FBSDID(s) .ident s
+#else
+#define __FBSDID(s) /* nothing */
+#endif /* not lint and not STRIP_FBSDID */
+
+#define _C_LABEL(x) x
+
+#define ENTRY(sym) \
+ .text; .globl sym; .type sym,@function; .align 4; sym: .cfi_startproc;
+#define END(sym) .cfi_endproc; .size sym, . - sym
+
+#define EENTRY(sym) \
+ .globl sym; sym:
+#define EEND(sym)
+
+#define WEAK_REFERENCE(sym, alias) \
+ .weak alias; \
+ .set alias,sym
+
+#define SET_FAULT_HANDLER(handler, tmp) \
+ ld.d tmp, $r21, PC_CURTHREAD; \
+ ld.d tmp, tmp, TD_PCB; /* Load the pcb */ \
+ st.d handler, tmp, PCB_ONFAULT /* Set the handler */
+
+/* do nothing */
+#define ENTER_USER_ACCESS(tmp)
+
+#define EXIT_USER_ACCESS(tmp)
+
+#endif /* _MACHINE_ASM_H_ */
diff --git a/sys/loongarch/include/atomic.h b/sys/loongarch/include/atomic.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/atomic.h
@@ -0,0 +1,711 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_ATOMIC_H_
+#define _MACHINE_ATOMIC_H_
+
+#include <sys/atomic_common.h>
+
+/*
+ * Hint encoding:
+ *
+ * Bit4: ordering or completion (0: completion, 1: ordering)
+ * Bit3: barrier for previous read (0: true, 1: false)
+ * Bit2: barrier for previous write (0: true, 1: false)
+ * Bit1: barrier for succeeding read (0: true, 1: false)
+ * Bit0: barrier for succeeding write (0: true, 1: false)
+ *
+ * Hint 0x700: barrier for "read after read" from the same address
+ */
+
+#define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory")
+
+#define crwrw 0b00000
+#define cr_r_ 0b00101
+#define c_w_w 0b01010
+
+#define orwrw 0b10000
+#define or_r_ 0b10101
+#define o_w_w 0b11010
+
+#define orw_w 0b10010
+#define or_rw 0b10100
+
+#define c_sync() DBAR(crwrw)
+#define c_rsync() DBAR(cr_r_)
+#define c_wsync() DBAR(c_w_w)
+
+#define o_sync() DBAR(orwrw)
+#define o_rsync() DBAR(or_r_)
+#define o_wsync() DBAR(o_w_w)
+
+#define ldacq_mb() DBAR(or_rw)
+#define strel_mb() DBAR(orw_w)
+
+#define mb() c_sync()
+#define rmb() c_rsync()
+#define wmb() c_wsync()
+#define iob() c_sync()
+#define wbflush() c_sync()
+
+#define __smp_mb() o_sync()
+#define __smp_rmb() o_rsync()
+#define __smp_wmb() o_wsync()
+
+#ifdef SMP
+#define __WEAK_LLSC_MB " dbar 0x700 \n"
+#else
+#define __WEAK_LLSC_MB " \n"
+#endif
+
+#define fence() c_sync()
+
+static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t);
+static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t);
+static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t);
+static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *,
+ uint16_t);
+
+#define ATOMIC_ACQ_REL(NAME, WIDTH) \
+static __inline void \
+atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
+{ \
+ atomic_##NAME##_##WIDTH(p, v); \
+ ldacq_mb(); \
+} \
+ \
+static __inline void \
+atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
+{ \
+ strel_mb(); \
+ atomic_##NAME##_##WIDTH(p, v); \
+}
+
+#define ATOMIC_CMPSET_ACQ_REL(WIDTH) \
+static __inline int \
+atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
+ uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
+{ \
+ int retval; \
+ \
+ retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \
+ ldacq_mb(); \
+ return (retval); \
+} \
+ \
+static __inline int \
+atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
+ uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
+{ \
+ strel_mb(); \
+ return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \
+}
+
+#define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \
+static __inline int \
+atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
+ uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
+{ \
+ int retval; \
+ \
+ retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \
+ ldacq_mb(); \
+ return (retval); \
+} \
+ \
+static __inline int \
+atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
+ uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
+{ \
+ strel_mb(); \
+ return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \
+}
+
+ATOMIC_CMPSET_ACQ_REL(8);
+ATOMIC_FCMPSET_ACQ_REL(8);
+ATOMIC_CMPSET_ACQ_REL(16);
+ATOMIC_FCMPSET_ACQ_REL(16);
+
+#define atomic_cmpset_char atomic_cmpset_8
+#define atomic_cmpset_acq_char atomic_cmpset_acq_8
+#define atomic_cmpset_rel_char atomic_cmpset_rel_8
+#define atomic_fcmpset_char atomic_fcmpset_8
+#define atomic_fcmpset_acq_char atomic_fcmpset_acq_8
+#define atomic_fcmpset_rel_char atomic_fcmpset_rel_8
+
+#define atomic_cmpset_short atomic_cmpset_16
+#define atomic_cmpset_acq_short atomic_cmpset_acq_16
+#define atomic_cmpset_rel_short atomic_cmpset_rel_16
+#define atomic_fcmpset_short atomic_fcmpset_16
+#define atomic_fcmpset_acq_short atomic_fcmpset_acq_16
+#define atomic_fcmpset_rel_short atomic_fcmpset_rel_16
+
+static __inline void
+atomic_add_32(volatile uint32_t *p, uint32_t val)
+{
+
+ __asm __volatile("amadd.w $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+}
+
+static __inline void
+atomic_subtract_32(volatile uint32_t *p, uint32_t val)
+{
+
+ __asm __volatile("amadd.w $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (-val)
+ : "memory");
+}
+
+static __inline void
+atomic_set_32(volatile uint32_t *p, uint32_t val)
+{
+
+ __asm __volatile("amor.w $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+}
+
+static __inline void
+atomic_clear_32(volatile uint32_t *p, uint32_t val)
+{
+
+ __asm __volatile("amand.w $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (~val)
+ : "memory");
+}
+
+static __inline int
+atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
+{
+ uint32_t tmp;
+ int res;
+
+ __asm __volatile(
+ "li.w %1, 0\n" /* Preset to fail */
+ "0:"
+ "ll.w %0, %2\n" /* tmp = *p */
+ "bne %0, %z3, 1f\n" /* cmpval == *p ? */
+ "move %1, %z4\n"
+ "sc.w %1, %2\n" /* *p = newval */
+ "beqz %1, 0b\n" /* loop */
+ "1:"
+ __WEAK_LLSC_MB
+ : "=&r" (tmp), "=&r" (res), "+ZB" (*p)
+ : "rJ" ((long)(int32_t)cmpval), "rJ" (newval)
+ : "memory");
+
+ return (res);
+}
+
+static __inline int
+atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
+{
+ uint32_t tmp;
+ int res;
+
+ __asm __volatile(
+ "li.w %1, 0\n" /* Preset to fail */
+ "0:"
+ "ll.w %0, %2\n" /* Load old value */
+ "bne %0, %z4, 1f\n" /* Compare */
+ "move %1, %z5\n" /* newval to res */
+ "sc.w %1, %2\n" /* Try to store new value */
+ "beqz %1, 0b\n" /* Retry on sc failure */
+ "b 2f\n"
+ "1:"
+ "st.w %0, %3\n" /* Save old value */
+ "2:"
+ __WEAK_LLSC_MB
+ : "=&r" (tmp), "=&r" (res), "+ZB" (*p), "+ZB" (*cmpval)
+ : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval)
+ : "memory");
+
+ return (res);
+}
+
+static __inline uint32_t
+atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
+{
+ uint32_t ret;
+
+ __asm __volatile("amadd.w %0, %2, %1"
+ : "=&r" (ret), "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+
+ return (ret);
+}
+
+static __inline uint32_t
+atomic_readandclear_32(volatile uint32_t *p)
+{
+ uint32_t ret;
+ uint32_t val;
+
+ val = 0;
+
+ __asm __volatile("amswap.w %0, %2, %1"
+ : "=&r"(ret), "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+
+ return (ret);
+}
+
+static __inline int
+atomic_testandset_32(volatile uint32_t *p, u_int val)
+{
+ uint32_t mask, old;
+
+ mask = 1u << (val & 31);
+ __asm __volatile("amor.w %0, %2, %1"
+ : "=&r"(old), "+ZB" (*p)
+ : "r" (mask)
+ : "memory");
+
+ return ((old & mask) != 0);
+}
+
+static __inline int
+atomic_testandclear_32(volatile uint32_t *p, u_int val)
+{
+ uint32_t mask, old;
+
+ mask = 1u << (val & 31);
+ __asm __volatile("amand.w %0, %2, %1"
+ : "=&r"(old), "+ZB" (*p)
+ : "r" (~mask)
+ : "memory");
+
+ return ((old & mask) != 0);
+}
+
+#define atomic_add_int atomic_add_32
+#define atomic_clear_int atomic_clear_32
+#define atomic_cmpset_int atomic_cmpset_32
+#define atomic_fcmpset_int atomic_fcmpset_32
+#define atomic_fetchadd_int atomic_fetchadd_32
+#define atomic_readandclear_int atomic_readandclear_32
+#define atomic_set_int atomic_set_32
+#define atomic_subtract_int atomic_subtract_32
+#define atomic_testandclear_int atomic_testandclear_32
+#define atomic_testandset_int atomic_testandset_32
+
+ATOMIC_ACQ_REL(set, 32)
+ATOMIC_ACQ_REL(clear, 32)
+ATOMIC_ACQ_REL(add, 32)
+ATOMIC_ACQ_REL(subtract, 32)
+
+ATOMIC_CMPSET_ACQ_REL(32);
+ATOMIC_FCMPSET_ACQ_REL(32);
+
+static __inline uint32_t
+atomic_load_acq_32(const volatile uint32_t *p)
+{
+ uint32_t ret;
+
+ ret = *p;
+
+ ldacq_mb();
+
+ return (ret);
+}
+
+static __inline void
+atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
+{
+
+ strel_mb();
+
+ *p = val;
+}
+
+#define atomic_add_acq_int atomic_add_acq_32
+#define atomic_clear_acq_int atomic_clear_acq_32
+#define atomic_cmpset_acq_int atomic_cmpset_acq_32
+#define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
+#define atomic_load_acq_int atomic_load_acq_32
+#define atomic_set_acq_int atomic_set_acq_32
+#define atomic_subtract_acq_int atomic_subtract_acq_32
+
+#define atomic_add_rel_int atomic_add_rel_32
+#define atomic_clear_rel_int atomic_clear_rel_32
+#define atomic_cmpset_rel_int atomic_cmpset_rel_32
+#define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
+#define atomic_set_rel_int atomic_set_rel_32
+#define atomic_subtract_rel_int atomic_subtract_rel_32
+#define atomic_store_rel_int atomic_store_rel_32
+
+static __inline void
+atomic_add_64(volatile uint64_t *p, uint64_t val)
+{
+
+ __asm __volatile("amadd.d $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+}
+
+static __inline void
+atomic_subtract_64(volatile uint64_t *p, uint64_t val)
+{
+
+ __asm __volatile("amadd.d $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (-val)
+ : "memory");
+}
+
+static __inline void
+atomic_set_64(volatile uint64_t *p, uint64_t val)
+{
+
+ __asm __volatile("amor.d $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+}
+
+static __inline void
+atomic_clear_64(volatile uint64_t *p, uint64_t val)
+{
+
+ __asm __volatile("amand.d $zero, %1, %0"
+ : "+ZB" (*p)
+ : "r" (~val)
+ : "memory");
+}
+
+static __inline int
+atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
+{
+ uint64_t tmp;
+ uint64_t res;
+
+ __asm __volatile(
+ "li.d %1, 0\n" /* Preset to fail */
+ "0:"
+ "ll.d %0, %2\n" /* tmp = *p */
+ "bne %0, %z3, 1f\n" /* *p == cmpval */
+ "move %1, %z4\n"
+ "sc.d %1, %2\n" /* *p = newval */
+ "beqz %1, 0b\n"
+ "1:"
+ __WEAK_LLSC_MB
+ : "=&r" (tmp), "=&r" (res), "+ZB" (*p)
+ : "rJ" (cmpval), "rJ" (newval)
+ : "memory");
+
+ return (res);
+}
+
+static __inline int
+atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
+{
+ uint64_t tmp;
+ uint64_t res;
+
+ __asm __volatile(
+ "li.d %1, 0\n" /* Preset to fail */
+ "0:"
+ "ll.d %0, %2\n" /* Load old value */
+ "bne %0, %z4, 1f\n" /* Compare */
+ "move %1, %z5\n"
+ "sc.d %1, %2\n" /* Try to store new value */
+ "beqz %1, 0b\n" /* Retry on sc failure */
+ "b 2f\n"
+ "1:"
+ "st.d %0, %3\n" /* Save old value */
+ "2:"
+ __WEAK_LLSC_MB
+ : "=&r" (tmp), "=&r" (res), "+ZB" (*p), "+ZB" (*cmpval)
+ : "rJ" (*cmpval), "rJ" (newval)
+ : "memory");
+
+ return (res);
+}
+
+static __inline uint64_t
+atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
+{
+ uint64_t ret;
+
+ __asm __volatile("amadd.d %0, %2, %1"
+ : "=&r" (ret), "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+
+ return (ret);
+}
+
+static __inline uint64_t
+atomic_readandclear_64(volatile uint64_t *p)
+{
+ uint64_t ret;
+ uint64_t val;
+
+ val = 0;
+
+ __asm __volatile("amswap.d %0, %2, %1"
+ : "=&r"(ret), "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+
+ return (ret);
+}
+
+static __inline uint32_t
+atomic_swap_32(volatile uint32_t *p, uint32_t val)
+{
+ uint32_t old;
+
+ __asm __volatile("amswap.w %0, %2, %1"
+ : "=&r"(old), "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+
+ return (old);
+}
+
+static __inline uint64_t
+atomic_swap_64(volatile uint64_t *p, uint64_t val)
+{
+ uint64_t old;
+
+ __asm __volatile("amswap.d %0, %2, %1"
+ : "=&r"(old), "+ZB" (*p)
+ : "r" (val)
+ : "memory");
+
+ return (old);
+}
+
+static __inline int
+atomic_testandset_64(volatile uint64_t *p, u_int val)
+{
+ uint64_t mask, old;
+
+ mask = 1ul << (val & 63);
+ __asm __volatile("amor.d %0, %2, %1"
+ : "=&r"(old), "+ZB" (*p)
+ : "r" (mask)
+ : "memory");
+
+ return ((old & mask) != 0);
+}
+
+static __inline int
+atomic_testandclear_64(volatile uint64_t *p, u_int val)
+{
+ uint64_t mask, old;
+
+ mask = 1ul << (val & 63);
+ __asm __volatile("amand.d %0, %2, %1"
+ : "=&r"(old), "+ZB" (*p)
+ : "r" (~mask)
+ : "memory");
+
+ return ((old & mask) != 0);
+}
+
+#define atomic_swap_int atomic_swap_32
+
+#define atomic_add_long atomic_add_64
+#define atomic_clear_long atomic_clear_64
+#define atomic_cmpset_long atomic_cmpset_64
+#define atomic_fcmpset_long atomic_fcmpset_64
+#define atomic_fetchadd_long atomic_fetchadd_64
+#define atomic_readandclear_long atomic_readandclear_64
+#define atomic_set_long atomic_set_64
+#define atomic_subtract_long atomic_subtract_64
+#define atomic_swap_long atomic_swap_64
+#define atomic_testandclear_long atomic_testandclear_64
+#define atomic_testandset_long atomic_testandset_64
+
+#define atomic_add_ptr atomic_add_64
+#define atomic_clear_ptr atomic_clear_64
+#define atomic_cmpset_ptr atomic_cmpset_64
+#define atomic_fcmpset_ptr atomic_fcmpset_64
+#define atomic_fetchadd_ptr atomic_fetchadd_64
+#define atomic_readandclear_ptr atomic_readandclear_64
+#define atomic_set_ptr atomic_set_64
+#define atomic_subtract_ptr atomic_subtract_64
+#define atomic_swap_ptr atomic_swap_64
+#define atomic_testandclear_ptr atomic_testandclear_64
+#define atomic_testandset_ptr atomic_testandset_64
+
+ATOMIC_ACQ_REL(set, 64)
+ATOMIC_ACQ_REL(clear, 64)
+ATOMIC_ACQ_REL(add, 64)
+ATOMIC_ACQ_REL(subtract, 64)
+
+ATOMIC_CMPSET_ACQ_REL(64);
+ATOMIC_FCMPSET_ACQ_REL(64);
+
+static __inline int
+atomic_testandset_acq_64(volatile uint64_t *p, u_int val)
+{
+ int ret;
+
+ ret = atomic_testandset_64(p, val);
+ ldacq_mb();
+
+ return (ret);
+}
+
+static __inline int
+atomic_testandclear_acq_64(volatile uint64_t *p, u_int val)
+{
+ int ret;
+
+ ret = atomic_testandclear_64(p, val);
+ ldacq_mb();
+
+ return (ret);
+}
+
+static __inline uint64_t
+atomic_load_acq_64(const volatile uint64_t *p)
+{
+ uint64_t ret;
+
+ ret = *p;
+
+ ldacq_mb();
+
+ return (ret);
+}
+
+static __inline void
+atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
+{
+
+ strel_mb();
+
+ *p = val;
+}
+
+static __inline int
+atomic_testandset_rel_64(volatile uint64_t *p, u_int val)
+{
+ int ret;
+
+ strel_mb();
+ ret = atomic_testandset_64(p, val);
+
+ return (ret);
+}
+
+static __inline int
+atomic_testandclear_rel_64(volatile uint64_t *p, u_int val)
+{
+ int ret;
+
+ strel_mb();
+ ret = atomic_testandclear_64(p, val);
+
+ return (ret);
+}
+
+#define atomic_add_acq_long atomic_add_acq_64
+#define atomic_clear_acq_long atomic_clear_acq_64
+#define atomic_cmpset_acq_long atomic_cmpset_acq_64
+#define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
+#define atomic_load_acq_long atomic_load_acq_64
+#define atomic_set_acq_long atomic_set_acq_64
+#define atomic_subtract_acq_long atomic_subtract_acq_64
+#define atomic_testandclear_acq_long atomic_testandclear_acq_64
+#define atomic_testandset_acq_long atomic_testandset_acq_64
+
+#define atomic_add_acq_ptr atomic_add_acq_64
+#define atomic_clear_acq_ptr atomic_clear_acq_64
+#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
+#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
+#define atomic_load_acq_ptr atomic_load_acq_64
+#define atomic_set_acq_ptr atomic_set_acq_64
+#define atomic_subtract_acq_ptr atomic_subtract_acq_64
+#define atomic_testandclear_acq_ptr atomic_testandclear_acq_64
+#define atomic_testandset_acq_ptr atomic_testandset_acq_64
+
+#undef ATOMIC_ACQ_REL
+
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ ldacq_mb();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+ strel_mb();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+ __smp_mb();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ mb();
+}
+
+#define atomic_add_rel_long atomic_add_rel_64
+#define atomic_clear_rel_long atomic_clear_rel_64
+#define atomic_cmpset_rel_long atomic_cmpset_rel_64
+#define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
+#define atomic_set_rel_long atomic_set_rel_64
+#define atomic_subtract_rel_long atomic_subtract_rel_64
+#define atomic_store_rel_long atomic_store_rel_64
+#define atomic_testandclear_rel_long atomic_testandclear_rel_64
+#define atomic_testandset_rel_long atomic_testandset_rel_64
+
+#define atomic_add_rel_ptr atomic_add_rel_64
+#define atomic_clear_rel_ptr atomic_clear_rel_64
+#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
+#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
+#define atomic_set_rel_ptr atomic_set_rel_64
+#define atomic_subtract_rel_ptr atomic_subtract_rel_64
+#define atomic_store_rel_ptr atomic_store_rel_64
+#define atomic_testandclear_rel_ptr atomic_testandclear_rel_64
+#define atomic_testandset_rel_ptr atomic_testandset_rel_64
+
+#include <sys/_atomic_subword.h>
+
+#endif /* _MACHINE_ATOMIC_H_ */
diff --git a/sys/loongarch/include/bus.h b/sys/loongarch/include/bus.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/bus.h
@@ -0,0 +1,492 @@
+/* $NetBSD: bus.h,v 1.11 2003/07/28 17:35:54 thorpej Exp $ */
+
+/*-
+ * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*-
+ * Copyright (c) 1996 Charles M. Hannum. All rights reserved.
+ * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by Christopher G. Demetriou
+ * for the NetBSD Project.
+ * 4. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * From: sys/arm/include/bus.h
+ */
+
+#ifndef _MACHINE_BUS_H_
+#define _MACHINE_BUS_H_
+
+#include <machine/_bus.h>
+
+#define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t)
+
+#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFFUL
+#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFFUL
+#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFFUL
+#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFFUL
+
+#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFUL
+#define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFUL
+
+#define BUS_SPACE_MAP_CACHEABLE 0x01
+#define BUS_SPACE_MAP_LINEAR 0x02
+#define BUS_SPACE_MAP_PREFETCHABLE 0x04
+
+#define BUS_SPACE_UNRESTRICTED (~0)
+
+#define BUS_SPACE_BARRIER_READ 0x01
+#define BUS_SPACE_BARRIER_WRITE 0x02
+
+struct bus_space {
+ /* cookie */
+ void *bs_cookie;
+
+ /* mapping/unmapping */
+ int (*bs_map) (void *, bus_addr_t, bus_size_t,
+ int, bus_space_handle_t *);
+ void (*bs_unmap) (void *, bus_space_handle_t,
+ bus_size_t);
+ int (*bs_subregion) (void *, bus_space_handle_t,
+ bus_size_t, bus_size_t, bus_space_handle_t *);
+
+ /* allocation/deallocation */
+ int (*bs_alloc) (void *, bus_addr_t, bus_addr_t,
+ bus_size_t, bus_size_t, bus_size_t, int,
+ bus_addr_t *, bus_space_handle_t *);
+ void (*bs_free) (void *, bus_space_handle_t,
+ bus_size_t);
+
+ /* get kernel virtual address */
+ /* barrier */
+ void (*bs_barrier) (void *, bus_space_handle_t,
+ bus_size_t, bus_size_t, int);
+
+ /* read single */
+ u_int8_t (*bs_r_1) (void *, bus_space_handle_t, bus_size_t);
+ u_int16_t (*bs_r_2) (void *, bus_space_handle_t, bus_size_t);
+ u_int32_t (*bs_r_4) (void *, bus_space_handle_t, bus_size_t);
+ u_int64_t (*bs_r_8) (void *, bus_space_handle_t, bus_size_t);
+
+ /* read multiple */
+ void (*bs_rm_1) (void *, bus_space_handle_t, bus_size_t,
+ u_int8_t *, bus_size_t);
+ void (*bs_rm_2) (void *, bus_space_handle_t, bus_size_t,
+ u_int16_t *, bus_size_t);
+ void (*bs_rm_4) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t *, bus_size_t);
+ void (*bs_rm_8) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t *, bus_size_t);
+
+ /* read region */
+ void (*bs_rr_1) (void *, bus_space_handle_t,
+ bus_size_t, u_int8_t *, bus_size_t);
+ void (*bs_rr_2) (void *, bus_space_handle_t,
+ bus_size_t, u_int16_t *, bus_size_t);
+ void (*bs_rr_4) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t *, bus_size_t);
+ void (*bs_rr_8) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t *, bus_size_t);
+
+ /* write single */
+ void (*bs_w_1) (void *, bus_space_handle_t,
+ bus_size_t, u_int8_t);
+ void (*bs_w_2) (void *, bus_space_handle_t,
+ bus_size_t, u_int16_t);
+ void (*bs_w_4) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t);
+ void (*bs_w_8) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t);
+
+ /* write multiple */
+ void (*bs_wm_1) (void *, bus_space_handle_t,
+ bus_size_t, const u_int8_t *, bus_size_t);
+ void (*bs_wm_2) (void *, bus_space_handle_t,
+ bus_size_t, const u_int16_t *, bus_size_t);
+ void (*bs_wm_4) (void *, bus_space_handle_t,
+ bus_size_t, const u_int32_t *, bus_size_t);
+ void (*bs_wm_8) (void *, bus_space_handle_t,
+ bus_size_t, const u_int64_t *, bus_size_t);
+
+ /* write region */
+ void (*bs_wr_1) (void *, bus_space_handle_t,
+ bus_size_t, const u_int8_t *, bus_size_t);
+ void (*bs_wr_2) (void *, bus_space_handle_t,
+ bus_size_t, const u_int16_t *, bus_size_t);
+ void (*bs_wr_4) (void *, bus_space_handle_t,
+ bus_size_t, const u_int32_t *, bus_size_t);
+ void (*bs_wr_8) (void *, bus_space_handle_t,
+ bus_size_t, const u_int64_t *, bus_size_t);
+
+ /* set multiple */
+ void (*bs_sm_1) (void *, bus_space_handle_t,
+ bus_size_t, u_int8_t, bus_size_t);
+ void (*bs_sm_2) (void *, bus_space_handle_t,
+ bus_size_t, u_int16_t, bus_size_t);
+ void (*bs_sm_4) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t, bus_size_t);
+ void (*bs_sm_8) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t, bus_size_t);
+
+ /* set region */
+ void (*bs_sr_1) (void *, bus_space_handle_t,
+ bus_size_t, u_int8_t, bus_size_t);
+ void (*bs_sr_2) (void *, bus_space_handle_t,
+ bus_size_t, u_int16_t, bus_size_t);
+ void (*bs_sr_4) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t, bus_size_t);
+ void (*bs_sr_8) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t, bus_size_t);
+
+ /* copy */
+ void (*bs_c_1) (void *, bus_space_handle_t, bus_size_t,
+ bus_space_handle_t, bus_size_t, bus_size_t);
+ void (*bs_c_2) (void *, bus_space_handle_t, bus_size_t,
+ bus_space_handle_t, bus_size_t, bus_size_t);
+ void (*bs_c_4) (void *, bus_space_handle_t, bus_size_t,
+ bus_space_handle_t, bus_size_t, bus_size_t);
+ void (*bs_c_8) (void *, bus_space_handle_t, bus_size_t,
+ bus_space_handle_t, bus_size_t, bus_size_t);
+
+ /* read single stream */
+ u_int8_t (*bs_r_1_s) (void *, bus_space_handle_t, bus_size_t);
+ u_int16_t (*bs_r_2_s) (void *, bus_space_handle_t, bus_size_t);
+ u_int32_t (*bs_r_4_s) (void *, bus_space_handle_t, bus_size_t);
+ u_int64_t (*bs_r_8_s) (void *, bus_space_handle_t, bus_size_t);
+
+ /* read multiple stream */
+ void (*bs_rm_1_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int8_t *, bus_size_t);
+ void (*bs_rm_2_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int16_t *, bus_size_t);
+ void (*bs_rm_4_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t *, bus_size_t);
+ void (*bs_rm_8_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t *, bus_size_t);
+
+ /* read region stream */
+ void (*bs_rr_1_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int8_t *, bus_size_t);
+ void (*bs_rr_2_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int16_t *, bus_size_t);
+ void (*bs_rr_4_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t *, bus_size_t);
+ void (*bs_rr_8_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t *, bus_size_t);
+
+ /* write single stream */
+ void (*bs_w_1_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int8_t);
+ void (*bs_w_2_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int16_t);
+ void (*bs_w_4_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int32_t);
+ void (*bs_w_8_s) (void *, bus_space_handle_t,
+ bus_size_t, u_int64_t);
+
+ /* write multiple stream */
+ void (*bs_wm_1_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int8_t *, bus_size_t);
+ void (*bs_wm_2_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int16_t *, bus_size_t);
+ void (*bs_wm_4_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int32_t *, bus_size_t);
+ void (*bs_wm_8_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int64_t *, bus_size_t);
+
+ /* write region stream */
+ void (*bs_wr_1_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int8_t *, bus_size_t);
+ void (*bs_wr_2_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int16_t *, bus_size_t);
+ void (*bs_wr_4_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int32_t *, bus_size_t);
+ void (*bs_wr_8_s) (void *, bus_space_handle_t,
+ bus_size_t, const u_int64_t *, bus_size_t);
+};
+
+/*
+ * Utility macros; INTERNAL USE ONLY.
+ */
+#define __bs_c(a,b) __CONCAT(a,b)
+#define __bs_opname(op,size) __bs_c(__bs_c(__bs_c(bs_,op),_),size)
+
+#define __bs_rs(sz, t, h, o) \
+ (*(t)->__bs_opname(r,sz))((t)->bs_cookie, h, o)
+#define __bs_ws(sz, t, h, o, v) \
+ (*(t)->__bs_opname(w,sz))((t)->bs_cookie, h, o, v)
+#define __bs_nonsingle(type, sz, t, h, o, a, c) \
+ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c)
+#define __bs_set(type, sz, t, h, o, v, c) \
+ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, v, c)
+#define __bs_copy(sz, t, h1, o1, h2, o2, cnt) \
+ (*(t)->__bs_opname(c,sz))((t)->bs_cookie, h1, o1, h2, o2, cnt)
+
+#define __bs_opname_s(op,size) \
+ __bs_c(__bs_c(__bs_c(__bs_c(bs_,op),_),size),_s)
+#define __bs_rs_s(sz, t, h, o) \
+ (*(t)->__bs_opname_s(r,sz))((t)->bs_cookie, h, o)
+#define __bs_ws_s(sz, t, h, o, v) \
+ (*(t)->__bs_opname_s(w,sz))((t)->bs_cookie, h, o, v)
+#define __bs_nonsingle_s(type, sz, t, h, o, a, c) \
+ (*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c)
+
+/*
+ * Mapping and unmapping operations.
+ */
+#define bus_space_map(t, a, s, c, hp) \
+ (*(t)->bs_map)((t)->bs_cookie, (a), (s), (c), (hp))
+#define bus_space_unmap(t, h, s) \
+ (*(t)->bs_unmap)((t)->bs_cookie, (h), (s))
+#define bus_space_subregion(t, h, o, s, hp) \
+ (*(t)->bs_subregion)((t)->bs_cookie, (h), (o), (s), (hp))
+
+/*
+ * Allocation and deallocation operations.
+ */
+#define bus_space_alloc(t, rs, re, s, a, b, c, ap, hp) \
+ (*(t)->bs_alloc)((t)->bs_cookie, (rs), (re), (s), (a), (b), \
+ (c), (ap), (hp))
+#define bus_space_free(t, h, s) \
+ (*(t)->bs_free)((t)->bs_cookie, (h), (s))
+
+/*
+ * Bus barrier operations.
+ */
+#define bus_space_barrier(t, h, o, l, f) \
+ (*(t)->bs_barrier)((t)->bs_cookie, (h), (o), (l), (f))
+
+/*
+ * Bus read (single) operations.
+ */
+#define bus_space_read_1(t, h, o) __bs_rs(1,(t),(h),(o))
+#define bus_space_read_2(t, h, o) __bs_rs(2,(t),(h),(o))
+#define bus_space_read_4(t, h, o) __bs_rs(4,(t),(h),(o))
+#define bus_space_read_8(t, h, o) __bs_rs(8,(t),(h),(o))
+
+#define bus_space_read_stream_1(t, h, o) \
+ __bs_rs_s(1,(t), (h), (o))
+#define bus_space_read_stream_2(t, h, o) \
+ __bs_rs_s(2,(t), (h), (o))
+#define bus_space_read_stream_4(t, h, o) \
+ __bs_rs_s(4,(t), (h), (o))
+#define bus_space_read_stream_8(t, h, o) \
+ __bs_rs_s(8,(t), (h), (o))
+
+/*
+ * Bus read multiple operations.
+ */
+#define bus_space_read_multi_1(t, h, o, a, c) \
+ __bs_nonsingle(rm,1,(t),(h),(o),(a),(c))
+#define bus_space_read_multi_2(t, h, o, a, c) \
+ __bs_nonsingle(rm,2,(t),(h),(o),(a),(c))
+#define bus_space_read_multi_4(t, h, o, a, c) \
+ __bs_nonsingle(rm,4,(t),(h),(o),(a),(c))
+#define bus_space_read_multi_8(t, h, o, a, c) \
+ __bs_nonsingle(rm,8,(t),(h),(o),(a),(c))
+
+#define bus_space_read_multi_stream_1(t, h, o, a, c) \
+ __bs_nonsingle_s(rm,1,(t),(h),(o),(a),(c))
+#define bus_space_read_multi_stream_2(t, h, o, a, c) \
+ __bs_nonsingle_s(rm,2,(t),(h),(o),(a),(c))
+#define bus_space_read_multi_stream_4(t, h, o, a, c) \
+ __bs_nonsingle_s(rm,4,(t),(h),(o),(a),(c))
+#define bus_space_read_multi_stream_8(t, h, o, a, c) \
+ __bs_nonsingle_s(rm,8,(t),(h),(o),(a),(c))
+
+/*
+ * Bus read region operations.
+ */
+#define bus_space_read_region_1(t, h, o, a, c) \
+ __bs_nonsingle(rr,1,(t),(h),(o),(a),(c))
+#define bus_space_read_region_2(t, h, o, a, c) \
+ __bs_nonsingle(rr,2,(t),(h),(o),(a),(c))
+#define bus_space_read_region_4(t, h, o, a, c) \
+ __bs_nonsingle(rr,4,(t),(h),(o),(a),(c))
+#define bus_space_read_region_8(t, h, o, a, c) \
+ __bs_nonsingle(rr,8,(t),(h),(o),(a),(c))
+
+#define bus_space_read_region_stream_1(t, h, o, a, c) \
+ __bs_nonsingle_s(rr,1,(t),(h),(o),(a),(c))
+#define bus_space_read_region_stream_2(t, h, o, a, c) \
+ __bs_nonsingle_s(rr,2,(t),(h),(o),(a),(c))
+#define bus_space_read_region_stream_4(t, h, o, a, c) \
+ __bs_nonsingle_s(rr,4,(t),(h),(o),(a),(c))
+#define bus_space_read_region_stream_8(t, h, o, a, c) \
+ __bs_nonsingle_s(rr,8,(t),(h),(o),(a),(c))
+
+/*
+ * Bus write (single) operations.
+ */
+#define bus_space_write_1(t, h, o, v) __bs_ws(1,(t),(h),(o),(v))
+#define bus_space_write_2(t, h, o, v) __bs_ws(2,(t),(h),(o),(v))
+#define bus_space_write_4(t, h, o, v) __bs_ws(4,(t),(h),(o),(v))
+#define bus_space_write_8(t, h, o, v) __bs_ws(8,(t),(h),(o),(v))
+
+#define bus_space_write_stream_1(t, h, o, v) \
+ __bs_ws_s(1,(t),(h),(o),(v))
+#define bus_space_write_stream_2(t, h, o, v) \
+ __bs_ws_s(2,(t),(h),(o),(v))
+#define bus_space_write_stream_4(t, h, o, v) \
+ __bs_ws_s(4,(t),(h),(o),(v))
+#define bus_space_write_stream_8(t, h, o, v) \
+ __bs_ws_s(8,(t),(h),(o),(v))
+
+/*
+ * Bus write multiple operations.
+ */
+#define bus_space_write_multi_1(t, h, o, a, c) \
+ __bs_nonsingle(wm,1,(t),(h),(o),(a),(c))
+#define bus_space_write_multi_2(t, h, o, a, c) \
+ __bs_nonsingle(wm,2,(t),(h),(o),(a),(c))
+#define bus_space_write_multi_4(t, h, o, a, c) \
+ __bs_nonsingle(wm,4,(t),(h),(o),(a),(c))
+#define bus_space_write_multi_8(t, h, o, a, c) \
+ __bs_nonsingle(wm,8,(t),(h),(o),(a),(c))
+
+#define bus_space_write_multi_stream_1(t, h, o, a, c) \
+ __bs_nonsingle_s(wm,1,(t),(h),(o),(a),(c))
+#define bus_space_write_multi_stream_2(t, h, o, a, c) \
+ __bs_nonsingle_s(wm,2,(t),(h),(o),(a),(c))
+#define bus_space_write_multi_stream_4(t, h, o, a, c) \
+ __bs_nonsingle_s(wm,4,(t),(h),(o),(a),(c))
+#define bus_space_write_multi_stream_8(t, h, o, a, c) \
+ __bs_nonsingle_s(wm,8,(t),(h),(o),(a),(c))
+
+/*
+ * Bus write region operations.
+ */
+#define bus_space_write_region_1(t, h, o, a, c) \
+ __bs_nonsingle(wr,1,(t),(h),(o),(a),(c))
+#define bus_space_write_region_2(t, h, o, a, c) \
+ __bs_nonsingle(wr,2,(t),(h),(o),(a),(c))
+#define bus_space_write_region_4(t, h, o, a, c) \
+ __bs_nonsingle(wr,4,(t),(h),(o),(a),(c))
+#define bus_space_write_region_8(t, h, o, a, c) \
+ __bs_nonsingle(wr,8,(t),(h),(o),(a),(c))
+
+#define bus_space_write_region_stream_1(t, h, o, a, c) \
+ __bs_nonsingle_s(wr,1,(t),(h),(o),(a),(c))
+#define bus_space_write_region_stream_2(t, h, o, a, c) \
+ __bs_nonsingle_s(wr,2,(t),(h),(o),(a),(c))
+#define bus_space_write_region_stream_4(t, h, o, a, c) \
+ __bs_nonsingle_s(wr,4,(t),(h),(o),(a),(c))
+#define bus_space_write_region_stream_8(t, h, o, a, c) \
+ __bs_nonsingle_s(wr,8,(t),(h),(o),(a),(c))
+
+/*
+ * Set multiple operations.
+ */
+#define bus_space_set_multi_1(t, h, o, v, c) \
+ __bs_set(sm,1,(t),(h),(o),(v),(c))
+#define bus_space_set_multi_2(t, h, o, v, c) \
+ __bs_set(sm,2,(t),(h),(o),(v),(c))
+#define bus_space_set_multi_4(t, h, o, v, c) \
+ __bs_set(sm,4,(t),(h),(o),(v),(c))
+#define bus_space_set_multi_8(t, h, o, v, c) \
+ __bs_set(sm,8,(t),(h),(o),(v),(c))
+
+/*
+ * Set region operations.
+ */
+#define bus_space_set_region_1(t, h, o, v, c) \
+ __bs_set(sr,1,(t),(h),(o),(v),(c))
+#define bus_space_set_region_2(t, h, o, v, c) \
+ __bs_set(sr,2,(t),(h),(o),(v),(c))
+#define bus_space_set_region_4(t, h, o, v, c) \
+ __bs_set(sr,4,(t),(h),(o),(v),(c))
+#define bus_space_set_region_8(t, h, o, v, c) \
+ __bs_set(sr,8,(t),(h),(o),(v),(c))
+
+/*
+ * Copy operations.
+ */
+#define bus_space_copy_region_1(t, h1, o1, h2, o2, c) \
+ __bs_copy(1, t, h1, o1, h2, o2, c)
+#define bus_space_copy_region_2(t, h1, o1, h2, o2, c) \
+ __bs_copy(2, t, h1, o1, h2, o2, c)
+#define bus_space_copy_region_4(t, h1, o1, h2, o2, c) \
+ __bs_copy(4, t, h1, o1, h2, o2, c)
+#define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \
+ __bs_copy(8, t, h1, o1, h2, o2, c)
+
+#define BUS_PEEK_FUNC(width, type) \
+ static inline int \
+ bus_space_peek_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset, type *value) \
+ { \
+ type tmp; \
+ tmp = bus_space_read_##width(tag, hnd, offset); \
+ *value = (type)tmp; \
+ return (0); \
+ }
+BUS_PEEK_FUNC(1, uint8_t)
+BUS_PEEK_FUNC(2, uint16_t)
+BUS_PEEK_FUNC(4, uint32_t)
+BUS_PEEK_FUNC(8, uint64_t)
+
+#define BUS_POKE_FUNC(width, type) \
+ static inline int \
+ bus_space_poke_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset, type value) \
+ { \
+ bus_space_write_##width(tag, hnd, offset, value); \
+ return (0); \
+ }
+BUS_POKE_FUNC(1, uint8_t)
+BUS_POKE_FUNC(2, uint16_t)
+BUS_POKE_FUNC(4, uint32_t)
+BUS_POKE_FUNC(8, uint64_t)
+
+#include <machine/bus_dma.h>
+
+#endif /* _MACHINE_BUS_H_ */
diff --git a/sys/loongarch/include/bus_dma.h b/sys/loongarch/include/bus_dma.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/bus_dma.h
@@ -0,0 +1,140 @@
+
+#ifndef _MACHINE_BUS_DMA_H_
+#define _MACHINE_BUS_DMA_H_
+
+#define WANT_INLINE_DMAMAP
+#include <sys/bus_dma.h>
+
+#include <machine/bus_dma_impl.h>
+
+/*
+ * Allocate a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static inline int
+bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->map_create(dmat, flags, mapp));
+}
+
+/*
+ * Destroy a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static inline int
+bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->map_destroy(dmat, map));
+}
+
+/*
+ * Allocate a piece of memory that can be efficiently mapped into
+ * bus device space based on the constraints listed in the dma tag.
+ * A dmamap to for use with dmamap_load is also allocated.
+ */
+static inline int
+bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
+ bus_dmamap_t *mapp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->mem_alloc(dmat, vaddr, flags, mapp));
+}
+
+/*
+ * Free a piece of memory and it's allociated dmamap, that was allocated
+ * via bus_dmamem_alloc. Make the same choice for free/contigfree.
+ */
+static inline void
+bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->mem_free(dmat, vaddr, map);
+}
+
+/*
+ * Release the mapping held by map.
+ */
+static inline void
+bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->map_unload(dmat, map);
+}
+
+static inline void
+bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->map_sync(dmat, map, op);
+}
+
+static inline int
+_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
+ bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->load_phys(dmat, map, buf, buflen, flags, segs,
+ segp));
+}
+
+static inline int
+_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma,
+ bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs,
+ int *segp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->load_ma(dmat, map, ma, tlen, ma_offs, flags,
+ segs, segp));
+}
+
+static inline int
+_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
+ bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,
+ int *segp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,
+ segp));
+}
+
+static inline void
+_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->map_waitok(dmat, map, mem, callback, callback_arg);
+}
+
+static inline bus_dma_segment_t *
+_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dma_segment_t *segs, int nsegs, int error)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->map_complete(dmat, map, segs, nsegs, error));
+}
+
+#endif /* !_MACHINE_BUS_DMA_H_ */
diff --git a/sys/loongarch/include/bus_dma_impl.h b/sys/loongarch/include/bus_dma_impl.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/bus_dma_impl.h
@@ -0,0 +1,88 @@
+/*-
+ * Copyright (c) 2013 The FreeBSD Foundation
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_BUS_DMA_IMPL_H_
+#define _MACHINE_BUS_DMA_IMPL_H_
+
+struct bus_dma_tag_common {
+ struct bus_dma_impl *impl;
+ bus_size_t alignment;
+ bus_addr_t boundary;
+ bus_addr_t lowaddr;
+ bus_addr_t highaddr;
+ bus_size_t maxsize;
+ u_int nsegments;
+ bus_size_t maxsegsz;
+ int flags;
+ bus_dma_lock_t *lockfunc;
+ void *lockfuncarg;
+ int domain;
+};
+
+struct bus_dma_impl {
+ int (*tag_create)(bus_dma_tag_t parent,
+ bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr,
+ bus_addr_t highaddr, bus_size_t maxsize, int nsegments,
+ bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
+ void *lockfuncarg, bus_dma_tag_t *dmat);
+ int (*tag_destroy)(bus_dma_tag_t dmat);
+ int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp);
+ int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map);
+ int (*tag_set_domain)(bus_dma_tag_t);
+ bool (*id_mapped)(bus_dma_tag_t, vm_paddr_t, bus_size_t);
+ int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags,
+ bus_dmamap_t *mapp);
+ void (*mem_free)(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map);
+ int (*load_ma)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
+ bus_dma_segment_t *segs, int *segp);
+ int (*load_phys)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ vm_paddr_t buf, bus_size_t buflen, int flags,
+ bus_dma_segment_t *segs, int *segp);
+ int (*load_buffer)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ void *buf, bus_size_t buflen, struct pmap *pmap, int flags,
+ bus_dma_segment_t *segs, int *segp);
+ void (*map_waitok)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct memdesc *mem, bus_dmamap_callback_t *callback,
+ void *callback_arg);
+ bus_dma_segment_t *(*map_complete)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dma_segment_t *segs, int nsegs, int error);
+ void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map);
+ void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dmasync_op_t op);
+};
+
+int common_bus_dma_tag_create(struct bus_dma_tag_common *parent,
+ bus_size_t alignment,
+ bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
+ bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags,
+ bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat);
+
+extern struct bus_dma_impl bus_dma_bounce_impl;
+
+#endif
diff --git a/sys/loongarch/include/clock.h b/sys/loongarch/include/clock.h
new file mode 100644
diff --git a/sys/loongarch/include/counter.h b/sys/loongarch/include/counter.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/counter.h
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_COUNTER_H_
+#define _MACHINE_COUNTER_H_
+
+#include <sys/pcpu.h>
+#include <machine/atomic.h>
+
+#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter
+
+#define counter_enter() do {} while (0)
+#define counter_exit() do {} while (0)
+
+#ifdef IN_SUBR_COUNTER_C
+static inline uint64_t
+counter_u64_read_one(uint64_t *p, int cpu)
+{
+
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ALLOC_SIZE * cpu));
+}
+
+static inline uint64_t
+counter_u64_fetch_inline(uint64_t *p)
+{
+ uint64_t r;
+ int i;
+
+ r = 0;
+ CPU_FOREACH(i)
+ r += counter_u64_read_one(p, i);
+
+ return (r);
+}
+
+static void
+counter_u64_zero_one_cpu(void *arg)
+{
+
+ *(zpcpu_get((counter_u64_t *)arg)) = 0;
+}
+
+static inline void
+counter_u64_zero_inline(counter_u64_t c)
+{
+
+ smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu,
+ smp_no_rendezvous_barrier, c);
+}
+#endif
+
+#define counter_u64_add_protected(c, inc) counter_u64_add(c, inc)
+
+static inline void
+counter_u64_add(counter_u64_t c, int64_t inc)
+{
+
+ atomic_add_64((uint64_t *)zpcpu_get(c), inc);
+}
+
+#endif /* ! _MACHINE_COUNTER_H_ */
diff --git a/sys/loongarch/include/cpu.h b/sys/loongarch/include/cpu.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/cpu.h
@@ -0,0 +1,287 @@
+/*-
+ * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_CPU_H_
+#define _MACHINE_CPU_H_
+
+#include <machine/atomic.h>
+#include <machine/cpufunc.h>
+#include <machine/frame.h>
+#include <machine/param.h>
+
+/*
+ * As described in LoongArch specs from Loongson Technology, the PRID register
+ * (CPUCFG.00) has the following layout:
+ *
+ * +---------------+----------------+------------+--------------------+
+ * | Reserved | Company ID | Series ID | Product ID |
+ * +---------------+----------------+------------+--------------------+
+ * 31 24 23 16 15 12 11 0
+ */
+
+/*
+ * Assigned Company values for bits 23:16 of the PRID register.
+ */
+
+#define PRID_COMP_MASK 0xff0000
+
+#define PRID_COMP_LOONGSON 0x140000
+
+/*
+ * Assigned Series ID values for bits 15:12 of the PRID register. In order
+ * to detect a certain CPU type exactly eventually additional registers may
+ * need to be examined.
+ */
+
+#define PRID_SERIES_MASK 0xf000
+
+#define PRID_SERIES_LA132 0x8000 /* Loongson 32bit */
+#define PRID_SERIES_LA264 0xa000 /* Loongson 64bit, 2-issue */
+#define PRID_SERIES_LA364 0xb000 /* Loongson 64bit, 3-issue */
+#define PRID_SERIES_LA464 0xc000 /* Loongson 64bit, 4-issue */
+#define PRID_SERIES_LA664 0xd000 /* Loongson 64bit, 6-issue */
+
+#define TRAPF_PC(tfp) ((tfp)->tf_era)
+#define TRAPF_USERMODE(tfp) \
+ (((tfp)->tf_prmd & PLV_MASK) == PLV_USER)
+
+#define cpu_getstack(td) ((td)->td_frame->tf_sp)
+#define cpu_setstack(td, sp) ((td)->td_frame->tf_sp = (sp))
+#define cpu_spinwait() __asm__ __volatile__("": : :"memory")
+#define cpu_lock_delay() DELAY(1)
+
+/*
+ * Particular Product ID values for bits 11:0 of the PRID register.
+ */
+
+#define PRID_PRODUCT_MASK 0x0fff
+
+
+#define LOONGARCH_ARCH_MASK 0x3
+
+#if !defined(__ASSEMBLY__)
+
+enum cpu_type_enum {
+ CPU_UNKNOWN,
+ CPU_LOONGSON32,
+ CPU_LOONGSON64,
+ CPU_LAST
+};
+
+#endif /* !__ASSEMBLY */
+
+/*
+ * ISA Level encodings
+ *
+ */
+
+#define LOONGARCH_CPU_ISA_LA32R 0x00000001
+#define LOONGARCH_CPU_ISA_LA32S 0x00000002
+#define LOONGARCH_CPU_ISA_LA64 0x00000004
+
+#define LOONGARCH_CPU_ISA_32BIT \
+ (LOONGARCH_CPU_ISA_LA32R | LOONGARCH_CPU_ISA_LA32S)
+#define LOONGARCH_CPU_ISA_64BIT LOONGARCH_CPU_ISA_LA64
+
+/*
+ * CPU Option encodings
+ */
+#define CPU_FEATURE_CPUCFG 0 /* CPU has CPUCFG */
+#define CPU_FEATURE_LAM 1 /* CPU has Atomic instructions */
+#define CPU_FEATURE_UAL 2 /* CPU supports unaligned access */
+#define CPU_FEATURE_FPU 3 /* CPU has FPU */
+#define CPU_FEATURE_LSX 4 /* CPU has LSX (128-bit SIMD) */
+#define CPU_FEATURE_LASX 5 /* CPU has LASX (256-bit SIMD) */
+#define CPU_FEATURE_CRC32 6 /* CPU has CRC32 instructions */
+#define CPU_FEATURE_COMPLEX 7 /* CPU has Complex instructions */
+#define CPU_FEATURE_CRYPTO 8 /* CPU has Crypto instructions */
+#define CPU_FEATURE_LVZ 9 /* CPU has Virtualization extension */
+#define CPU_FEATURE_LBT_X86 10 /* CPU has X86 Binary Translation */
+#define CPU_FEATURE_LBT_ARM 11 /* CPU has ARM Binary Translation */
+#define CPU_FEATURE_LBT_MIPS 12 /* CPU has MIPS Binary Translation */
+#define CPU_FEATURE_TLB 13 /* CPU has TLB */
+#define CPU_FEATURE_CSR 14 /* CPU has CSR */
+#define CPU_FEATURE_WATCH 15 /* CPU has watchpoint registers */
+#define CPU_FEATURE_VINT 16 /* CPU has vectored interrupts */
+#define CPU_FEATURE_CSRIPI 17 /* CPU has CSR-IPI */
+#define CPU_FEATURE_EXTIOI 18 /* CPU has EXT-IOI */
+#define CPU_FEATURE_PREFETCH 19 /* CPU has prefetch instructions */
+#define CPU_FEATURE_PMP 20 /* CPU has perfermance counter */
+#define CPU_FEATURE_SCALEFREQ 21 /* CPU supports cpufreq scaling */
+#define CPU_FEATURE_FLATMODE 22 /* CPU has flat mode */
+#define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */
+#define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */
+#define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */
+#define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */
+
+#define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG)
+#define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM)
+#define LOONGARCH_CPU_UAL BIT_ULL(CPU_FEATURE_UAL)
+#define LOONGARCH_CPU_FPU BIT_ULL(CPU_FEATURE_FPU)
+#define LOONGARCH_CPU_LSX BIT_ULL(CPU_FEATURE_LSX)
+#define LOONGARCH_CPU_LASX BIT_ULL(CPU_FEATURE_LASX)
+#define LOONGARCH_CPU_CRC32 BIT_ULL(CPU_FEATURE_CRC32)
+#define LOONGARCH_CPU_COMPLEX BIT_ULL(CPU_FEATURE_COMPLEX)
+#define LOONGARCH_CPU_CRYPTO BIT_ULL(CPU_FEATURE_CRYPTO)
+#define LOONGARCH_CPU_LVZ BIT_ULL(CPU_FEATURE_LVZ)
+#define LOONGARCH_CPU_LBT_X86 BIT_ULL(CPU_FEATURE_LBT_X86)
+#define LOONGARCH_CPU_LBT_ARM BIT_ULL(CPU_FEATURE_LBT_ARM)
+#define LOONGARCH_CPU_LBT_MIPS BIT_ULL(CPU_FEATURE_LBT_MIPS)
+#define LOONGARCH_CPU_TLB BIT_ULL(CPU_FEATURE_TLB)
+#define LOONGARCH_CPU_CSR BIT_ULL(CPU_FEATURE_CSR)
+#define LOONGARCH_CPU_WATCH BIT_ULL(CPU_FEATURE_WATCH)
+#define LOONGARCH_CPU_VINT BIT_ULL(CPU_FEATURE_VINT)
+#define LOONGARCH_CPU_CSRIPI BIT_ULL(CPU_FEATURE_CSRIPI)
+#define LOONGARCH_CPU_EXTIOI BIT_ULL(CPU_FEATURE_EXTIOI)
+#define LOONGARCH_CPU_PREFETCH BIT_ULL(CPU_FEATURE_PREFETCH)
+#define LOONGARCH_CPU_PMP BIT_ULL(CPU_FEATURE_PMP)
+#define LOONGARCH_CPU_SCALEFREQ BIT_ULL(CPU_FEATURE_SCALEFREQ)
+#define LOONGARCH_CPU_FLATMODE BIT_ULL(CPU_FEATURE_FLATMODE)
+#define LOONGARCH_CPU_EIODECODE BIT_ULL(CPU_FEATURE_EIODECODE)
+#define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID)
+#define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR)
+#define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW)
+
+/* Flags passed in AT_HWCAP */
+#define HWCAP_LOONGARCH_CPUCFG (1 << 0)
+#define HWCAP_LOONGARCH_LAM (1 << 1)
+#define HWCAP_LOONGARCH_UAL (1 << 2)
+#define HWCAP_LOONGARCH_FPU (1 << 3)
+#define HWCAP_LOONGARCH_LSX (1 << 4)
+#define HWCAP_LOONGARCH_LASX (1 << 5)
+#define HWCAP_LOONGARCH_CRC32 (1 << 6)
+#define HWCAP_LOONGARCH_COMPLEX (1 << 7)
+#define HWCAP_LOONGARCH_CRYPTO (1 << 8)
+#define HWCAP_LOONGARCH_LVZ (1 << 9)
+#define HWCAP_LOONGARCH_LBT_X86 (1 << 10)
+#define HWCAP_LOONGARCH_LBT_ARM (1 << 11)
+#define HWCAP_LOONGARCH_LBT_MIPS (1 << 12)
+#define HWCAP_LOONGARCH_PTW (1 << 13)
+
+/* cache_desc->flags */
+enum {
+ CACHE_PRESENT = (1 << 0),
+ CACHE_PRIVATE = (1 << 1), /* core private cache */
+ CACHE_INCLUSIVE = (1 << 2), /* include the inner level caches */
+};
+
+/*
+ * Descriptor for a cache
+ */
+struct cache_desc {
+ unsigned char type;
+ unsigned char level;
+ unsigned short sets; /* Number of lines per set */
+ unsigned char ways; /* Number of ways */
+ unsigned char linesz; /* Size of line in bytes */
+ unsigned char flags; /* Flags describing cache properties */
+};
+
+#define CACHE_LEVEL_MAX 3
+#define CACHE_LEAVES_MAX 6
+
+struct cpuinfo_loongarch {
+ uint64_t asid_cache;
+ unsigned long asid_mask;
+ uint32_t arch;
+ uint32_t hwcap;
+
+ /*
+ * Capability and feature descriptor structure for LoongArch CPU
+ */
+ unsigned long long options;
+ unsigned int processor_id;
+ unsigned int fpu_vers;
+ unsigned int fpu_csr0;
+ unsigned int fpu_mask;
+ unsigned int cputype;
+ int isa_level;
+ int tlbsize;
+ int tlbsizemtlb;
+ int tlbsizestlbsets;
+ int tlbsizestlbways;
+ int cache_leaves_present;
+ /* number of cache_leaves[] elements */
+ struct cache_desc cache_leaves[CACHE_LEAVES_MAX];
+ int core; /* physical core number in package */
+ int package;/* physical package number */
+ int global_id;
+ /* physical global thread number */
+ int vabits; /* Virtual Address size in bits */
+ int pabits; /* Physical Address size in bits */
+ unsigned int ksave_mask; /* Usable KSave mask. */
+ unsigned int watch_dreg_count;
+ /* Number data breakpoints */
+ unsigned int watch_ireg_count;
+ /* Number instruction breakpoints */
+ unsigned int watch_reg_use_cnt;
+ /* min(NUM_WATCH_REGS,
+ watch_dreg_count + watch_ireg_count),
+ Usable by ptrace */
+};
+
+struct cpu_desc {
+ const char *cpu_mvendor_name;
+ const char *cpu_march_name;
+
+ struct cpuinfo_loongarch cpuinfo;
+};
+
+extern struct cpu_desc cpu_desc[MAXCPU];
+
+#define __cpucfg(reg) \
+({ \
+ uint64_t val; \
+ __asm __volatile("cpucfg %0, %1" \
+ :"=r" (val) \
+ : "r" (reg) \
+ ); \
+ val & 0xffffffff; \
+})
+
+#define read_cpucfg(reg) __cpucfg(reg)
+
+#ifdef _KERNEL
+
+extern char btext[];
+extern char etext[];
+
+void cpu_halt(void) __dead2;
+void cpu_reset(void) __dead2;
+void fork_trampoline(void);
+void identify_cpu(u_int cpu);
+void printcpuinfo(u_int cpu);
+
+static __inline uint64_t
+get_cyclecount(void)
+{
+
+ return (rdcycle());
+}
+
+#endif
+
+#endif /* !_MACHINE_CPU_H_ */
diff --git a/sys/loongarch/include/cpufunc.h b/sys/loongarch/include/cpufunc.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/cpufunc.h
@@ -0,0 +1,131 @@
+/*-
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_CPUFUNC_H_
+#define _MACHINE_CPUFUNC_H_
+
+static __inline void
+breakpoint(void)
+{
+ __asm __volatile__("break 0");
+}
+
+#ifdef _KERNEL
+
+#include <machine/loongarchreg.h>
+
+static __inline register_t
+intr_disable(void)
+{
+ /*
+ * csrxchg $rd, $rj, csr:
+ * new_csr = ($rd & $rj) | (old_csr & ~$rj)
+ * $rd = old_csr
+ * $rj is the bitmask of fields to modify.
+ * $rd provides the new values for those fields.
+ * To clear IE: $rj = CSR_CRMD_IE (modify only IE),
+ * $rd bit 2 = 0 (clear IE).
+ * Use $rd = 1 instead of 0 to avoid $r0 allocation,
+ * since bit 2 is what matters and 1 & CSR_CRMD_IE = 0.
+ * The old CRMD value is returned so IE state can be restored.
+ */
+ uint32_t val = 1;
+ uint32_t mask = CSR_CRMD_IE;
+
+ __asm__ __volatile__(
+ "csrxchg %[val], %[mask], %[reg]\n\t"
+ : [val] "+r" (val)
+ : [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD)
+ : "memory");
+
+ return (val & (CSR_CRMD_IE));
+}
+
+static __inline void
+intr_restore(register_t s)
+{
+ /*
+ * csrxchg $rd, $rj, csr:
+ * new_csr = ($rd & $rj) | (old_csr & ~$rj)
+ * $rd = old_csr
+ * $rj = CSR_CRMD_IE: modify only the IE bit.
+ * $rd = (s | 1): IE bit from s, ORed with 1 to
+ * ensure $rd is never 0 (csrxchg forbids $r0).
+ */
+ uint32_t val = ((uint32_t)s) | 1;
+ uint32_t mask = CSR_CRMD_IE;
+
+ __asm__ __volatile__(
+ "csrxchg %[val], %[mask], %[reg]\n\t"
+ : [val] "+r" (val)
+ : [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD)
+ : "memory");
+}
+
+static __inline void
+intr_enable(void)
+{
+ intr_restore(CSR_CRMD_IE);
+}
+
+static __inline void
+flush_icache(void)
+{
+ __asm __volatile("ibar 0" ::);
+}
+
+/* timer */
+static inline unsigned long rdtime(void)
+{
+ unsigned long val = 0;
+
+ __asm __volatile(
+ "rdtime.d %0, $zero\n\t"
+ : "=r"(val)
+ :
+ );
+ return val;
+}
+
+#define rdcycle() rdtime()
+
+extern int64_t dcache_line_size;
+extern int64_t icache_line_size;
+
+#define cpu_dcache_wbinv_range(a, s)
+#define cpu_dcache_inv_range(a, s)
+#define cpu_dcache_wb_range(a, s)
+
+#define cpu_idcache_wbinv_range(a, s)
+#define cpu_icache_sync_range(a, s)
+#define cpu_icache_sync_range_checked(a, s)
+
+#define cpufunc_nullop() loongarch_nullop()
+
+void loongarch_nullop(void);
+
+#endif /* _KERNEL */
+#endif /* _MACHINE_CPUFUNC_H_ */
diff --git a/sys/loongarch/include/db_machdep.h b/sys/loongarch/include/db_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/db_machdep.h
@@ -0,0 +1,100 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ */
+
+#ifndef _MACHINE_DB_MACHDEP_H_
+#define _MACHINE_DB_MACHDEP_H_
+
+#include <machine/loongarchreg.h>
+#include <machine/frame.h>
+#include <machine/trap.h>
+
+#define T_BREAKPOINT (EXCCODE_BP)
+#define T_WATCHPOINT (EXCCODE_WATCH)
+
+#define HAS_HW_BREAKPOINT
+#define NHBREAKPOINTS 4
+
+typedef vm_offset_t db_addr_t;
+typedef long db_expr_t;
+
+#define PC_REGS() ((db_addr_t)kdb_frame->tf_era)
+
+/*
+ * LoongArch 'break 0' instruction: 0x002a0000
+ * Format: 0000 0000 0010 1010 0000 0000 0000 0000
+ * bits[31:26] = 0b000000
+ * bits[25:21] = 0b00000
+ * bits[20:10] = 0b0101010000
+ * bits[9:5] = 0b00000
+ * bits[4:0] = break code (0 for `break 0`)
+ */
+#define BKPT_INST (0x002a0000)
+#define BKPT_SIZE (INSN_SIZE) /* Always 4 bytes */
+#define BKPT_SET(inst) (BKPT_INST)
+
+/*
+ * LoongArch has no compressed instructions, skip is always +4.
+ */
+#define BKPT_SKIP do { \
+ kdb_frame->tf_era += BKPT_SIZE; \
+} while (0)
+
+#define db_clear_single_step kdb_cpu_clear_singlestep
+#define db_set_single_step kdb_cpu_set_singlestep
+
+#define IS_BREAKPOINT_TRAP(type, code) (type == T_BREAKPOINT)
+#define IS_WATCHPOINT_TRAP(type, code) (type == T_WATCHPOINT)
+
+/*
+ * LoongArch instruction recognition for DDB backtrace.
+ *
+ * ertn (return from exception): 0x06488000
+ * bits[31:26] = 0b000110 (privileged)
+ */
+#define inst_trap_return(ins) ((ins) == 0x06488000) /* ertn */
+
+/*
+ * ret = jirl $zero, $ra, 0: 0x4c000020
+ * bits[31:26] = 0b010011 (jirl), rd=$zero, rj=$ra, offset=0
+ */
+#define inst_return(ins) ((ins) == 0x4c000020) /* ret */
+
+/*
+ * bl (branch and link): bits[31:26] = 0b010101
+ * jirl (jump and link reg): bits[31:26] = 0b010011
+ */
+#define inst_call(ins) ((((ins) >> 26) == 0x15) ||\
+ (((ins) >> 26) == 0x13)) /* bl, jirl */
+
+#define inst_load(ins) ({ \
+ uint32_t tmp_instr = db_get_value(PC_REGS(), \
+ sizeof(uint32_t), FALSE); \
+ is_load_instr(tmp_instr); \
+})
+
+#define inst_store(ins) ({ \
+ uint32_t tmp_instr = db_get_value(PC_REGS(), \
+ sizeof(uint32_t), FALSE); \
+ is_store_instr(tmp_instr); \
+})
+
+/*
+ * LoongArch load instructions: bits[31:24] == 0x28 (opcode 0b0010_1000xx)
+ * Covers LD.B, LD.BU, LD.H, LD.HU, LD.W, LD.WU, LD.D, LDX.{B,BU,H,HU,W,WU,D}
+ */
+#define is_load_instr(ins) (((ins) >> 24) == 0x28)
+
+/*
+ * LoongArch store instructions: bits[31:24] == 0x29 (opcode 0b0010_1001xx)
+ * Covers ST.B, ST.H, ST.W, ST.D, STX.{B,H,W,D}
+ */
+#define is_store_instr(ins) (((ins) >> 24) == 0x29)
+
+#define next_instr_address(pc, bd) ((bd) ? (pc) : ((pc) + 4))
+
+#define DB_ELFSIZE 64
+
+#endif /* !_MACHINE_DB_MACHDEP_H_ */
diff --git a/sys/loongarch/include/dump.h b/sys/loongarch/include/dump.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/dump.h
@@ -0,0 +1,83 @@
+/*-
+ * Copyright (c) 2014 EMC Corp.
+ * Author: Conrad Meyer <conrad.meyer@isilon.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_DUMP_H_
+#define _MACHINE_DUMP_H_
+
+#define KERNELDUMP_ARCH_VERSION KERNELDUMP_LOONGARCH_VERSION
+#define EM_VALUE EM_LOONGARCH
+
+/* 20 phys_avail entry pairs correspond to 10 pa's */
+#define DUMPSYS_MD_PA_NPAIRS 10
+#define DUMPSYS_NUM_AUX_HDRS 0
+
+/* How often to check the dump progress bar? */
+#define DUMPSYS_PB_CHECK_BITS 22 /* Every 4MB */
+
+static inline void
+dumpsys_pa_init(void)
+{
+
+ dumpsys_gen_pa_init();
+}
+
+static inline struct dump_pa *
+dumpsys_pa_next(struct dump_pa *p)
+{
+
+ return (dumpsys_gen_pa_next(p));
+}
+
+static inline void
+dumpsys_wbinv_all(void)
+{
+
+ dumpsys_gen_wbinv_all();
+}
+
+static inline void
+dumpsys_unmap_chunk(vm_paddr_t pa, size_t s, void *va)
+{
+
+ dumpsys_gen_unmap_chunk(pa, s, va);
+}
+
+static inline int
+dumpsys_write_aux_headers(struct dumperinfo *di)
+{
+
+ return (dumpsys_gen_write_aux_headers(di));
+}
+
+static inline int
+dumpsys(struct dumperinfo *di)
+{
+
+ return (dumpsys_generic(di));
+}
+
+#endif /* !_MACHINE_DUMP_H_ */
diff --git a/sys/loongarch/include/efi.h b/sys/loongarch/include/efi.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/efi.h
@@ -0,0 +1,11 @@
+/*-
+ * This file is in the public domain since it's just boilerplate.
+ */
+
+#ifndef __LOONGARCH_INCLUDE_EFI_H_
+#define __LOONGARCH_INCLUDE_EFI_H_
+
+#define EFIABI_ATTR
+
+
+#endif /* __LOONGARCH_INCLUDE_EFI_H_ */
diff --git a/sys/loongarch/include/elf.h b/sys/loongarch/include/elf.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/elf.h
@@ -0,0 +1,133 @@
+/*-
+ * Copyright (c) 1996-1997 John D. Polstra.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_ELF_H_
+#define _MACHINE_ELF_H_
+
+/*
+ * ELF definitions for the LoongArch architecture.
+ */
+
+#include <sys/elf32.h> /* Definitions common to all 32 bit architectures. */
+#include <sys/elf64.h> /* Definitions common to all 64 bit architectures. */
+
+#define __ELF_WORD_SIZE 64 /* Used by <sys/elf_generic.h> */
+#include <sys/elf_generic.h>
+
+/*
+ * Auxiliary vector entries for passing information to the interpreter.
+ */
+
+typedef struct { /* Auxiliary vector entry on initial stack */
+ int a_type; /* Entry type. */
+ union {
+ int a_val; /* Integer value. */
+ } a_un;
+} Elf32_Auxinfo;
+
+typedef struct { /* Auxiliary vector entry on initial stack */
+ long a_type; /* Entry type. */
+ union {
+ long a_val; /* Integer value. */
+ void *a_ptr; /* Address. */
+ void (*a_fcn)(void); /* Function pointer (not used). */
+ } a_un;
+} Elf64_Auxinfo;
+
+__ElfType(Auxinfo);
+
+#define ELF_ARCH EM_LOONGARCH
+
+#define ELF_MACHINE_OK(x) ((x) == (ELF_ARCH))
+
+/* Define "machine" characteristics */
+#define ELF_TARG_CLASS ELFCLASS64
+#define ELF_TARG_DATA ELFDATA2LSB
+#define ELF_TARG_MACH EM_LOONGARCH
+#define ELF_TARG_VER 1
+
+/* TODO: set correct value */
+#define ET_DYN_LOAD_ADDR 0x100000
+
+/* LoongArch note types (core dumps / PT_GETREGSET), per the LoongArch psABI. */
+#define NT_LOONGARCH_LSX 0xa02 /* LoongArch SIMD (LSX, 128-bit)
+ * registers */
+#define NT_LOONGARCH_LASX 0xa03 /* LoongArch Adv. SIMD (LASX, 256-bit)
+ * registers */
+
+/*
+ * LoongArch ELF relocation types. The authoritative set lives in
+ * <sys/elf_common.h> (included via <sys/elf64.h>); the subset below is kept
+ * for the entries not present there (the PCADD family and R_LARCH_64_PCREL,
+ * used by the kernel module loader). The remainder are redundant with
+ * elf_common.h and match it.
+ */
+#define R_LARCH_NONE 0
+#define R_LARCH_32 1
+#define R_LARCH_64 2
+#define R_LARCH_RELATIVE 3
+#define R_LARCH_COPY 4
+#define R_LARCH_JUMP_SLOT 5
+#define R_LARCH_IRELATIVE 12
+#define R_LARCH_MARK_LA 20
+#define R_LARCH_MARK_PCREL 21
+#define R_LARCH_SOP_PUSH_PCREL 22
+#define R_LARCH_SOP_PUSH_ABSOLUTE 23
+#define R_LARCH_SOP_PUSH_DUP 24
+#define R_LARCH_SOP_PUSH_PLT_PCREL 29
+#define R_LARCH_SOP_SUB 32
+#define R_LARCH_SOP_SL 33
+#define R_LARCH_SOP_SR 34
+#define R_LARCH_SOP_ADD 35
+#define R_LARCH_SOP_AND 36
+#define R_LARCH_SOP_IF_ELSE 37
+#define R_LARCH_SOP_POP_32_S_10_5 38
+#define R_LARCH_SOP_POP_32_U_10_12 39
+#define R_LARCH_SOP_POP_32_S_10_12 40
+#define R_LARCH_SOP_POP_32_S_10_16 41
+#define R_LARCH_SOP_POP_32_S_10_16_S2 42
+#define R_LARCH_SOP_POP_32_S_5_20 43
+#define R_LARCH_SOP_POP_32_S_0_5_10_16_S2 44
+#define R_LARCH_SOP_POP_32_S_0_10_10_16_S2 45
+#define R_LARCH_SOP_POP_32_U 46
+#define R_LARCH_ADD32 50
+#define R_LARCH_ADD64 51
+#define R_LARCH_SUB32 55
+#define R_LARCH_SUB64 56
+#define R_LARCH_B26 66
+#define R_LARCH_PCALA_HI20 71
+#define R_LARCH_PCALA_LO12 72
+#define R_LARCH_PCALA64_LO20 73
+#define R_LARCH_PCALA64_HI12 74
+#define R_LARCH_GOT_PC_HI20 75
+#define R_LARCH_GOT_PC_LO12 76
+#define R_LARCH_64_PCREL 109
+#define R_LARCH_PCADD_HI20 128
+#define R_LARCH_PCADD_LO12 129
+#define R_LARCH_GOT_PCADD_HI20 130
+#define R_LARCH_GOT_PCADD_LO12 131
+
+#endif /* !_MACHINE_ELF_H_ */
diff --git a/sys/loongarch/include/encoding.h b/sys/loongarch/include/encoding.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/encoding.h
@@ -0,0 +1,8 @@
+/*-
+ * This file is in the public domain.
+ */
+
+/* Automatically generated by parse-opcodes. */
+#ifndef LOONGARCH_ENCODING_H
+#define LOONGARCH_ENCODING_H
+#endif
diff --git a/sys/loongarch/include/endian.h b/sys/loongarch/include/endian.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/endian.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2001 David E. O'Brien
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)endian.h 8.1 (Berkeley) 6/10/93
+ * $NetBSD: endian.h,v 1.7 1999/08/21 05:53:51 simonb Exp $
+ */
+
+#ifndef _MACHINE_ENDIAN_H_
+#define _MACHINE_ENDIAN_H_
+
+#include <sys/_types.h>
+#include <sys/_endian.h>
+
+#endif /* !_MACHINE_ENDIAN_H_ */
diff --git a/sys/loongarch/include/exec.h b/sys/loongarch/include/exec.h
new file mode 100644
diff --git a/sys/loongarch/include/float.h b/sys/loongarch/include/float.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/float.h
@@ -0,0 +1,93 @@
+/*-
+ * Copyright (c) 1989 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: @(#)float.h 7.1 (Berkeley) 5/8/90
+ */
+
+#ifndef _MACHINE_FLOAT_H_
+#define _MACHINE_FLOAT_H_
+
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+extern int __flt_rounds(void);
+__END_DECLS
+
+#define FLT_RADIX 2 /* b */
+#define FLT_ROUNDS __flt_rounds()
+#if __ISO_C_VISIBLE >= 1999
+#define FLT_EVAL_METHOD 0
+#define DECIMAL_DIG 17 /* max precision in decimal digits */
+#endif
+
+#define FLT_MANT_DIG 24 /* p */
+#define FLT_EPSILON 1.19209290E-07F /* b**(1-p) */
+#define FLT_DIG 6 /* floor((p-1)*log10(b))+(b == 10) */
+#define FLT_MIN_EXP (-125) /* emin */
+#define FLT_MIN 1.17549435E-38F /* b**(emin-1) */
+#define FLT_MIN_10_EXP (-37) /* ceil(log10(b**(emin-1))) */
+#define FLT_MAX_EXP 128 /* emax */
+#define FLT_MAX 3.40282347E+38F /* (1-b**(-p))*b**emax */
+#define FLT_MAX_10_EXP 38 /* floor(log10((1-b**(-p))*b**emax)) */
+#if __ISO_C_VISIBLE >= 2011
+#define FLT_TRUE_MIN 1.40129846E-45F /* b**(emin-p) */
+#define FLT_DECIMAL_DIG 9 /* ceil(1+p*log10(b)) */
+#define FLT_HAS_SUBNORM 1
+#endif /* __ISO_C_VISIBLE >= 2011 */
+
+#define DBL_MANT_DIG 53
+#define DBL_EPSILON 2.2204460492503131E-16
+#define DBL_DIG 15
+#define DBL_MIN_EXP (-1021)
+#define DBL_MIN 2.2250738585072014E-308
+#define DBL_MIN_10_EXP (-307)
+#define DBL_MAX_EXP 1024
+#define DBL_MAX 1.7976931348623157E+308
+#define DBL_MAX_10_EXP 308
+#if __ISO_C_VISIBLE >= 2011
+#define DBL_TRUE_MIN 4.9406564584124654E-324
+#define DBL_DECIMAL_DIG 17
+#define DBL_HAS_SUBNORM 1
+#endif /* __ISO_C_VISIBLE >= 2011 */
+
+#define LDBL_MANT_DIG 113
+#define LDBL_EPSILON 1.925929944387235853055977942584927319E-34L
+#define LDBL_DIG 33
+#define LDBL_MIN_EXP (-16381)
+#define LDBL_MIN 3.362103143112093506262677817321752603E-4932L
+#define LDBL_MIN_10_EXP (-4931)
+#define LDBL_MAX_EXP (+16384)
+#define LDBL_MAX 1.189731495357231765085759326628007016E+4932L
+#define LDBL_MAX_10_EXP (+4932)
+#if __ISO_C_VISIBLE >= 2011
+#define LDBL_TRUE_MIN 6.475175119438025110924438958227646552E-4966L
+#define LDBL_DECIMAL_DIG 36
+#define LDBL_HAS_SUBNORM 1
+#endif /* __ISO_C_VISIBLE >= 2011 */
+
+#endif /* _MACHINE_FLOAT_H_ */
diff --git a/sys/loongarch/include/floatingpoint.h b/sys/loongarch/include/floatingpoint.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/floatingpoint.h
@@ -0,0 +1,2 @@
+
+#include <machine/ieeefp.h>
diff --git a/sys/loongarch/include/fpe.h b/sys/loongarch/include/fpe.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/fpe.h
@@ -0,0 +1,33 @@
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_FPE_H_
+#define _MACHINE_FPE_H_
+
+void fpe_state_save(struct thread *td);
+void fpe_state_clear(void);
+
+#endif /* !_MACHINE_FPE_H_ */
diff --git a/sys/loongarch/include/frame.h b/sys/loongarch/include/frame.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/frame.h
@@ -0,0 +1,114 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_FRAME_H_
+#define _MACHINE_FRAME_H_
+
+#ifndef LOCORE
+
+#include <sys/signal.h>
+#include <sys/ucontext.h>
+
+/*
+ * struct trapframe is NOT byte-identical to struct reg: struct reg keeps
+ * only the GPRs plus era/prmd (see <machine/reg.h>).
+ */
+struct trapframe {
+ /* general registers */
+ uint64_t r0;
+ uint64_t ra;
+ uint64_t tp;
+ uint64_t sp;
+ uint64_t a[8];
+ uint64_t t[9];
+ uint64_t r21;
+ uint64_t fp;
+ uint64_t s[9];
+
+#define tf_a a
+#define tf_t t
+#define tf_s s
+#define tf_sp sp
+#define tf_tp tp
+#define tf_ra ra
+#define tf_fp fp
+
+ /* Special CSR registers. */
+ uint64_t tf_crmd;
+ uint64_t tf_prmd;
+ uint64_t tf_euen;
+ uint64_t tf_misc;
+ uint64_t tf_ecfg;
+ uint64_t tf_estat;
+ uint64_t tf_era;
+ uint64_t tf_badvaddr;
+
+ uint64_t fa[8];
+ uint64_t ft[16];
+ uint64_t fs[8];
+ uint64_t fcsr0;
+ uint64_t fcc;
+ /*
+ * Vector register storage for LSX (128-bit) / LASX (256-bit).
+ * 32 slots of 32 bytes each; saved only when EUEN has LSXEN/LASXEN
+ * set. The low 64/128 bits alias the scalar FP / LSX view of the
+ * same physical register.
+ */
+ uint64_t tf_vregs[32][4];
+ /* Original syscal arg0 */
+ uint64_t tf_a0;
+};
+
+/*
+ * Signal frame. Pushed onto user stack before calling sigcode.
+ */
+/*
+ * Optional SIMD extension appended to struct sigframe (kernel-internal
+ * layout; not part of the user signal ABI). When mcontext flags
+ * _MC_FP_SIMD, mc_spare[0] holds the user-space address of this block.
+ */
+#define LOONGARCH_SIMD_MAGIC_LSX 0x53580001 /* "SX\1" */
+#define LOONGARCH_SIMD_MAGIC_LASX 0x41535801 /* "ASX\1" */
+struct loongarch_simd_ctx {
+ uint32_t d_magic; /* LSX or LASX magic */
+ uint32_t d_size; /* sizeof(this struct) */
+ uint32_t d_pad[2]; /* align payload to 16 bytes */
+ uint8_t d_regs[32][32]; /* 32 vector regs: low 16 bytes
+ * valid for LSX, full 32 for LASX */
+};
+
+struct sigframe {
+ siginfo_t sf_si; /* actual saved siginfo */
+ ucontext_t sf_uc; /* actual saved ucontext */
+ struct loongarch_simd_ctx sf_simd; /* optional SIMD state */
+};
+
+#endif /* !LOCORE */
+
+/* Definitions for syscalls */
+#define NARGREG 8 /* 8 args in regs */
+
+#endif /* !_MACHINE_FRAME_H_ */
diff --git a/sys/loongarch/include/gdb_machdep.h b/sys/loongarch/include/gdb_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/gdb_machdep.h
@@ -0,0 +1,83 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Mitchell Horne <mhorne@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_GDB_MACHDEP_H_
+#define _MACHINE_GDB_MACHDEP_H_
+
+#define GDB_BUFSZ 4096
+#define GDB_NREGS 33 /* 32 GPRs (r0-r31) + PC */
+
+#define GDB_REG_ZERO 0
+#define GDB_REG_RA 1
+#define GDB_REG_TP 2
+#define GDB_REG_SP 3
+#define GDB_REG_A0 4
+#define GDB_REG_T0 12
+#define GDB_REG_R21 21
+#define GDB_REG_FP 22
+#define GDB_REG_S0 23
+#define GDB_REG_PC 32
+_Static_assert(GDB_BUFSZ >= (GDB_NREGS * 8), "buffer fits 'g' regs");
+
+static __inline size_t
+gdb_cpu_regsz(int regnum __unused)
+{
+
+ return (8);
+}
+
+static __inline int
+gdb_cpu_query(void)
+{
+ return (0);
+}
+
+static __inline void *
+gdb_begin_write(void)
+{
+
+ return (NULL);
+}
+
+static __inline void
+gdb_end_write(void *arg __unused)
+{
+
+}
+
+static __inline void
+gdb_cpu_stop_reason(int type __unused, int code __unused)
+{
+
+}
+
+void *gdb_cpu_getreg(int, size_t *);
+void gdb_cpu_setreg(int, void *);
+int gdb_cpu_signal(int, int);
+
+#endif /* !_MACHINE_GDB_MACHDEP_H_ */
diff --git a/sys/loongarch/include/ieeefp.h b/sys/loongarch/include/ieeefp.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/ieeefp.h
@@ -0,0 +1,8 @@
+
+#ifndef _MACHINE_IEEEFP_H_
+#define _MACHINE_IEEEFP_H_
+
+/* TODO */
+typedef int fp_except_t;
+
+#endif /* _MACHINE_IEEEFP_H_ */
diff --git a/sys/loongarch/include/ifunc.h b/sys/loongarch/include/ifunc.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/ifunc.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 2015-2018 The FreeBSD Foundation
+ * Copyright (c) 2024 Jessica Clarke <jrtc27@FreeBSD.org>
+ *
+ * Part of this software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __LOONGARCH_IFUNC_H
+#define __LOONGARCH_IFUNC_H
+
+#define DEFINE_IFUNC(qual, ret_type, name, args) \
+ static ret_type (*name##_resolver(void))args __used; \
+ qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \
+ static ret_type (*name##_resolver(void))args
+
+#define DEFINE_UIFUNC(qual, ret_type, name, args) \
+ static ret_type (*name##_resolver(unsigned long, unsigned long, \
+ unsigned long, unsigned long, unsigned long, unsigned long, \
+ unsigned long, unsigned long))args __used; \
+ qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \
+ static ret_type (*name##_resolver(unsigned long elf_hwcap __unused, \
+ unsigned long _arg2 __unused, unsigned long _arg3 __unused, \
+ unsigned long _arg4 __unused, unsigned long _arg5 __unused, \
+ unsigned long _arg6 __unused, unsigned long _arg7 __unused, \
+ unsigned long _arg8 __unused))args
+
+#endif
diff --git a/sys/loongarch/include/in_cksum.h b/sys/loongarch/include/in_cksum.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/in_cksum.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from tahoe: in_cksum.c 1.2 86/01/05
+ * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91
+ * from: Id: in_cksum.c,v 1.8 1995/12/03 18:35:19 bde Exp
+ */
+
+#ifndef _MACHINE_IN_CKSUM_H_
+#define _MACHINE_IN_CKSUM_H_ 1
+
+#ifdef _KERNEL
+#define in_cksum(m, len) in_cksum_skip(m, len, 0)
+u_short in_addword(u_short sum, u_short b);
+u_short in_cksum_skip(struct mbuf *m, int len, int skip);
+u_int do_cksum(const void *, int);
+#if defined(IPVERSION) && (IPVERSION == 4)
+u_int in_cksum_hdr(const struct ip *);
+#endif
+
+u_short in_pseudo(u_int sum, u_int b, u_int c);
+
+#endif /* _KERNEL */
+#endif /* _MACHINE_IN_CKSUM_H_ */
diff --git a/sys/loongarch/include/intr.h b/sys/loongarch/include/intr.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/intr.h
@@ -0,0 +1,140 @@
+/*-
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_INTR_MACHDEP_H_
+#define _MACHINE_INTR_MACHDEP_H_
+
+#define MAX_IO_PICS 2
+#define NR_IRQS (64 + (256 * MAX_IO_PICS))
+
+#define NIRQ NR_IRQS
+#define INTR_ROOT_COUNT 1
+#define INTR_ROOT_IRQ 0
+
+/* Interrupt numbers */
+#define INT_SWI0 0 /* Software Interrupts */
+#define INT_SWI1 1
+#define INT_HWI0 2 /* Hardware Interrupts */
+#define INT_HWI1 3
+#define INT_HWI2 4
+#define INT_HWI3 5
+#define INT_HWI4 6
+#define INT_HWI5 7
+#define INT_HWI6 8
+#define INT_HWI7 9
+#define INT_PCOV 10 /* Performance Counter Overflow */
+#define INT_TI 11 /* Timer */
+#define INT_IPI 12
+
+#ifndef LOCORE
+#ifdef INTRNG
+#include <sys/intr.h>
+#endif
+
+struct trapframe;
+
+/* EIOINTC PIC driver interface */
+void eiointc_enable_vector(device_t dev, u_int vector);
+void eiointc_disable_vector(device_t dev, u_int vector);
+void eiointc_init_cpu(void);
+
+/* PCH-PIC FDT xref, for PCI INTx routing in nexus. */
+intptr_t pch_pic_get_xref(void);
+
+/* CPU-local interrupt setup via the cpuic driver (timer, IPI). */
+int cpuic_setup_cpu_intr(const char *, driver_filter_t *,
+ void (*)(void *), void *, u_int, int, void **);
+void cpuic_setup_ipi(driver_filter_t *);
+
+/* ACPI MSI cross-reference for the generic PCI host bridge (placeholder
+ * until the PCH MSI controller is properly wired via MADT MSI_PIC). */
+#define ACPI_MSI_XREF 2
+
+/*
+ * ACPI cross-reference for the interrupt controller that owns the GSI
+ * (global system interrupt) space. On ACPI the PCH-PIC maps device GSIs
+ * (LOONGSON_PCH_IRQ_BASE == 64 and up) to EIOINTC vectors, so it owns this
+ * xref. nexus_acpi_map_intr resolves every device interrupt through it.
+ * xref 1 is cpuic, 2 is eioic; this must not collide with those.
+ */
+#define ACPI_INTR_XREF 3
+
+/* IPI/mailbox hardware helpers */
+void csr_mail_send(uint64_t data, int cpu, int mailbox);
+void eiointc_enable_extioi(void);
+uint32_t ipi_read_clear(int cpu);
+void ipi_write_action(int cpu, uint32_t action);
+void ipi_all_but_self(u_int ipi);
+void ipi_cpu(int cpu, u_int ipi);
+void ipi_selected(cpuset_t cpus, u_int ipi);
+
+#define CORES_PER_EIO_NODE 4
+
+#define LOONGSON_CPU_UART0_VEC 10 /* CPU UART0 */
+#define LOONGSON_CPU_THSENS_VEC 14 /* CPU Thsens */
+#define LOONGSON_CPU_HT0_VEC 16 /* CPU HT0 irq vector base number */
+#define LOONGSON_CPU_HT1_VEC 24 /* CPU HT1 irq vector base number */
+
+/* IRQ number definitions */
+#define LOONGSON_LPC_IRQ_BASE 0
+#define LOONGSON_LPC_LAST_IRQ (LOONGSON_LPC_IRQ_BASE + 15)
+
+#define LOONGSON_CPU_IRQ_BASE 16
+#define LOONGSON_CPU_LAST_IRQ (LOONGSON_CPU_IRQ_BASE + 14)
+
+#define LOONGSON_PCH_IRQ_BASE 64
+#define LOONGSON_PCH_ACPI_IRQ (LOONGSON_PCH_IRQ_BASE + 47)
+#define LOONGSON_PCH_LAST_IRQ (LOONGSON_PCH_IRQ_BASE + 64 - 1)
+
+#define LOONGSON_MSI_IRQ_BASE (LOONGSON_PCH_IRQ_BASE + 64)
+#define LOONGSON_MSI_LAST_IRQ (LOONGSON_PCH_IRQ_BASE + 256 - 1)
+
+#define GSI_MIN_LPC_IRQ LOONGSON_LPC_IRQ_BASE
+#define GSI_MAX_LPC_IRQ (LOONGSON_LPC_IRQ_BASE + 16 - 1)
+#define GSI_MIN_CPU_IRQ LOONGSON_CPU_IRQ_BASE
+#define GSI_MAX_CPU_IRQ (LOONGSON_CPU_IRQ_BASE + 48 - 1)
+#define GSI_MIN_PCH_IRQ LOONGSON_PCH_IRQ_BASE
+#define GSI_MAX_PCH_IRQ (LOONGSON_PCH_IRQ_BASE + 256 - 1)
+
+enum {
+ IRQ_SWI0 = INT_SWI0,
+ IRQ_SWI1 = INT_SWI1,
+ IRQ_HWI0 = INT_HWI0,
+ IRQ_HWI1 = INT_HWI1,
+ IRQ_HWI2 = INT_HWI2,
+ IRQ_HWI3 = INT_HWI3,
+ IRQ_HWI4 = INT_HWI4,
+ IRQ_HWI5 = INT_HWI5,
+ IRQ_HWI6 = INT_HWI6,
+ IRQ_HWI7 = INT_HWI7,
+ IRQ_PCOV = INT_PCOV, /* Performance counter overflow */
+ IRQ_TI = INT_TI, /* Timer */
+ IRQ_IPI = INT_IPI,
+ INTC_NIRQS
+};
+#endif
+
+#endif /* !_MACHINE_INTR_MACHDEP_H_ */
diff --git a/sys/loongarch/include/iodev.h b/sys/loongarch/include/iodev.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/iodev.h
@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ */
+
+#ifndef _MACHINE_IODEV_H_
+#define _MACHINE_IODEV_H_
+
+/*
+ * Memory-mapped I/O read/write macros. LoongArch accesses MMIO through
+ * normal load/store with the uncached attribute (DMW0 or device PTE).
+ */
+
+#define iodev_read_1(a) (*(volatile uint8_t *)(uintptr_t)(a))
+#define iodev_read_2(a) (*(volatile uint16_t *)(uintptr_t)(a))
+#define iodev_read_4(a) (*(volatile uint32_t *)(uintptr_t)(a))
+
+#define iodev_write_1(a, v) \
+ (*(volatile uint8_t *)(uintptr_t)(a) = (v))
+#define iodev_write_2(a, v) \
+ (*(volatile uint16_t *)(uintptr_t)(a) = (v))
+#define iodev_write_4(a, v) \
+ (*(volatile uint32_t *)(uintptr_t)(a) = (v))
+
+#endif /* _MACHINE_IODEV_H_ */
diff --git a/sys/loongarch/include/kdb.h b/sys/loongarch/include/kdb.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/kdb.h
@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 2004 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_KDB_H_
+#define _MACHINE_KDB_H_
+
+#include <machine/cpufunc.h>
+
+void kdb_cpu_clear_singlestep(void);
+void kdb_cpu_set_singlestep(void);
+
+static __inline void
+kdb_cpu_sync_icache(unsigned char *addr, size_t size)
+{
+
+ /*
+ * Other CPUs flush their instruction cache when resuming from
+ * IPI_STOP.
+ */
+ flush_icache();
+}
+
+static __inline void
+kdb_cpu_trap(int type, int code)
+{
+}
+
+int kdb_cpu_set_watchpoint(vm_offset_t addr, vm_size_t size, int access);
+int kdb_cpu_clr_watchpoint(vm_offset_t addr, vm_size_t size);
+int kdb_cpu_set_breakpoint(vm_offset_t addr);
+int kdb_cpu_clr_breakpoint(vm_offset_t addr);
+
+#endif /* _MACHINE_KDB_H_ */
diff --git a/sys/loongarch/include/kexec.h b/sys/loongarch/include/kexec.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/kexec.h
@@ -0,0 +1,40 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2025 Juniper Networks, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _LOONGARCH_KEXEC_H_
+#define _LOONGARCH_KEXEC_H_
+
+int
+kexec_load_md(struct kexec_image *image)
+{
+ return (ENOSYS);
+}
+
+#define kexec_reboot_md(x) do {} while (0)
+#endif /* _LOONGARCH_KEXEC_H_ */
+
diff --git a/sys/loongarch/include/limits.h b/sys/loongarch/include/limits.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/limits.h
@@ -0,0 +1,41 @@
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1988, 1993
+ * The Regents of the University of California. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)limits.h 8.3 (Berkeley) 1/4/94
+ */
+
+#ifndef _MACHINE_LIMITS_H_
+#define _MACHINE_LIMITS_H_
+
+#warning "machine/limits.h is deprecated. Include sys/limits.h instead."
+
+#include <sys/limits.h>
+
+#endif /* !_MACHINE_LIMITS_H_ */
diff --git a/sys/loongarch/include/loongarchreg.h b/sys/loongarch/include/loongarchreg.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/loongarchreg.h
@@ -0,0 +1,452 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * LoongArch CSR and register definitions.
+ */
+
+#ifndef _MACHINE_LOONGARCH_H_
+#define _MACHINE_LOONGARCH_H_
+
+#define INSN_SIZE 4
+
+/* Generic bitmask helper (used by identcpu.c) */
+#define _GENMASK(h, l) (((~0UL) >> (63 - (h))) & ((~0UL) << (l)))
+
+/* Bit fields for CPUCFG registers */
+#define LOONGARCH_CPUCFG0 0x0
+
+#define BIT_U(n) (1U << (n))
+#define BIT_ULL(nr) (1ULL << (nr))
+
+#define LOONGARCH_CPUCFG1 0x1
+#define CPUCFG1_ISGR32 BIT_U(0)
+#define CPUCFG1_ISGR64 BIT_U(1)
+#define CPUCFG1_UAL BIT_U(20)
+#define CPUCFG1_CRC32 BIT_U(25)
+
+#define LOONGARCH_CPUCFG2 0x2
+#define CPUCFG2_FP BIT_U(0)
+#define CPUCFG2_LSX BIT_U(6)
+#define CPUCFG2_LASX BIT_U(7)
+#define CPUCFG2_COMPLEX BIT_U(8)
+#define CPUCFG2_CRYPTO BIT_U(9)
+#define CPUCFG2_LVZP BIT_U(10)
+#define CPUCFG2_LLFTP BIT_U(14)
+#define CPUCFG2_X86BT BIT_U(18)
+#define CPUCFG2_ARMBT BIT_U(19)
+#define CPUCFG2_MIPSBT BIT_U(20)
+#define CPUCFG2_LAM BIT_U(22)
+#define CPUCFG2_PTW BIT_U(24)
+
+#define LOONGARCH_CPUCFG4 0x4
+
+#define LOONGARCH_CPUCFG5 0x5
+
+#define LOONGARCH_CPUCFG6 0x6
+#define CPUCFG6_PMP BIT_U(0)
+
+/* CSR */
+
+#define __csrrd(reg) \
+({ u_long val; \
+ __asm __volatile("csrrd %0, " #reg : "=r" (val)); \
+ val; \
+})
+
+#define __csrwr(val, reg) \
+({ \
+ uint64_t __temp = val; \
+ __asm __volatile( \
+ "csrwr %0, " #reg : "+r" (__temp) \
+ :: "memory" \
+ ); \
+})
+
+#define csr_read32(reg) __csrrd(reg)
+#define csr_read64(reg) __csrrd(reg)
+#define csr_write32(val, reg) __csrwr(val, reg)
+#define csr_write64(val, reg) __csrwr(val, reg)
+
+/* IOCSR */
+
+#define __iocsrrd_w(reg) \
+({ uint32_t val; \
+ __asm __volatile( \
+ "iocsrrd.w %0, %1" \
+ : "=r"(val) \
+ : "r"(reg)); \
+ val; \
+})
+
+#define __iocsrrd_d(reg) \
+({ uint64_t val; \
+ __asm __volatile( \
+ "iocsrrd.d %0, %1" \
+ : "=r"(val) \
+ : "r" (reg)); \
+ val; \
+})
+
+#define __iocsrwr_w(val, reg) \
+({ \
+ __asm __volatile( \
+ "iocsrwr.w %0, %1\n" \
+ :: "r"(val), "r"(reg)); \
+})
+
+#define __iocsrwr_d(val, reg) \
+({ \
+ __asm __volatile( \
+ "iocsrwr.d %0, %1\n" \
+ :: "r" (val), "r"(reg)); \
+})
+
+#define iocsr_read32(reg) __iocsrrd_w(reg)
+#define iocsr_read64(reg) __iocsrrd_d(reg)
+#define iocsr_write32(val, reg) __iocsrwr_w(val, reg)
+#define iocsr_write64(val, reg) __iocsrwr_d(val, reg)
+
+
+/* CSR register number */
+
+#define _ULCAST_(x) (x##UL)
+
+/* Basic CSR registers */
+#define LOONGARCH_CSR_CRMD 0x0 /* Current mode info */
+#define CSR_CRMD_WE_SHIFT 9
+#define CSR_CRMD_WE (_ULCAST_(0x1) << CSR_CRMD_WE_SHIFT)
+#define CSR_CRMD_IE_SHIFT 2
+#define CSR_CRMD_IE (_ULCAST_(0x1) << CSR_CRMD_IE_SHIFT)
+#define CSR_CRMD_PLV_SHIFT 0
+
+#define PLV_USER 3
+#define PLV_MASK 0x3
+
+#define LOONGARCH_CSR_PRMD 0x1 /* Prev-exception mode info */
+#define CSR_PRMD_PPLV PLV_MASK /* bits [1:0]: prev PLV */
+#define CSR_PRMD_PWE_SHIFT 3
+#define CSR_PRMD_PWE (_ULCAST_(0x1) << CSR_PRMD_PWE_SHIFT)
+#define CSR_PRMD_PIE_SHIFT 2
+#define CSR_PRMD_PIE (_ULCAST_(0x1) << CSR_PRMD_PIE_SHIFT)
+#define CSR_PRMD_PPLV_SHIFT 0
+
+#define LOONGARCH_CSR_EUEN 0x2 /* Extended unit enable */
+#define CSR_EUEN_LASXEN_SHIFT 2
+#define CSR_EUEN_LASXEN (_ULCAST_(0x1) << CSR_EUEN_LASXEN_SHIFT)
+#define CSR_EUEN_LSXEN_SHIFT 1
+#define CSR_EUEN_LSXEN (_ULCAST_(0x1) << CSR_EUEN_LSXEN_SHIFT)
+#define CSR_EUEN_FPEN_SHIFT 0
+#define CSR_EUEN_FPEN (_ULCAST_(0x1) << CSR_EUEN_FPEN_SHIFT)
+
+#define LOONGARCH_CSR_MISC 0x3 /* Misc config */
+
+#define LOONGARCH_CSR_ECFG 0x4 /* Exception config */
+#define CSR_ECFG_IM_SHIFT 0
+#define CSR_ECFG_IM (_ULCAST_(0x3fff) << CSR_ECFG_IM_SHIFT)
+
+#define LOONGARCH_CSR_ESTAT 0x5 /* Exception status */
+#define CSR_ESTAT_EXC_SHIFT 16
+#define CSR_ESTAT_EXC (_ULCAST_(0x3f) << CSR_ESTAT_EXC_SHIFT)
+#define CSR_ESTAT_IS_SHIFT 0
+#define CSR_ESTAT_IS (_ULCAST_(0x3fff) << CSR_ESTAT_IS_SHIFT)
+
+#define LOONGARCH_CSR_ERA 0x6 /* ERA */
+
+#define LOONGARCH_CSR_BADV 0x7 /* Bad virtual address */
+
+#define LOONGARCH_CSR_EENTRY 0xc /* Exception entry */
+
+/* TLB related CSR registers */
+#define LOONGARCH_CSR_TLBIDX 0x10 /* TLB Index, EHINV, PageSize, NP */
+#define CSR_TLBIDX_EHINV_SHIFT 31
+#define CSR_TLBIDX_EHINV (_ULCAST_(1) << CSR_TLBIDX_EHINV_SHIFT)
+#define CSR_TLBIDX_PS_SHIFT 24
+#define CSR_TLBIDX_PS (_ULCAST_(0x3f) << CSR_TLBIDX_PS_SHIFT)
+
+#define LOONGARCH_CSR_TLBEHI 0x11 /* TLB EntryHi */
+
+#define LOONGARCH_CSR_TLBELO0 0x12 /* TLB EntryLo0 */
+
+#define LOONGARCH_CSR_TLBELO1 0x13 /* TLB EntryLo1 */
+
+#define LOONGARCH_CSR_ASID 0x18 /* ASID */
+#define CSR_ASID_BIT_SHIFT 16 /* ASIDBits */
+#define CSR_ASID_BIT (_ULCAST_(0xff) << CSR_ASID_BIT_SHIFT)
+#define CSR_ASID_ASID_SHIFT 0
+#define CSR_ASID_ASID (_ULCAST_(0x3ff) << CSR_ASID_ASID_SHIFT)
+
+#define LOONGARCH_CSR_PGDL 0x19 /* Page table base address when
+ * VA[VALEN-1] = 0 */
+
+#define LOONGARCH_CSR_PGDH 0x1a /* Page table base address when
+ * VA[VALEN-1] = 1 */
+
+#define LOONGARCH_CSR_PGD 0x1b /* Page table base */
+
+#define LOONGARCH_CSR_PWCTL0 0x1c /* PWCtl0 */
+
+#define LOONGARCH_CSR_PWCTL1 0x1d /* PWCtl1 */
+#define CSR_PWCTL1_PTW_SHIFT 24
+#define CSR_PWCTL1_PTW (_ULCAST_(0x1) << CSR_PWCTL1_PTW_SHIFT)
+
+#define LOONGARCH_CSR_STLBPGSIZE 0x1e
+
+/* Config CSR registers */
+#define LOONGARCH_CSR_CPUID 0x20 /* CPU core id */
+
+#define LOONGARCH_CSR_PRCFG1 0x21 /* Config1 */
+#define CSR_CONF1_KSNUM _ULCAST_(0xf)
+
+#define LOONGARCH_CSR_PRCFG3 0x23 /* Config3 */
+#define CSR_CONF3_STLBIDX_SHIFT 20
+#define CSR_CONF3_STLBIDX (_ULCAST_(0x3f) << CSR_CONF3_STLBIDX_SHIFT)
+#define CSR_CONF3_STLBWAYS_SHIFT 12
+#define CSR_CONF3_STLBWAYS (_ULCAST_(0xff) << CSR_CONF3_STLBWAYS_SHIFT)
+#define CSR_CONF3_MTLBSIZE_SHIFT 4
+#define CSR_CONF3_MTLBSIZE (_ULCAST_(0xff) << CSR_CONF3_MTLBSIZE_SHIFT)
+#define CSR_CONF3_TLBTYPE_SHIFT 0
+#define CSR_CONF3_TLBTYPE (_ULCAST_(0xf) << CSR_CONF3_TLBTYPE_SHIFT)
+
+/* KSave registers */
+#define LOONGARCH_CSR_KS0 0x30
+#define LOONGARCH_CSR_KS1 0x31
+#define LOONGARCH_CSR_KS2 0x32
+#define LOONGARCH_CSR_KS3 0x33
+#define LOONGARCH_CSR_KS4 0x34
+
+/* Exception allocated KS0, KS1, KS2, and KS3 statically */
+#define EXCEPTION_KS0 LOONGARCH_CSR_KS0
+#define EXCEPTION_KS1 LOONGARCH_CSR_KS1
+#define EXCEPTION_KS2 LOONGARCH_CSR_KS2
+#define EXCEPTION_KS3 LOONGARCH_CSR_KS3
+#define EXC_KSAVE_MASK (1 << 0 | 1 << 1 | 1 << 2 | 1 << 3)
+
+/* Percpu-data base allocated KS4 statically */
+#define PERCPU_BASE_KS LOONGARCH_CSR_KS4
+#define PERCPU_KSAVE_MASK (1 << 4)
+
+/* Timer registers */
+#define LOONGARCH_CSR_TMID 0x40 /* Timer ID */
+
+#define LOONGARCH_CSR_TCFG 0x41 /* Timer config */
+#define CSR_TCFG_VAL_SHIFT 2
+#define CSR_TCFG_VAL (_ULCAST_(0x3fffffffffff) << CSR_TCFG_VAL_SHIFT)
+#define CSR_TCFG_PERIOD_SHIFT 1
+#define CSR_TCFG_PERIOD (_ULCAST_(0x1) << CSR_TCFG_PERIOD_SHIFT)
+#define CSR_TCFG_EN (_ULCAST_(0x1))
+
+#define LOONGARCH_CSR_TINTCLR 0x44 /* Timer interrupt clear */
+
+/* TLB Refill registers */
+#define LOONGARCH_CSR_TLBRENTRY 0x88 /* TLB refill exception entry */
+#define LOONGARCH_CSR_TLBRSAVE 0x8b /* KSave for TLB refill exception */
+
+/* Direct Map windows registers */
+#define LOONGARCH_CSR_DMWIN0 0x180 /* 64 direct map win0: MEM & IF */
+
+/* Direct Map window 0/1 */
+#define DMW_PABITS 48
+
+#define CSR_DMW0_PLV0 (1 << 0)
+#define CSR_DMW0_VSEG (_ULCAST_(0x8000))
+#define CSR_DMW0_BASE (CSR_DMW0_VSEG << DMW_PABITS)
+#define CSR_DMW0_INIT (CSR_DMW0_BASE | CSR_DMW0_PLV0)
+
+/* Debug registers */
+#define LOONGARCH_CSR_MWPC 0x300 /* data breakpoint config */
+#define LOONGARCH_CSR_MWPS 0x301 /* data breakpoint status */
+
+#define LOONGARCH_CSR_DB0ADDR 0x310 /* data breakpoint 0 address */
+#define LOONGARCH_CSR_DB0MASK 0x311 /* data breakpoint 0 mask */
+#define LOONGARCH_CSR_DB0CTRL 0x312 /* data breakpoint 0 control */
+#define LOONGARCH_CSR_DB0ASID 0x313 /* data breakpoint 0 asid */
+
+#define LOONGARCH_CSR_DB1ADDR 0x318 /* data breakpoint 1 address */
+#define LOONGARCH_CSR_DB1MASK 0x319 /* data breakpoint 1 mask */
+#define LOONGARCH_CSR_DB1CTRL 0x31a /* data breakpoint 1 control */
+#define LOONGARCH_CSR_DB1ASID 0x31b /* data breakpoint 1 asid */
+
+#define LOONGARCH_CSR_DB2ADDR 0x320 /* data breakpoint 2 address */
+#define LOONGARCH_CSR_DB2MASK 0x321 /* data breakpoint 2 mask */
+#define LOONGARCH_CSR_DB2CTRL 0x322 /* data breakpoint 2 control */
+#define LOONGARCH_CSR_DB2ASID 0x323 /* data breakpoint 2 asid */
+
+#define LOONGARCH_CSR_DB3ADDR 0x328 /* data breakpoint 3 address */
+#define LOONGARCH_CSR_DB3MASK 0x329 /* data breakpoint 3 mask */
+#define LOONGARCH_CSR_DB3CTRL 0x32a /* data breakpoint 3 control */
+#define LOONGARCH_CSR_DB3ASID 0x32b /* data breakpoint 3 asid */
+
+#define LOONGARCH_CSR_DB4ADDR 0x330 /* data breakpoint 4 address */
+#define LOONGARCH_CSR_DB4MASK 0x331 /* data breakpoint 4 maks */
+#define LOONGARCH_CSR_DB4CTRL 0x332 /* data breakpoint 4 control */
+#define LOONGARCH_CSR_DB4ASID 0x333 /* data breakpoint 4 asid */
+
+#define LOONGARCH_CSR_DB5ADDR 0x338 /* data breakpoint 5 address */
+#define LOONGARCH_CSR_DB5MASK 0x339 /* data breakpoint 5 mask */
+#define LOONGARCH_CSR_DB5CTRL 0x33a /* data breakpoint 5 control */
+#define LOONGARCH_CSR_DB5ASID 0x33b /* data breakpoint 5 asid */
+
+#define LOONGARCH_CSR_DB6ADDR 0x340 /* data breakpoint 6 address */
+#define LOONGARCH_CSR_DB6MASK 0x341 /* data breakpoint 6 mask */
+#define LOONGARCH_CSR_DB6CTRL 0x342 /* data breakpoint 6 control */
+#define LOONGARCH_CSR_DB6ASID 0x343 /* data breakpoint 6 asid */
+
+#define LOONGARCH_CSR_DB7ADDR 0x348 /* data breakpoint 7 address */
+#define LOONGARCH_CSR_DB7MASK 0x349 /* data breakpoint 7 mask */
+#define LOONGARCH_CSR_DB7CTRL 0x34a /* data breakpoint 7 control */
+#define LOONGARCH_CSR_DB7ASID 0x34b /* data breakpoint 7 asid */
+
+/*
+ * Data watchpoint control (DBnCTRL) bit layout. A slot is armed by setting a
+ * PLV-enable bit (selects which privilege levels trigger); cleared by writing
+ * DBnCTRL = 0. Size 0=8B,1=4B,2=2B,3=1B.
+ */
+#define CSR_DBC_LOADEN (1ul << 8) /* trigger on loads */
+#define CSR_DBC_STOREEN (1ul << 9) /* trigger on stores */
+#define CSR_DBC_SIZE_SHIFT 10
+#define CSR_DBC_SIZE_MASK (0x3ul << CSR_DBC_SIZE_SHIFT)
+#define CSR_DBC_LEN_8 (0x0ul << CSR_DBC_SIZE_SHIFT)
+#define CSR_DBC_LEN_4 (0x1ul << CSR_DBC_SIZE_SHIFT)
+#define CSR_DBC_LEN_2 (0x2ul << CSR_DBC_SIZE_SHIFT)
+#define CSR_DBC_LEN_1 (0x3ul << CSR_DBC_SIZE_SHIFT)
+#define CSR_DBC_PLV0 (1ul << 1)
+#define CSR_DBC_PLV3 (1ul << 4)
+/* MWPC low bits = number of data watchpoint slots. */
+#define CSR_MWPC_SLOTS 0x3f
+
+#define LOONGARCH_CSR_FWPC 0x380 /* instruction breakpoint config */
+#define LOONGARCH_CSR_FWPS 0x381 /* instruction breakpoint status */
+
+#define LOONGARCH_CSR_IB0ADDR 0x390 /* inst breakpoint 0 address */
+#define LOONGARCH_CSR_IB0MASK 0x391 /* inst breakpoint 0 mask */
+#define LOONGARCH_CSR_IB0CTRL 0x392 /* inst breakpoint 0 control */
+#define LOONGARCH_CSR_IB0ASID 0x393 /* inst breakpoint 0 asid */
+
+#define LOONGARCH_CSR_IB1ADDR 0x398 /* inst breakpoint 1 address */
+#define LOONGARCH_CSR_IB1MASK 0x399 /* inst breakpoint 1 mask */
+#define LOONGARCH_CSR_IB1CTRL 0x39a /* inst breakpoint 1 control */
+#define LOONGARCH_CSR_IB1ASID 0x39b /* inst breakpoint 1 asid */
+
+#define LOONGARCH_CSR_IB2ADDR 0x3a0 /* inst breakpoint 2 address */
+#define LOONGARCH_CSR_IB2MASK 0x3a1 /* inst breakpoint 2 mask */
+#define LOONGARCH_CSR_IB2CTRL 0x3a2 /* inst breakpoint 2 control */
+#define LOONGARCH_CSR_IB2ASID 0x3a3 /* inst breakpoint 2 asid */
+
+#define LOONGARCH_CSR_IB3ADDR 0x3a8 /* inst breakpoint 3 address */
+#define LOONGARCH_CSR_IB3MASK 0x3a9 /* breakpoint 3 mask */
+#define LOONGARCH_CSR_IB3CTRL 0x3aa /* inst breakpoint 3 control */
+#define LOONGARCH_CSR_IB3ASID 0x3ab /* inst breakpoint 3 asid */
+
+#define LOONGARCH_CSR_IB4ADDR 0x3b0 /* inst breakpoint 4 address */
+#define LOONGARCH_CSR_IB4MASK 0x3b1 /* inst breakpoint 4 mask */
+#define LOONGARCH_CSR_IB4CTRL 0x3b2 /* inst breakpoint 4 control */
+#define LOONGARCH_CSR_IB4ASID 0x3b3 /* inst breakpoint 4 asid */
+
+#define LOONGARCH_CSR_IB5ADDR 0x3b8 /* inst breakpoint 5 address */
+#define LOONGARCH_CSR_IB5MASK 0x3b9 /* inst breakpoint 5 mask */
+#define LOONGARCH_CSR_IB5CTRL 0x3ba /* inst breakpoint 5 control */
+#define LOONGARCH_CSR_IB5ASID 0x3bb /* inst breakpoint 5 asid */
+
+#define LOONGARCH_CSR_IB6ADDR 0x3c0 /* inst breakpoint 6 address */
+#define LOONGARCH_CSR_IB6MASK 0x3c1 /* inst breakpoint 6 mask */
+#define LOONGARCH_CSR_IB6CTRL 0x3c2 /* inst breakpoint 6 control */
+#define LOONGARCH_CSR_IB6ASID 0x3c3 /* inst breakpoint 6 asid */
+
+#define LOONGARCH_CSR_IB7ADDR 0x3c8 /* inst breakpoint 7 address */
+#define LOONGARCH_CSR_IB7MASK 0x3c9 /* inst breakpoint 7 mask */
+#define LOONGARCH_CSR_IB7CTRL 0x3ca /* inst breakpoint 7 control */
+#define LOONGARCH_CSR_IB7ASID 0x3cb /* inst breakpoint 7 asid */
+
+#define CSR_FWPS_SKIP_SHIFT 16
+#define CSR_FWPS_SKIP (_ULCAST_(1) << CSR_FWPS_SKIP_SHIFT)
+
+/*
+ * CSR_ECFG IM
+ */
+#define ECFGF(hwirq) (_ULCAST_(1) << hwirq)
+
+#define LOONGARCH_IOCSR_FEATURES 0x8
+#define IOCSRF_EXTIOI BIT_ULL(3)
+#define IOCSRF_CSRIPI BIT_ULL(4)
+#define IOCSRF_FREQSCALE BIT_ULL(6)
+#define IOCSRF_EIODECODE BIT_ULL(9)
+#define IOCSRF_FLATMODE BIT_ULL(10)
+#define IOCSRF_VM BIT_ULL(11)
+
+#define LOONGARCH_IOCSR_MISC_FUNC 0x420
+#define IOCSR_MISC_FUNC_EXT_IOI_EN BIT_ULL(48)
+
+/* PerCore CSR, only accessible by local cores */
+#define LOONGARCH_IOCSR_IPI_STATUS 0x1000
+#define LOONGARCH_IOCSR_IPI_EN 0x1004
+#define LOONGARCH_IOCSR_IPI_CLEAR 0x100c
+
+#define LOONGARCH_IOCSR_IPI_SEND 0x1040
+#define IOCSR_IPI_SEND_CPU_SHIFT 16
+#define IOCSR_IPI_SEND_BLOCKING BIT_U(31)
+
+#define LOONGARCH_IOCSR_MBUF_SEND 0x1048
+#define IOCSR_MBUF_SEND_BLOCKING BIT_ULL(31)
+#define IOCSR_MBUF_SEND_BOX_SHIFT 2
+#define IOCSR_MBUF_SEND_BOX_LO(box) (box << 1)
+#define IOCSR_MBUF_SEND_BOX_HI(box) ((box << 1) + 1)
+#define IOCSR_MBUF_SEND_CPU_SHIFT 16
+#define IOCSR_MBUF_SEND_BUF_SHIFT 32
+#define IOCSR_MBUF_SEND_H32_MASK 0xFFFFFFFF00000000ULL
+
+/* Register offset and bit definition for CSR access */
+
+#define LOONGARCH_IOCSR_EXTIOI_NODEMAP_BASE 0x14a0
+#define LOONGARCH_IOCSR_EXTIOI_IPMAP_BASE 0x14c0
+#define LOONGARCH_IOCSR_EXTIOI_EN_BASE 0x1600
+#define LOONGARCH_IOCSR_EXTIOI_BOUNCE_BASE 0x1680
+#define LOONGARCH_IOCSR_EXTIOI_ISR_BASE 0x1800
+#define LOONGARCH_IOCSR_EXTIOI_ROUTE_BASE 0x1c00
+
+#define read_csr_prcfg1() csr_read64(LOONGARCH_CSR_PRCFG1)
+#define read_csr_prcfg3() csr_read64(LOONGARCH_CSR_PRCFG3)
+
+
+/* ExStatus.ExcCode */
+#define EXCCODE_RSV 0 /* Reserved */
+#define EXCCODE_TLBL 1 /* TLB miss on a load */
+#define EXCCODE_TLBS 2 /* TLB miss on a store */
+#define EXCCODE_TLBI 3 /* TLB miss on a ifetch */
+#define EXCCODE_TLBM 4 /* TLB modified fault */
+#define EXCCODE_TLBNR 5 /* TLB Read-Inhibit exception */
+#define EXCCODE_TLBNX 6 /* TLB Execution-Inhibit exception */
+#define EXCCODE_TLBPE 7 /* TLB Privilege Error */
+#define EXCCODE_ADE 8 /* Address Error */
+#define EXCCODE_ALE 9 /* Unalign Access */
+#define EXCCODE_BCE 10 /* Bounds Check Error */
+#define EXCCODE_SYS 11 /* System call */
+#define EXCCODE_BP 12 /* Breakpoint */
+#define EXCCODE_INE 13 /* Inst. Not Exist */
+#define EXCCODE_IPE 14 /* Inst. Privileged Error */
+#define EXCCODE_FPDIS 15 /* FPU Disabled */
+#define EXCCODE_LSXDIS 16 /* LSX Disabled */
+#define EXCCODE_LASXDIS 17 /* LASX Disabled */
+#define EXCCODE_FPE 18 /* Floating Point Exception */
+#define EXCCODE_WATCH 19 /* WatchPoint Exception */
+#define EXCCODE_BTDIS 20 /* Binary Trans. Disabled */
+
+/*
+ * X the exception cause indicator
+ * E the exception enable
+ * S the sticky/flag bit
+ */
+#define FPU_CSR_ALL_X 0x1f000000
+#define FPU_CSR_INV_X 0x10000000
+#define FPU_CSR_DIV_X 0x08000000
+#define FPU_CSR_OVF_X 0x04000000
+#define FPU_CSR_UDF_X 0x02000000
+#define FPU_CSR_INE_X 0x01000000
+
+#define FPU_CSR_ALL_E 0x0000001f
+
+#define GRLEN 64
+#define GRLEN_BYTES (GRLEN / 8)
+
+#endif /* !_MACHINE_LOONGARCH_H_ */
diff --git a/sys/loongarch/include/machdep.h b/sys/loongarch/include/machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/machdep.h
@@ -0,0 +1,68 @@
+/*-
+ * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_MACHDEP_H_
+#define _MACHINE_MACHDEP_H_
+
+struct loongarch_bootparams {
+ vm_offset_t kern_pgd; /* Kernel L1 base */
+ vm_offset_t kern_phys; /* Kernel base (physical) addr */
+ vm_offset_t kern_stack;
+ vm_offset_t dtbp_virt; /* Device tree blob virtual addr */
+ vm_offset_t dtbp_phys; /* Device tree blob physical addr */
+ vm_offset_t modulep; /* loader(8) metadata */
+};
+
+void initloongarch(struct loongarch_bootparams *);
+void ged_init(void);
+
+/* Early serial output via the DMW0 UART window; safe before cninit() and
+ * across pmap_bootstrap (independent of the page-table devmap). */
+void loongarch_early_puts(const char *);
+void loongarch_early_put_hex(uint64_t);
+vm_paddr_t loongarch_efi_acpi_rsdp(void); /* ACPI RSDP from EFI config table */
+
+/* LoongArch ACPI MADT (APIC) walkers -- defined when DEV_ACPI. */
+int loongarch_acpi_madt_cpu_count(void);
+void loongarch_acpi_madt_foreach_cpu(bool (*)(u_int));
+
+/*
+ * Look up the first MADT BIO_PIC (PCH-PIC) subtable and return its
+ * parameters: the MMIO register base (Address), window size (Size) and
+ * the GSI base for this instance. Returns true on success.
+ */
+bool loongarch_acpi_pch_pic_info(vm_paddr_t *addr, u_int *size,
+ u_int *gsi_base);
+
+/*
+ * Look up the first MADT MSI_PIC (PCH-MSI) subtable: the doorbell MMIO
+ * address and the EIOINTC vector range [start, start+count) used for MSI.
+ * Returns true on success.
+ */
+bool loongarch_acpi_pch_msi_info(vm_paddr_t *doorbell, u_int *start,
+ u_int *count);
+
+#endif /* _MACHINE_MACHDEP_H_ */
diff --git a/sys/loongarch/include/md_var.h b/sys/loongarch/include/md_var.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/md_var.h
@@ -0,0 +1,52 @@
+/*-
+ * Copyright (c) 1995 Bruce D. Evans.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: FreeBSD: src/sys/i386/include/md_var.h,v 1.40 2001/07/12
+ */
+
+#ifndef _MACHINE_MD_VAR_H_
+#define _MACHINE_MD_VAR_H_
+
+extern long Maxmem;
+extern char sigcode[];
+extern int szsigcode;
+extern u_long elf_hwcap;
+extern register_t mvendorid;
+extern register_t marchid;
+extern register_t mimpid;
+
+/* Supervisor-mode extension support */
+extern bool has_sstc;
+extern bool has_sscofpmf;
+
+struct dumperinfo;
+struct minidumpstate;
+
+int cpu_minidumpsys(struct dumperinfo *, const struct minidumpstate *);
+
+#endif /* !_MACHINE_MD_VAR_H_ */
diff --git a/sys/loongarch/include/memdev.h b/sys/loongarch/include/memdev.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/memdev.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2004 Mark R V Murray
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer
+ * in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_MEMDEV_H_
+#define _MACHINE_MEMDEV_H_
+
+#define CDEV_MINOR_MEM 0
+#define CDEV_MINOR_KMEM 1
+
+d_open_t memopen;
+d_read_t memrw;
+d_ioctl_t memioctl_md;
+d_mmap_t memmmap;
+
+#endif /* _MACHINE_MEMDEV_H_ */
diff --git a/sys/loongarch/include/metadata.h b/sys/loongarch/include/metadata.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/metadata.h
@@ -0,0 +1,56 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Mitchell Horne <mhorne@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_METADATA_H_
+#define _MACHINE_METADATA_H_
+
+#define MODINFOMD_DTBP 0x1001
+#define MODINFOMD_EFI_MAP 0x1002
+#define MODINFOMD_EFI_FB 0x1003
+
+struct efi_map_header {
+ size_t memory_size;
+ size_t descriptor_size;
+ uint32_t descriptor_version;
+};
+
+/*
+ * Placeholder for now
+ */
+struct efi_fb {
+ uint64_t fb_addr;
+ uint64_t fb_size;
+ uint32_t fb_height;
+ uint32_t fb_width;
+ uint32_t fb_stride;
+ uint32_t fb_mask_red;
+ uint32_t fb_mask_green;
+ uint32_t fb_mask_blue;
+ uint32_t fb_mask_reserved;
+};
+
+#endif /* !_MACHINE_METADATA_H_ */
diff --git a/sys/loongarch/include/minidump.h b/sys/loongarch/include/minidump.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/minidump.h
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2006 Peter Wemm
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * From i386: FreeBSD: 157909 2006-04-21 04:28:43Z peter
+ */
+
+#ifndef _MACHINE_MINIDUMP_H_
+#define _MACHINE_MINIDUMP_H_ 1
+
+#define MINIDUMP_MAGIC "minidump FreeBSD/loongarch"
+#define MINIDUMP_VERSION 2
+
+struct minidumphdr {
+ char magic[32];
+ uint32_t version;
+ uint32_t msgbufsize;
+ uint32_t bitmapsize;
+ uint32_t pmapsize;
+ uint64_t kernbase;
+ uint64_t dmapphys;
+ uint64_t dmapbase;
+ uint64_t dmapend;
+ uint32_t dumpavailsize;
+};
+
+#endif /* _MACHINE_MINIDUMP_H_ */
diff --git a/sys/loongarch/include/ofw_machdep.h b/sys/loongarch/include/ofw_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/ofw_machdep.h
@@ -0,0 +1,37 @@
+/*-
+ * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_OFW_MACHDEP_H_
+#define _MACHINE_OFW_MACHDEP_H_
+
+#include <vm/vm.h>
+
+typedef uint32_t cell_t;
+struct mem_region {
+ vm_offset_t mr_start;
+ vm_size_t mr_size;
+};
+
+#endif /* _MACHINE_OFW_MACHDEP_H_ */
diff --git a/sys/loongarch/include/param.h b/sys/loongarch/include/param.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/param.h
@@ -0,0 +1,108 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: @(#)param.h 5.8 (Berkeley) 6/28/91
+ */
+
+#ifndef _MACHINE_PARAM_H_
+#define _MACHINE_PARAM_H_
+
+/*
+ * Machine dependent constants for LoongArch.
+ */
+
+#include <sys/_align.h>
+
+#define STACKALIGNBYTES (16 - 1)
+
+#define __HAVE_STATIC_DEVMAP
+
+#ifndef MACHINE
+#define MACHINE "loongarch"
+#endif
+#ifndef MACHINE_ARCH
+#define MACHINE_ARCH "loongarch64"
+#endif
+
+#ifdef SMP
+#ifndef MAXCPU
+#define MAXCPU 16
+#endif
+#else
+#define MAXCPU 1
+#endif
+
+#ifndef MAXMEMDOM
+#define MAXMEMDOM 1
+#endif
+
+#define ALIGNBYTES _ALIGNBYTES
+#define ALIGN(p) _ALIGN(p)
+/*
+ * ALIGNED_POINTER is a boolean macro that checks whether an address
+ * is valid to fetch data elements of type t from on this architecture.
+ * This does not reflect the optimal alignment, just the possibility
+ * (within reasonable limits).
+ */
+#define ALIGNED_POINTER(p, t) \
+ ((((u_long)(p)) & (sizeof(t) - 1)) == 0)
+
+/*
+ * CACHE_LINE_SIZE is the compile-time maximum cache line size for an
+ * architecture. It should be used with appropriate caution.
+ */
+#define CACHE_LINE_SHIFT 6
+#define CACHE_LINE_SIZE (1 << CACHE_LINE_SHIFT)
+
+#define PAGE_SHIFT_4K 12
+#define PAGE_SIZE_4K (1 << PAGE_SHIFT_4K)
+
+#define PAGE_SHIFT_16K 14
+#define PAGE_SIZE_16K (1 << PAGE_SHIFT_16K)
+
+#define PAGE_SHIFT PAGE_SHIFT_4K
+#define PAGE_SIZE (1 << PAGE_SHIFT)
+#define PAGE_MASK (PAGE_SIZE - 1)
+
+#define MAXPAGESIZES 3 /* maximum number of supported page sizes */
+
+#ifndef KSTACK_PAGES
+#define KSTACK_PAGES 8 /* pages of kernel stack (with pcb) */
+#endif
+
+#define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */
+#define PCPU_PAGES 1
+
+/*
+ * Mach derived conversion macros
+ */
+
+#define loongarch_btop(x) ((unsigned long)(x) >> PAGE_SHIFT)
+#define loongarch_ptob(x) ((unsigned long)(x) << PAGE_SHIFT)
+
+#endif /* !_MACHINE_PARAM_H_ */
diff --git a/sys/loongarch/include/pcb.h b/sys/loongarch/include/pcb.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/pcb.h
@@ -0,0 +1,70 @@
+/*-
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_PCB_H_
+#define _MACHINE_PCB_H_
+
+#ifndef LOCORE
+
+struct trapframe;
+
+struct pcb {
+ uint64_t r0;
+ uint64_t ra;
+ uint64_t tp;
+ uint64_t sp;
+ uint64_t a[8];
+ uint64_t t[9];
+ uint64_t r21;
+ uint64_t fp;
+ uint64_t s[9];
+
+#define pcb_a a
+#define pcb_t t
+#define pcb_s s
+#define pcb_ra ra
+#define pcb_sp sp
+ uint64_t fa[8];
+ uint64_t ft[16];
+ uint64_t fs[8];
+ uint64_t pcb_fcsr0;
+ uint64_t pcb_a0;
+ uint64_t pcb_fpflags;
+#define PCB_FP_STARTED 0x1
+#define PCB_FP_LSX_STARTED 0x2
+#define PCB_FP_LASX_STARTED 0x4
+#define PCB_FP_USERMASK 0x1
+ vm_offset_t pcb_onfault; /* Copyinout fault handler */
+};
+
+#ifdef _KERNEL
+void makectx(struct trapframe *tf, struct pcb *pcb);
+int savectx(struct pcb *pcb) __returns_twice;
+#endif
+
+#endif /* !LOCORE */
+
+#endif /* !_MACHINE_PCB_H_ */
diff --git a/sys/loongarch/include/pci_cfgreg.h b/sys/loongarch/include/pci_cfgreg.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/pci_cfgreg.h
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ */
+
+/*
+ * Stub PCI config register header for ACPI compilation. LoongArch PCI
+ * config space access via ACPI MCFG will be implemented when the ACPI
+ * PCI bus driver is wired in.
+ */
+
+#ifndef _MACHINE_PCI_CFGREG_H
+#define _MACHINE_PCI_CFGREG_H
+
+#endif /* !_MACHINE_PCI_CFGREG_H */
diff --git a/sys/loongarch/include/pcpu.h b/sys/loongarch/include/pcpu.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/pcpu.h
@@ -0,0 +1,79 @@
+/*-
+ * Copyright (c) 1999 Luoqi Chen <luoqi@freebsd.org>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27
+ */
+
+#ifndef _MACHINE_PCPU_H_
+#define _MACHINE_PCPU_H_
+
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+
+/* Keep in sync with db_show_mdpcpu() */
+#define PCPU_MD_FIELDS \
+ struct pmap *pc_curpmap; /* Currently active pmap */ \
+ uint32_t pc_pending_ipis; /* IPIs pending to this CPU */ \
+ uint64_t pc_kernel_sp; /* kernel mode stack */ \
+ uint64_t pc_asid_value; /* current ASID value */ \
+ uint32_t pc_asid_mask; /* ASID bit mask */ \
+ u_int pc_acpi_id; /* ACPI CPU id */ \
+ char __pad[32]; /* Pad to factor of PAGE_SIZE */
+
+#ifdef _KERNEL
+
+struct pcb;
+struct pcpu;
+
+static inline struct pcpu *
+get_pcpu(void)
+{
+ struct pcpu *pcpu;
+
+ __asm __volatile("move %0, $r21" : "=&r"(pcpu));
+
+ return (pcpu);
+}
+
+static inline struct thread *
+get_curthread(void)
+{
+ struct thread *td;
+ __asm __volatile("ld.d %0, $r21, 0" : "=&r"(td));
+
+ return (td);
+}
+
+#define curthread get_curthread()
+
+#define PCPU_GET(member) (get_pcpu()->pc_ ## member)
+#define PCPU_ADD(member, value) (get_pcpu()->pc_ ## member += (value))
+#define PCPU_PTR(member) (&get_pcpu()->pc_ ## member)
+#define PCPU_SET(member,value) (get_pcpu()->pc_ ## member = (value))
+
+#endif /* _KERNEL */
+
+#endif /* !_MACHINE_PCPU_H_ */
diff --git a/sys/loongarch/include/pcpu_aux.h b/sys/loongarch/include/pcpu_aux.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/pcpu_aux.h
@@ -0,0 +1,53 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 The FreeBSD Foundation
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_PCPU_AUX_H_
+#define _MACHINE_PCPU_AUX_H_
+
+#ifndef _KERNEL
+#error "Not for userspace"
+#endif
+
+#ifndef _SYS_PCPU_H_
+#error "Do not include machine/pcpu_aux.h directly"
+#endif
+
+/*
+ * To minimize memory waste in per-cpu UMA zones, the page size should
+ * be a multiple of the size of struct pcpu.
+ */
+_Static_assert(PAGE_SIZE % sizeof(struct pcpu) == 0, "fix pcpu size");
+_Static_assert(offsetof(struct pcpu, __pad) +
+ sizeof(((struct pcpu *)0)->__pad) == sizeof(struct pcpu),
+ "fix pcpu padding");
+
+extern struct pcpu __pcpu[];
+
+#endif /* _MACHINE_PCPU_AUX_H_ */
diff --git a/sys/loongarch/include/pmap.h b/sys/loongarch/include/pmap.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/pmap.h
@@ -0,0 +1,272 @@
+/*-
+ * Copyright (c) 1991 Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * the Systems Programming Group of the University of Utah Computer
+ * Science Department and William Jolitz of UUNET Technologies Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_PMAP_H_
+#define _MACHINE_PMAP_H_
+
+#include <machine/pte.h>
+
+#ifndef LOCORE
+
+#include <sys/queue.h>
+#include <sys/_cpuset.h>
+#include <sys/_lock.h>
+#include <sys/_mutex.h>
+#include <sys/_pv_entry.h>
+
+#include <vm/_vm_radix.h>
+#include <sys/pcpu.h>
+#include <machine/cpu.h>
+#include <machine/tlb.h>
+
+#ifdef _KERNEL
+
+#define vtophys(va) pmap_kextract((vm_offset_t)(va))
+
+#endif
+
+#define pmap_page_get_memattr(m) ((m)->md.pv_memattr)
+#define pmap_page_is_write_mapped(m) \
+ (((m)->a.flags & PGA_WRITEABLE) != 0)
+void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma);
+#define pmap_map_delete(pmap, sva, eva) \
+ pmap_remove(pmap, sva, eva)
+
+/*
+ * Pmap stuff
+ */
+
+struct md_page {
+ TAILQ_HEAD(,pv_entry) pv_list;
+ int pv_gen;
+ vm_memattr_t pv_memattr;
+};
+
+/*
+ * This structure is used to hold a virtual<->physical address
+ * association and is used mostly by bootstrap code
+ */
+struct pv_addr {
+ SLIST_ENTRY(pv_addr) pv_list;
+ vm_offset_t pv_va;
+ vm_paddr_t pv_pa;
+};
+
+struct pmap {
+ struct mtx pm_mtx;
+ struct pmap_statistics pm_stats; /* pmap statictics */
+ pd_entry_t *pm_top; /* top-level page table page */
+ u_long pm_pgdl;
+ /* page global directory base address lower half*/
+ u_long pm_pgdh;
+ /* page global directory base address higher half*/
+ cpuset_t pm_active; /* active on cpus */
+ TAILQ_HEAD(,pv_chunk) pm_pvchunk;
+ /* list of mappings in pmap */
+ struct vm_radix pm_root;
+ uint64_t asid[MAXCPU]; /* current asid */
+};
+
+typedef struct pmap *pmap_t;
+
+#ifdef _KERNEL
+extern struct pmap kernel_pmap_store;
+#define kernel_pmap (&kernel_pmap_store)
+#define pmap_kernel() kernel_pmap
+
+#define PMAP_ASSERT_LOCKED(pmap) \
+ mtx_assert(&(pmap)->pm_mtx, MA_OWNED)
+#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)
+#define PMAP_LOCK_ASSERT(pmap, type) \
+ mtx_assert(&(pmap)->pm_mtx, (type))
+#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)
+#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \
+ NULL, MTX_DEF | MTX_DUPOK)
+#define PMAP_OWNED(pmap) mtx_owned(&(pmap)->pm_mtx)
+#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)
+#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)
+#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)
+
+extern vm_offset_t virtual_avail;
+extern vm_offset_t virtual_end;
+
+/*
+ * Macros to test if a mapping is mappable with an L1 Section mapping
+ * or an L2 Large Page mapping.
+ */
+#define L1_MAPPABLE_P(va, pa, size) \
+ ((((va) | (pa)) & L1_OFFSET) == 0 && (size) >= L1_SIZE)
+
+enum pmap_mode {
+ PMAP_MODE_LA48,
+};
+
+extern enum pmap_mode pmap_mode;
+
+/* Check if an address resides in a mappable region. */
+#define VIRT_IS_VALID(va) \
+ ((va) < VM_MAX_USER_ADDRESS || (va) >= VM_MIN_KERNEL_ADDRESS)
+
+struct thread;
+
+#define pmap_vm_page_alloc_check(m)
+
+void pmap_activate_boot(pmap_t);
+void pmap_activate_sw(struct thread *);
+void pmap_bootstrap(vm_offset_t, vm_paddr_t, vm_size_t);
+int pmap_change_attr(void *va, vm_size_t size, int mode);
+void pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa,
+ int mode);
+void pmap_kenter_device(vm_offset_t, vm_size_t, vm_paddr_t);
+vm_paddr_t pmap_kextract(vm_offset_t va);
+void pmap_kremove(vm_offset_t);
+void pmap_kremove_device(vm_offset_t, vm_size_t);
+void *pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma);
+bool pmap_page_is_mapped(vm_page_t m);
+bool pmap_ps_enabled(pmap_t);
+
+void *pmap_mapdev(vm_paddr_t, vm_size_t);
+void *pmap_mapbios(vm_paddr_t, vm_size_t);
+void pmap_unmapdev(void *, vm_size_t);
+void pmap_unmapbios(void *, vm_size_t);
+void pmap_preboot_map_attr(vm_paddr_t, vm_offset_t, vm_size_t, vm_prot_t,
+ vm_memattr_t);
+
+bool pmap_map_io_transient(vm_page_t *, void **, int, bool);
+void pmap_unmap_io_transient(vm_page_t *, void **, int, bool);
+
+bool pmap_get_tables(pmap_t, vm_offset_t, pd_entry_t **, pd_entry_t **,
+ pt_entry_t **);
+
+int pmap_fault(pmap_t, vm_offset_t, vm_prot_t);
+bool pmap_update_tlb(pmap_t, vm_offset_t);
+
+static inline int
+pmap_vmspace_copy(pmap_t dst_pmap __unused, pmap_t src_pmap __unused)
+{
+
+ return (0);
+}
+
+static inline void
+asid_init(int cpu)
+{
+ struct pcpu *pc = pcpu_find(cpu);
+
+ pc->pc_asid_mask = cpu_desc[cpu].cpuinfo.asid_mask;
+ pc->pc_asid_value = pc->pc_asid_mask + 1;
+ /* initial version: 1, asid: 0 */
+}
+
+static inline void
+pt_walker_init(int cpu)
+{
+ uint32_t pwcl = PWCL;
+ uint32_t pwch = PWCH;
+
+ if (cpu_desc[cpu].cpuinfo.hwcap & HWCAP_LOONGARCH_PTW)
+ pwch |= CSR_PWCTL1_PTW;
+
+ csr_write32(pwcl, LOONGARCH_CSR_PWCTL0);
+ csr_write32(pwch, LOONGARCH_CSR_PWCTL1);
+ csr_write32(curcpu, LOONGARCH_CSR_TMID);
+}
+
+static inline uint64_t
+asid_version_mask(int cpu)
+{
+ struct pcpu *pc = pcpu_find(cpu);
+
+ return (~pc->pc_asid_mask);
+}
+
+static inline bool
+pmap_asid_valid(pmap_t pmap, int cpu)
+{
+ struct pcpu *pc;
+ uint64_t asid;
+
+ pc = pcpu_find(cpu);
+ asid = pc->pc_asid_value;
+
+ if ((pmap->asid[cpu] ^ asid) & asid_version_mask(cpu))
+ return false;
+
+ return true;
+}
+
+static inline void
+pmap_new_asid(pmap_t pmap, int cpu, bool *flush)
+{
+ struct pcpu *pc;
+ uint64_t asid;
+
+ pc = pcpu_find(cpu);
+ asid = pc->pc_asid_value + 1;
+
+ if ((asid & pc->pc_asid_mask) == 0)
+ *flush = true;
+
+ pc->pc_asid_value = pmap->asid[cpu] = asid;
+}
+
+static inline uint32_t
+pmap_get_asid(pmap_t pmap, int cpu)
+{
+ struct pcpu *pc = pcpu_find(cpu);
+ return (pmap->asid[cpu] & pc->pc_asid_mask);
+}
+
+static inline void
+pmap_update_asid(pmap_t pmap, int cpu)
+{
+ struct pcpu *pc = pcpu_find(cpu);
+ uint32_t asid = csr_read32(LOONGARCH_CSR_ASID) & pc->pc_asid_mask;
+
+ if (asid == pmap_get_asid(pmap, cpu)) {
+ bool need_tlb_flush = false;
+ pmap_new_asid(pmap, cpu, &need_tlb_flush);
+ csr_write32(pmap_get_asid(pmap, cpu), LOONGARCH_CSR_ASID);
+ if (need_tlb_flush)
+ flush_tlb_user();
+ } else {
+ /* mark as invalid */
+ pmap->asid[cpu] = 0;
+ }
+}
+
+#endif /* _KERNEL */
+
+#endif /* !LOCORE */
+
+#endif /* !_MACHINE_PMAP_H_ */
diff --git a/sys/loongarch/include/pmc_mdep.h b/sys/loongarch/include/pmc_mdep.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/pmc_mdep.h
@@ -0,0 +1,60 @@
+/*-
+ * Copyright (c) 2009 Rui Paulo <rpaulo@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_PMC_MDEP_H_
+#define _MACHINE_PMC_MDEP_H_
+
+#define PMC_MDEP_CLASS_INDEX_LOONGARCH 1
+#include <dev/hwpmc/hwpmc_loongarch.h>
+
+union pmc_md_op_pmcallocate {
+ uint64_t __pad[4];
+};
+
+/* Logging */
+#define PMCLOG_READADDR PMCLOG_READ64
+#define PMCLOG_EMITADDR PMCLOG_EMIT64
+
+#ifdef _KERNEL
+union pmc_md_pmc {
+ struct pmc_md_loongarch_pmc pm_loongarch;
+};
+
+#define PMC_IN_KERNEL_STACK(va) \
+ kstack_contains(curthread, (va), sizeof(va))
+#define PMC_IN_KERNEL(va) INKERNEL((va))
+#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS)
+#define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_ra)
+#define PMC_TRAPFRAME_TO_FP(TF) (0) /* stub */
+
+/*
+ * Prototypes
+ */
+struct pmc_mdep *pmc_loongarch_initialize(void);
+void pmc_loongarch_finalize(struct pmc_mdep *_md);
+#endif /* _KERNEL */
+
+#endif /* !_MACHINE_PMC_MDEP_H_ */
diff --git a/sys/loongarch/include/proc.h b/sys/loongarch/include/proc.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/proc.h
@@ -0,0 +1,52 @@
+/*-
+ * Copyright (c) 1991 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: @(#)proc.h 7.1 (Berkeley) 5/15/91
+ * from: FreeBSD: src/sys/i386/include/proc.h,v 1.11 2001/06/29
+ */
+
+#ifndef _MACHINE_PROC_H_
+#define _MACHINE_PROC_H_
+
+#include <machine/watch.h>
+
+struct mdthread {
+ int md_spinlock_count; /* (k) */
+ register_t md_saved_crmd_ie; /* (k) */
+ struct la_watch_image md_watch; /* (k) desired data-watchpoint slots */
+ uint64_t md_ss_addr; /* (k) single-step IB0 address */
+ int md_ss_active; /* (k) single-step armed */
+};
+
+struct mdproc {
+ int dummy;
+};
+
+#define KINFO_PROC_SIZE 1088
+
+#endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/loongarch/include/procctl.h b/sys/loongarch/include/procctl.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/procctl.h
@@ -0,0 +1,3 @@
+/*-
+ * This file is in the public domain.
+ */
diff --git a/sys/loongarch/include/profile.h b/sys/loongarch/include/profile.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/profile.h
@@ -0,0 +1,75 @@
+/*-
+ * SPDX-License-Identifier: MIT-CMU
+ *
+ * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
+ * All rights reserved.
+ *
+ * Author: Chris G. Demetriou
+ *
+ * Permission to use, copy, modify and distribute this software and
+ * its documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ *
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
+ * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ *
+ * Carnegie Mellon requests users of this software to return to
+ *
+ * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
+ * School of Computer Science
+ * Carnegie Mellon University
+ * Pittsburgh PA 15213-3890
+ *
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ *
+ * from: NetBSD: profile.h,v 1.9 1997/04/06 08:47:37 cgd Exp
+ * from: FreeBSD: src/sys/alpha/include/profile.h,v 1.4 1999/12/29
+ */
+
+#ifndef _MACHINE_PROFILE_H_
+#define _MACHINE_PROFILE_H_
+
+#define FUNCTION_ALIGNMENT 32
+
+typedef u_long fptrdiff_t;
+
+#ifdef _KERNEL
+
+#include <machine/cpufunc.h>
+
+#define _MCOUNT_DECL void mcount
+#define MCOUNT
+
+#define MCOUNT_DECL(s) register_t s;
+#define MCOUNT_ENTER(s) {s = intr_disable(); }
+#define MCOUNT_EXIT(s) {intr_restore(s); }
+
+void bintr(void);
+void btrap(void);
+void eintr(void);
+void user(void);
+
+#define MCOUNT_FROMPC_USER(pc) \
+ ((pc < (uintfptr_t)VM_MAXUSER_ADDRESS) ? (uintfptr_t)user : pc)
+
+#define MCOUNT_FROMPC_INTR(pc) \
+ ((pc >= (uintfptr_t)btrap && pc < (uintfptr_t)eintr) ? \
+ ((pc >= (uintfptr_t)bintr) ? (uintfptr_t)bintr : \
+ (uintfptr_t)btrap) : ~0UL)
+
+void mcount(uintfptr_t frompc, uintfptr_t selfpc);
+
+#else /* !_KERNEL */
+
+typedef __uintfptr_t uintfptr_t;
+
+#define _MCOUNT_DECL void mcount
+#define MCOUNT
+
+#endif /* _KERNEL */
+
+#endif /* !_MACHINE_PROFILE_H_ */
diff --git a/sys/loongarch/include/psl.h b/sys/loongarch/include/psl.h
new file mode 100644
diff --git a/sys/loongarch/include/pte.h b/sys/loongarch/include/pte.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/pte.h
@@ -0,0 +1,180 @@
+/*-
+ * Copyright (c) 2014 Andrew Turner
+ * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_PTE_H_
+#define _MACHINE_PTE_H_
+
+#ifndef LOCORE
+typedef uint64_t pd_entry_t; /* page directory entry */
+typedef uint64_t pt_entry_t; /* page table entry */
+typedef uint64_t pn_t; /* page number */
+#endif
+
+/*
+ * Uncomment TLB_LIFECYCLE_DEBUG to trace the full TLB lifecycle:
+ * invalidation, installation (tlbfill / tlbwr), and stale-hit
+ * diagnostics during page faults.
+ */
+#ifdef TLB_LIFECYCLE_DEBUG
+#define TLB_DBG(...) printf(__VA_ARGS__)
+#else
+#define TLB_DBG(...) do {} while (0)
+#endif
+
+/* extra 8M */
+#define LOCORE_EXTRA_MAP_LEN (8 * 1024 * 1024)
+
+#if PAGE_SIZE == PAGE_SIZE_4K
+#define L0_SHIFT 39
+#define L1_SHIFT 30
+#define L2_SHIFT 21
+#define L3_SHIFT 12
+#elif PAGE_SIZE == PAGE_SIZE_16K
+#define L0_SHIFT 47
+#define L1_SHIFT 36
+#define L2_SHIFT 25
+#define L3_SHIFT 14
+#else
+#error Unsupported page size
+#endif
+
+/* Level 0 table, 512GiB/128TiB per entry */
+#define L0_SIZE (1UL << L0_SHIFT)
+#define L0_OFFSET (L0_SIZE - 1)
+
+/* Level 1 table, 1GiB/64GiB per entry */
+#define L1_SIZE (1UL << L1_SHIFT)
+#define L1_OFFSET (L1_SIZE - 1)
+
+/* Level 2 table, 2MiB/32MiB per entry */
+#define L2_SIZE (1UL << L2_SHIFT)
+#define L2_OFFSET (L2_SIZE - 1)
+
+/* Level 3 table, 4KiB/16KiB per entry */
+#define L3_SIZE (1UL << L3_SHIFT)
+#define L3_OFFSET (L3_SIZE - 1)
+
+#if PAGE_SIZE == PAGE_SIZE_4K
+#define L0_ENTRIES_SHIFT 9
+#define Ln_ENTRIES_SHIFT 9
+#elif PAGE_SIZE == PAGE_SIZE_16K
+#define L0_ENTRIES_SHIFT 1
+#define Ln_ENTRIES_SHIFT 11
+#else
+#error Unsupported page size
+#endif
+
+#define L0_ENTRIES (1 << L0_ENTRIES_SHIFT)
+#define L0_ADDR_MASK (L0_ENTRIES - 1)
+
+#define Ln_ENTRIES (1 << Ln_ENTRIES_SHIFT)
+#define Ln_ADDR_MASK (Ln_ENTRIES - 1)
+
+/* Bits 11:9 are reserved */
+#define PTE_D (1 << 1) /* Dirty */
+#define PTE_G (1 << 6) /* Global */
+#define PTE_K (0 << 2) /* kernel, PLV 0 */
+#define PTE_U (0x3 << 2) /* User, PLV 3 */
+#define PTE_P (1 << 7) /* Present */
+#define PTE_CC (1 << 4) /* coherent cached */
+#define PTE_WUC (2 << 4) /* weakly-order uncached */
+#define PTE_SUC (0 << 4) /* strongly-order uncached */
+#define PTE_MAT_MASK (PTE_CC | PTE_WUC) /* memory attribute (MAT) field */
+#define PTE_NX (1UL << 62) /* Non Execute */
+#define PTE_NR (1UL << 61) /* Non Read */
+#define PTE_W (1 << 8) /* Write */
+#define PTE_V (1 << 0) /* Valid */
+#define PTE_A PTE_V
+#define PTE_HUGE_G (1 << 12) /* Huge page G bit */
+#define PTE_HUGE (1 << 6) /* Huge page marker */
+#define PTE_READABLE (PTE_V)
+#define PTE_WRITEABLE (PTE_D | PTE_W)
+
+#define PTE_SW_WIRED (1 << 9)
+#define PTE_SW_MANAGED (1 << 10)
+
+#define PTE_RX(pte) (((pte & PTE_NR) == 0) || ((pte & PTE_NX) == 0))
+#define PTE_RWX(pte) (PTE_RX(pte) || (((pte) & PTE_W) != 0))
+
+#define PMD_KERN (PTE_K | PTE_READABLE | PTE_WRITEABLE | PTE_CC | PTE_P)
+#define PMD_KERN_SUC (PTE_K | PTE_READABLE | PTE_WRITEABLE | PTE_SUC | PTE_P)
+#define PMD_KERN_HUGE (PMD_KERN | PTE_HUGE | PTE_HUGE_G)
+#define PMD_KERN_SUC_HUGE (PMD_KERN_SUC | PTE_HUGE | PTE_HUGE_G)
+
+
+#define PTE_KERN (PTE_K | PTE_READABLE | PTE_WRITEABLE | PTE_CC | PTE_P | PTE_G)
+#define PTE_KERN_SUC (PTE_K | PTE_READABLE | PTE_WRITEABLE | \
+ PTE_SUC | PTE_P | PTE_G)
+#define PTE_HI_MASK 0xffff000000000000ULL
+
+#define PTE_PROMOTE (PTE_V | PTE_D | PTE_G | PTE_U | PTE_SW_MANAGED | \
+ PTE_SW_WIRED | PTE_W | PTE_NX)
+
+#define PPNS L3_SHIFT
+#define PAGE_PFN_SHIFT L3_SHIFT
+
+#define PTE_SIZE 8
+#define PTE_WIDTH 0 /* 64 bit */
+
+#define PT_BASE L3_SHIFT
+#define PT_WIDTH Ln_ENTRIES_SHIFT
+#define DIR1_BASE L2_SHIFT
+#define DIR1_WIDTH Ln_ENTRIES_SHIFT
+#define DIR2_BASE L1_SHIFT
+#define DIR2_WIDTH Ln_ENTRIES_SHIFT
+#define DIR3_BASE L0_SHIFT
+#define DIR3_WIDTH L0_ENTRIES_SHIFT
+#define DIR4_BASE 48 /* 48 bit address */
+#define DIR4_WIDTH 0
+
+/* PWCL */
+#define PT_BASE_SHIFT 0
+#define PT_WIDTH_SHIFT 5
+#define DIR1_BASE_SHIFT 10
+#define DIR1_WIDTH_SHIFT 15
+#define DIR2_BASE_SHIFT 20
+#define DIR2_WIDTH_SHIFT 25
+#define PTE_WIDTH_SHIFT 30
+
+/* PWCH */
+#define DIR3_BASE_SHIFT 0
+#define DIR3_WIDTH_SHIFT 6
+#define DIR4_BASE_SHIFT 12
+#define DIR4_WIDTH_SHIFT 18
+
+#define PWCL \
+((PTE_WIDTH << PTE_WIDTH_SHIFT)|(DIR2_WIDTH << DIR2_WIDTH_SHIFT) \
+|(DIR2_BASE << DIR2_BASE_SHIFT)|(DIR1_WIDTH << DIR1_WIDTH_SHIFT) \
+|(DIR1_BASE << DIR1_BASE_SHIFT)|(PT_WIDTH << PT_WIDTH_SHIFT) \
+|PT_BASE << PT_BASE_SHIFT)
+
+#define PWCH \
+((DIR3_WIDTH << DIR3_WIDTH_SHIFT)|DIR3_BASE << DIR3_BASE_SHIFT)
+
+#endif /* !_MACHINE_PTE_H_ */
+
+/* End of pte.h */
diff --git a/sys/loongarch/include/ptrace.h b/sys/loongarch/include/ptrace.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/ptrace.h
@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ */
+
+#ifndef _MACHINE_PTRACE_H_
+#define _MACHINE_PTRACE_H_
+
+#endif /* !_MACHINE_PTRACE_H_ */
diff --git a/sys/loongarch/include/reg.h b/sys/loongarch/include/reg.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/reg.h
@@ -0,0 +1,93 @@
+/*-
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_REG_H_
+#define _MACHINE_REG_H_
+
+#include <sys/_types.h>
+
+/*
+ * 32 general-purpose registers in ABI order, followed by the exception
+ * return address (era) and the status CSR (prmd), matching the convention
+ * of the other architectures.
+ * prmd is exported by fill_regs() but ignored by set_regs().
+ */
+struct reg {
+ __uint64_t r0;
+ __uint64_t ra;
+ __uint64_t tp;
+ __uint64_t sp;
+ __uint64_t a[8];
+ __uint64_t t[9];
+ __uint64_t r21;
+ __uint64_t fp;
+ __uint64_t s[9];
+ __uint64_t era; /* PC (exception return address) */
+ __uint64_t prmd; /* status; read-only via set_regs */
+};
+
+struct fpreg {
+ /* Matches the LLVM/LLDB FPR set: fpr[32], fcc (8 bits), fcsr. */
+ __uint64_t fpr[32]; /* floating point registers */
+ __uint64_t fcc; /* 8 FP condition codes */
+ __uint32_t fcsr; /* FP control/status register */
+};
+
+/*
+ * LSX (128-bit) and LASX (256-bit) SIMD register sets, exposed via
+ * PT_GETREGSET / PT_SETREGSET and core-dump notes (NT_LOONGARCH_LSX /
+ * NT_LOONGARCH_LASX). Legacy PT_GETFPREGS / struct fpreg are unchanged.
+ */
+struct lsxreg {
+ __uint64_t xr_regs[32 * 2]; /* 32 x 128-bit LSX registers */
+};
+struct lasxreg {
+ __uint64_t xr_regs[32 * 4]; /* 32 x 256-bit LASX
+ * registers */
+};
+
+/*
+ * Hardware debug registers (ptrace PT_GETDBREGS / PT_SETDBREGS).
+ *
+ * Exposes the LoongArch data-watchpoint slots (DB0..DB7). db_ctrl holds the
+ * DBnCTRL bit layout (CSR_DBC_*): a non-zero value arms the slot, 0 leaves it
+ * unused. The ASID is managed internally from the target pmap and is not part
+ * of the ABI.
+ */
+#define LA_WATCH_SLOTS 8
+
+struct dbreg {
+ __uint8_t db_revision; /* layout revision */
+ __uint8_t db_nslots; /* usable watchpoint slots */
+ __uint8_t db_pad[6];
+ struct {
+ __uint64_t db_addr; /* watched VA */
+ __uint32_t db_ctrl; /* DBnCTRL bits (0 = unused) */
+ __uint32_t db_pad;
+ } db_slots[LA_WATCH_SLOTS];
+};
+
+#endif /* !_MACHINE_REG_H_ */
diff --git a/sys/loongarch/include/reloc.h b/sys/loongarch/include/reloc.h
new file mode 100644
diff --git a/sys/loongarch/include/resource.h b/sys/loongarch/include/resource.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/resource.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright 1998 Massachusetts Institute of Technology
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that both the above copyright notice and this
+ * permission notice appear in all copies, that both the above
+ * copyright notice and this permission notice appear in all
+ * supporting documentation, and that the name of M.I.T. not be used
+ * in advertising or publicity pertaining to distribution of the
+ * software without specific, written prior permission. M.I.T. makes
+ * no representations about the suitability of this software for any
+ * purpose. It is provided "as is" without express or implied
+ * warranty.
+ *
+ * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS
+ * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
+ * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_RESOURCE_H_
+#define _MACHINE_RESOURCE_H_ 1
+
+/*
+ * Definitions of resource types for Intel Architecture machines
+ * with support for legacy ISA devices and drivers.
+ */
+
+#define SYS_RES_IRQ 1 /* interrupt lines */
+#define SYS_RES_DRQ 2 /* isa dma lines */
+#define SYS_RES_MEMORY 3 /* i/o memory */
+#define SYS_RES_IOPORT 4 /* i/o ports */
+#define SYS_RES_GPIO 5 /* general purpose i/o */
+#define PCI_RES_BUS 6 /* PCI bus numbers */
+
+#endif /* !_MACHINE_RESOURCE_H_ */
diff --git a/sys/loongarch/include/runq.h b/sys/loongarch/include/runq.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/runq.h
@@ -0,0 +1,44 @@
+/*-
+ * Copyright (c) 2001 Jake Burkholder <jake@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_RUNQ_H_
+#define _MACHINE_RUNQ_H_
+
+#define RQB_LEN (1) /* Number of priority status words. */
+#define RQB_L2BPW (6) /* Log2(sizeof(rqb_word_t) * NBBY)). */
+#define RQB_BPW (1<<RQB_L2BPW) /* Bits in an rqb_word_t. */
+
+#define RQB_BIT(pri) (1ul << ((pri) & (RQB_BPW - 1)))
+#define RQB_WORD(pri) ((pri) >> RQB_L2BPW)
+
+#define RQB_FFS(word) (ffsl(word) - 1)
+
+/*
+ * Type of run queue status word.
+ */
+typedef unsigned long rqb_word_t;
+
+#endif
diff --git a/sys/loongarch/include/sdt_machdep.h b/sys/loongarch/include/sdt_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/sdt_machdep.h
@@ -0,0 +1,12 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#ifndef _SYS_SDT_MACHDEP_H_
+#define _SYS_SDT_MACHDEP_H_
+
+#define _SDT_ASM_PATCH_INSTR "nop"
+
+#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/loongarch/include/setjmp.h b/sys/loongarch/include/setjmp.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/setjmp.h
@@ -0,0 +1,57 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_SETJMP_H_
+#define _MACHINE_SETJMP_H_
+
+#include <sys/cdefs.h>
+
+#define _JBLEN 63 /* sp, ra, fp, s0-s8, fs0-fs7, fcsr0, magic val, sigmask */
+#define _JB_SIGMASK 27
+
+#ifdef __ASSEMBLER__
+#define _JB_MAGIC__SETJMP 0xbe87fd8a2910af00
+#define _JB_MAGIC_SETJMP 0xbe87fd8a2910af01
+#endif /* __ASSEMBLER__ */
+
+#ifndef __ASSEMBLER__
+/*
+ * jmp_buf and sigjmp_buf are encapsulated in different structs to force
+ * compile-time diagnostics for mismatches. The structs are the same
+ * internally to avoid some run-time errors for mismatches.
+ */
+#if __BSD_VISIBLE || __POSIX_VISIBLE || __XSI_VISIBLE
+typedef struct _sigjmp_buf {
+ long _sjb[_JBLEN + 1] __aligned(16);
+} sigjmp_buf[1];
+#endif
+
+typedef struct _jmp_buf {
+ long _jb[_JBLEN + 1] __aligned(16);
+} jmp_buf[1];
+#endif /* !__ASSEMBLER__ */
+
+#endif /* !_MACHINE_SETJMP_H_ */
diff --git a/sys/loongarch/include/sf_buf.h b/sys/loongarch/include/sf_buf.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/sf_buf.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 2003, 2005 Alan L. Cox <alc@cs.rice.edu>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_SF_BUF_H_
+#define _MACHINE_SF_BUF_H_
+
+/*
+ * On this machine, the only purpose for which sf_buf is used is to implement
+ * an opaque pointer required by the machine-independent parts of the kernel.
+ * That pointer references the vm_page that is "mapped" by the sf_buf. The
+ * actual mapping is provided by the direct virtual-to-physical mapping.
+ */
+static inline void *
+sf_buf_kva(struct sf_buf *sf)
+{
+
+ return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf)));
+}
+
+static inline vm_page_t
+sf_buf_page(struct sf_buf *sf)
+{
+
+ return ((vm_page_t)sf);
+}
+#endif /* !_MACHINE_SF_BUF_H_ */
diff --git a/sys/loongarch/include/sigframe.h b/sys/loongarch/include/sigframe.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/sigframe.h
@@ -0,0 +1 @@
+#include <machine/frame.h>
diff --git a/sys/loongarch/include/signal.h b/sys/loongarch/include/signal.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/signal.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 1986, 1989, 1991, 1993
+ * The Regents of the University of California. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)signal.h 8.1 (Berkeley) 6/11/93
+ * from: FreeBSD: src/sys/i386/include/signal.h,v 1.13 2000/11/09
+ * from: FreeBSD: src/sys/sparc64/include/signal.h,v 1.6 2001/09/30 18:52:17
+ */
+
+#ifndef _MACHINE_SIGNAL_H_
+#define _MACHINE_SIGNAL_H_
+
+#include <sys/cdefs.h>
+
+typedef long sig_atomic_t;
+
+#if __BSD_VISIBLE
+
+struct sigcontext {
+ int _dummy;
+};
+
+#endif
+
+#endif /* !_MACHINE_SIGNAL_H_ */
diff --git a/sys/loongarch/include/smp.h b/sys/loongarch/include/smp.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/smp.h
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_SMP_H_
+#define _MACHINE_SMP_H_
+
+#include <machine/pcb.h>
+
+#define IPI_AST (1 << 0)
+#define IPI_PREEMPT (1 << 1)
+#define IPI_RENDEZVOUS (1 << 2)
+#define IPI_STOP (1 << 3)
+#define IPI_STOP_HARD (1 << 4)
+#define IPI_HARDCLOCK (1 << 5)
+#define IPI_TLB (1 << 6)
+
+#define INTR_IPI_COUNT 1
+
+void ipi_all_but_self(u_int ipi);
+void ipi_cpu(int cpu, u_int ipi);
+void ipi_selected(cpuset_t cpus, u_int ipi);
+uint32_t ipi_read_clear(int cpu);
+void csr_mail_send(uint64_t data, int cpu, int mailbox);
+
+/* SMP TLB shootdown op codes (stored in a shootdown descriptor's op field). */
+#define LA_TLB_OP_PAGE 0
+#define LA_TLB_OP_RANGE 1
+#define LA_TLB_OP_ALL 2
+
+struct pmap;
+
+/* Synchronous TLB shootdown via IPIs with a 2-D scoreboard. */
+void smp_targeted_tlb_shootdown(struct pmap *pmap, vm_offset_t addr1,
+ vm_offset_t addr2, uint8_t op);
+
+/* Record a stale ASID whose TLB entries should be batch-flushed. */
+void la_stale_add(uint32_t asid);
+void la_stale_reset(void);
+
+#endif /* !_MACHINE_SMP_H_ */
diff --git a/sys/loongarch/include/stack.h b/sys/loongarch/include/stack.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/stack.h
@@ -0,0 +1,61 @@
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_STACK_H_
+#define _MACHINE_STACK_H_
+
+#define INKERNEL(va) ((va) >= VM_MIN_KERNEL_ADDRESS && \
+ (va) <= VM_MAX_KERNEL_ADDRESS)
+
+struct unwind_state {
+ uintptr_t fp;
+ uintptr_t sp;
+ uintptr_t pc;
+};
+
+bool unwind_frame(struct thread *, struct unwind_state *);
+
+#ifdef _SYS_PROC_H_
+
+#include <machine/pcb.h>
+
+/* Get the current kernel thread stack usage. */
+#define GET_STACK_USAGE(total, used) do { \
+ struct thread *td = curthread; \
+ (total) = ptoa(td->td_kstack_pages); \
+ (used) = td->td_kstack + (total) - (char *)&td; \
+} while (0)
+
+static __inline bool
+kstack_contains(struct thread *td, vm_offset_t va, size_t len)
+{
+ return (va >= (vm_offset_t)td->td_kstack && va + len >= va &&
+ va + len <= (vm_offset_t)(td->td_kstack +
+ ptoa(td->td_kstack_pages)));
+}
+#endif /* _SYS_PROC_H_ */
+
+#endif /* !_MACHINE_STACK_H_ */
diff --git a/sys/loongarch/include/stdarg.h b/sys/loongarch/include/stdarg.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/stdarg.h
@@ -0,0 +1,37 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2017 Poul-Henning Kamp. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_STDARG_H_
+#define _MACHINE_STDARG_H_
+
+#include <sys/_stdarg.h>
+
+#ifndef va_start
+ #error this file needs to be ported to your compiler
+#endif
+
+#endif /* !_MACHINE_STDARG_H_ */
diff --git a/sys/loongarch/include/sysarch.h b/sys/loongarch/include/sysarch.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/sysarch.h
@@ -0,0 +1,46 @@
+/*-
+ * Copyright (c) 1993 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: FreeBSD: src/sys/i386/include/sysarch.h,v 1.14 2000/09/21
+ */
+
+/*
+ * Architecture specific syscalls (LoongArch)
+ */
+#ifndef _MACHINE_SYSARCH_H_
+#define _MACHINE_SYSARCH_H_
+
+#ifndef _KERNEL
+
+__BEGIN_DECLS
+int sysarch(int _number, void *_args);
+__END_DECLS
+
+#endif
+
+#endif /* !_MACHINE_SYSARCH_H_ */
diff --git a/sys/loongarch/include/tlb.h b/sys/loongarch/include/tlb.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/tlb.h
@@ -0,0 +1,110 @@
+#ifndef _MACHINE_TLB_H_
+#define _MACHINE_TLB_H_
+
+#include <machine/loongarchreg.h>
+/*
+ * TLB Invalidate Flush
+ */
+static inline void tlbclr(void)
+{
+ __asm __volatile("tlbclr");
+}
+
+static inline void tlbflush(void)
+{
+ __asm __volatile("tlbflush");
+}
+
+/*
+ * TLB R/W operations.
+ */
+static inline void tlb_probe(void)
+{
+ __asm __volatile("tlbsrch");
+}
+
+static inline void tlb_read(void)
+{
+ __asm __volatile("tlbrd");
+}
+
+static inline void tlb_write_indexed(void)
+{
+ __asm __volatile("tlbwr");
+}
+
+static inline void tlb_write_random(void)
+{
+ __asm __volatile("tlbfill");
+}
+
+enum invtlb_ops {
+ /* Invalid all tlb */
+ INVTLB_ALL = 0x0,
+ /* Invalid current tlb */
+ INVTLB_CURRENT_ALL = 0x1,
+ /* Invalid all global=1 lines in current tlb */
+ INVTLB_CURRENT_GTRUE = 0x2,
+ /* Invalid all global=0 lines in current tlb */
+ INVTLB_CURRENT_GFALSE = 0x3,
+ /* Invalid global=0 and matched asid lines in current tlb */
+ INVTLB_GFALSE_AND_ASID = 0x4,
+ /* Invalid addr with global=0 and matched asid in current tlb */
+ INVTLB_ADDR_GFALSE_AND_ASID = 0x5,
+ /* Invalid addr with global=1 or matched asid in current tlb */
+ INVTLB_ADDR_GTRUE_OR_ASID = 0x6,
+ /* Invalid matched gid in guest tlb */
+ INVGTLB_GID = 0x9,
+ /* Invalid global=1, matched gid in guest tlb */
+ INVGTLB_GID_GTRUE = 0xa,
+ /* Invalid global=0, matched gid in guest tlb */
+ INVGTLB_GID_GFALSE = 0xb,
+ /* Invalid global=0, matched gid and asid in guest tlb */
+ INVGTLB_GID_GFALSE_ASID = 0xc,
+ /* Invalid global=0 , matched gid, asid and addr in guest tlb */
+ INVGTLB_GID_GFALSE_ASID_ADDR = 0xd,
+ /* Invalid global=1 , matched gid, asid and addr in guest tlb */
+ INVGTLB_GID_GTRUE_ASID_ADDR = 0xe,
+ /* Invalid all gid gva-->gpa guest tlb */
+ INVGTLB_ALLGID_GVA_TO_GPA = 0x10,
+ /* Invalid all gid gpa-->hpa tlb */
+ INVTLB_ALLGID_GPA_TO_HPA = 0x11,
+ /* Invalid all gid tlb, including gva-->gpa and gpa-->hpa */
+ INVTLB_ALLGID = 0x12,
+ /* Invalid matched gid gva-->gpa guest tlb */
+ INVGTLB_GID_GVA_TO_GPA = 0x13,
+ /* Invalid matched gid gpa-->hpa tlb */
+ INVTLB_GID_GPA_TO_HPA = 0x14,
+ /* Invalid matched gid tlb,including gva-->gpa and gpa-->hpa */
+ INVTLB_GID_ALL = 0x15,
+ /* Invalid matched gid and addr gpa-->hpa tlb */
+ INVTLB_GID_ADDR = 0x16,
+};
+
+static __always_inline void invtlb(uint32_t op, uint32_t info, uint64_t addr)
+{
+ __asm __volatile(
+ "invtlb %0, %1, %2\n\t"
+ :
+ : "i"(op), "r"(info), "r"(addr)
+ : "memory"
+ );
+}
+
+static __always_inline void
+invtlb_all(uint32_t op)
+{
+ __asm __volatile(
+ "invtlb %0, $zero, $zero\n\t"
+ :
+ : "i"(op)
+ : "memory"
+ );
+}
+
+static __always_inline void flush_tlb_user(void)
+{
+ invtlb_all(INVTLB_CURRENT_GFALSE);
+}
+
+#endif
diff --git a/sys/loongarch/include/tls.h b/sys/loongarch/include/tls.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/tls.h
@@ -0,0 +1,53 @@
+/*-
+ * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_TLS_H_
+#define _MACHINE_TLS_H_
+
+#include <sys/_tls_variant_i.h>
+
+#define TLS_DTV_OFFSET 0x800
+#define TLS_TCB_ALIGN 16
+#define TLS_TP_OFFSET 0
+
+static __inline void
+_tcb_set(struct tcb *tcb)
+{
+ __asm __volatile("addi.d $tp, %0, %1" :: "r" (tcb), "I" (TLS_TCB_SIZE));
+}
+
+static __inline struct tcb *
+_tcb_get(void)
+{
+ struct tcb *tcb;
+
+ __asm __volatile("addi.d %0, $tp, %1"
+ : "=r" (tcb) : "I" (-TLS_TCB_SIZE));
+ return (tcb);
+}
+
+#endif /* !_MACHINE_TLS_H_ */
diff --git a/sys/loongarch/include/trap.h b/sys/loongarch/include/trap.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/trap.h
@@ -0,0 +1,5 @@
+
+#ifndef __LOONGARCH_TRAP_H__
+#define __LOONGARCH_TRAP_H__
+#define RSIZE 8 /* 64 bit mode register size */
+#endif
diff --git a/sys/loongarch/include/ucontext.h b/sys/loongarch/include/ucontext.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/ucontext.h
@@ -0,0 +1,70 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_UCONTEXT_H_
+#define _MACHINE_UCONTEXT_H_
+
+/*
+ * 32 general-purpose registers in ABI order, followed by the exception
+ * return address (era) and the status CSR (prmd), matching struct reg and
+ * the convention of the other architectures.
+ */
+struct gpregs {
+ __register_t r0;
+ __register_t ra;
+ __register_t tp;
+ __register_t sp;
+ __register_t gp_a[8];
+ __register_t gp_t[9];
+ __register_t r21;
+ __register_t fp;
+ __register_t gp_s[9];
+
+ __register_t gp_era; /* PC (exception return address) */
+ __register_t gp_prmd; /* status */
+};
+
+struct fpregs {
+ __uint64_t fp_regs[32];
+ __uint64_t fp_fcsr;
+ __uint64_t fcc;
+ int fp_flags;
+ int pad;
+};
+
+struct __mcontext {
+ struct gpregs mc_gpregs;
+ struct fpregs mc_fpregs;
+ int mc_flags;
+#define _MC_FP_VALID 0x1 /* Set when mc_fpregs has valid data */
+#define _MC_FP_SIMD 0x2 /* SIMD block present; mc_spare[0] -> user ptr */
+ int mc_pad;
+ __uint64_t mc_spare[8]; /* Space for expansion */
+};
+
+typedef struct __mcontext mcontext_t;
+
+#endif /* !_MACHINE_UCONTEXT_H_ */
diff --git a/sys/loongarch/include/vdso.h b/sys/loongarch/include/vdso.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/vdso.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_VDSO_H_
+#define _MACHINE_VDSO_H_
+
+#define VDSO_TIMEHANDS_MD \
+ uint32_t th_res[8];
+
+#define VDSO_TH_ALGO_LOONGARCH_RDTIME VDSO_TH_ALGO_1
+
+#endif /* !_MACHINE_VDSO_H_ */
diff --git a/sys/loongarch/include/vm.h b/sys/loongarch/include/vm.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/vm.h
@@ -0,0 +1,60 @@
+/*-
+ * Copyright (c) 2009 Alan L. Cox <alc@cs.rice.edu>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_VM_H_
+#define _MACHINE_VM_H_
+
+/* Memory attribute configuration. */
+#define VM_MEMATTR_DEVICE 0
+#define VM_MEMATTR_UNCACHEABLE 1
+#define VM_MEMATTR_WRITE_BACK 2
+#define VM_MEMATTR_WRITE_COMBINING 3
+#define VM_MEMATTR_WRITE_PROTECTED VM_MEMATTR_WRITE_BACK
+
+#define VM_MEMATTR_DEFAULT VM_MEMATTR_WRITE_BACK
+
+#ifdef _KERNEL
+#include <sys/_null.h>
+
+static inline const char *
+vm_memattr_name(vm_memattr_t memattr)
+{
+ switch (memattr) {
+ case VM_MEMATTR_DEVICE:
+ return ("device");
+ case VM_MEMATTR_UNCACHEABLE:
+ return ("uncacheable");
+ case VM_MEMATTR_WRITE_BACK:
+ return ("write-back");
+ case VM_MEMATTR_WRITE_COMBINING:
+ return ("write-combining");
+ default:
+ return (NULL);
+ }
+}
+#endif
+
+#endif /* !_MACHINE_VM_H_ */
diff --git a/sys/loongarch/include/vmparam.h b/sys/loongarch/include/vmparam.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/vmparam.h
@@ -0,0 +1,271 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ * Copyright (c) 1994 John S. Dyson
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: @(#)vmparam.h 5.9 (Berkeley) 5/12/91
+ * from: FreeBSD: src/sys/i386/include/vmparam.h,v 1.33 2000/03/30
+ */
+
+#ifndef _MACHINE_VMPARAM_H_
+#define _MACHINE_VMPARAM_H_
+
+/*
+ * Virtual memory related constants, all in bytes
+ */
+#ifndef MAXTSIZ
+#define MAXTSIZ (1*1024*1024*1024) /* max text size */
+#endif
+#ifndef DFLDSIZ
+#define DFLDSIZ (128*1024*1024) /* initial data size limit */
+#endif
+#ifndef MAXDSIZ
+#define MAXDSIZ (1*1024*1024*1024) /* max data size */
+#endif
+#ifndef DFLSSIZ
+#define DFLSSIZ (128*1024*1024) /* initial stack size limit */
+#endif
+#ifndef MAXSSIZ
+#define MAXSSIZ (1*1024*1024*1024) /* max stack size */
+#endif
+#ifndef SGROWSIZ
+#define SGROWSIZ (128*1024) /* amount to grow stack */
+#endif
+
+/*
+ * The physical address space is sparsely populated.
+ */
+#define VM_PHYSSEG_SPARSE
+
+/*
+ * The number of PHYSSEG entries.
+ */
+#define VM_PHYSSEG_MAX 64
+
+/*
+ * Create two free page pools: VM_FREEPOOL_DEFAULT is the default pool
+ * from which physical pages are allocated and VM_FREEPOOL_DIRECT is
+ * the pool from which physical pages for small UMA objects are
+ * allocated.
+ */
+#define VM_NFREEPOOL 2
+#define VM_FREEPOOL_DEFAULT 0
+#define VM_FREEPOOL_DIRECT 1
+
+/*
+ * Create one free page list: VM_FREELIST_DEFAULT is for all physical
+ * pages.
+ */
+#define VM_NFREELIST 1
+#define VM_FREELIST_DEFAULT 0
+
+/*
+ * An allocation size of 16MB is supported in order to optimize the
+ * use of the direct map by UMA. Specifically, a cache line contains
+ * at most four TTEs, collectively mapping 16MB of physical memory.
+ * By reducing the number of distinct 16MB "pages" that are used by UMA,
+ * the physical memory allocator reduces the likelihood of both 4MB
+ * page TLB misses and cache misses caused by 4MB page TLB misses.
+ */
+#define VM_NFREEORDER 12
+
+/*
+ * Enable superpage reservations: 1 level.
+ */
+#ifndef VM_NRESERVLEVEL
+#define VM_NRESERVLEVEL 1
+#endif
+
+/*
+ * Level 0 reservations consist of 512 pages.
+ */
+#ifndef VM_LEVEL_0_ORDER
+#define VM_LEVEL_0_ORDER 9
+#endif
+
+/**
+ * Address space layout.
+ */
+
+#define VM_MIN_ADDRESS (0x0000000000000000UL)
+#define VM_MAX_ADDRESS (0xffffffffffffffffUL)
+
+/* 128G kernel space */
+#define VM_MIN_KERNEL_ADDRESS (0xffff800000000000UL)
+#define VM_MAX_KERNEL_ADDRESS (0xffff802000000000UL)
+
+/*
+ * Kernel modules must load within +/-2 GB of the kernel's symbol range:
+ * LoongArch pcalau12i/pcaddu12i (PCALA/GOT) reach only signed si20<<12, and
+ * the short 2-instruction forms the compiler emits (e.g. __stack_chk_guard)
+ * can't be extended at load time.
+ *
+ * The low 1 GB (LA_KMOD_VA_SKIP) is the first PUD region, which holds the
+ * kernel image and its bootstrap-allocated L2 page tables. Those L2 pages
+ * have no vm_page, so pmap_alloc_l2()'s PHYS_TO_VM_PAGE() on the PUD entry
+ * returns NULL and faults. Both kmem (kva_import) and modules therefore
+ * start above it. Modules get the next LA_KMOD_VA_SIZE window (within reach);
+ * kmem follows. Window size < 1 GB so the window's far edge stays well under
+ * the 2 GB reach from VM_MIN_KERNEL_ADDRESS.
+ */
+#define LA_KMOD_VA_SKIP 0x40000000UL /* 1 GB: kernel + bootstrap PT */
+#define LA_KMOD_VA_SIZE 0x30000000UL /* 768 MB module window */
+
+#define KMOD_MIN_ADDRESS (VM_MIN_KERNEL_ADDRESS + LA_KMOD_VA_SKIP)
+#define KMOD_MAX_ADDRESS (KMOD_MIN_ADDRESS + LA_KMOD_VA_SIZE)
+
+/* 256G direct map */
+#define DMAP_MIN_ADDRESS (0xffffc04000000000UL)
+#define DMAP_MAX_ADDRESS (0xffffc08000000000UL)
+
+#define DMAP_MIN_PHYSADDR (dmap_phys_base)
+#define DMAP_MAX_PHYSADDR (dmap_phys_max)
+
+/* True if pa is in the dmap range */
+#define PHYS_IN_DMAP(pa) ((pa) >= DMAP_MIN_PHYSADDR && \
+ (pa) < DMAP_MAX_PHYSADDR)
+/* True if va is in the dmap range */
+#define VIRT_IN_DMAP(va) ((va) >= DMAP_MIN_ADDRESS && \
+ (va) < (dmap_max_addr))
+
+#define PMAP_HAS_DMAP 1
+#define PHYS_TO_DMAP_ADDR(pa) \
+({ \
+ KASSERT(PHYS_IN_DMAP(pa), \
+ ("%s: PA out of range, PA: 0x%lx", __func__, \
+ (vm_paddr_t)(pa))); \
+ ((pa) - dmap_phys_base) + DMAP_MIN_ADDRESS; \
+})
+#define PHYS_TO_DMAP(x) ((void*)PHYS_TO_DMAP_ADDR(x))
+
+#define DMAP_TO_PHYS(va) \
+({ \
+ uintptr_t _va = (uintptr_t)(va); \
+ KASSERT(VIRT_IN_DMAP(_va), \
+ ("%s: VA out of range, VA: 0x%lx", __func__, \
+ (vm_offset_t)(va))); \
+ (_va - DMAP_MIN_ADDRESS) + dmap_phys_base; \
+})
+
+/*
+ * DMW0: the uncached direct-map window (CSR.DMW0, VSEG=0x8000) configured in
+ * locore.S. Unlike the page-table-backed DMAP, DMW0 is a hardware window: a
+ * VA whose bits [63:48] equal 0x8000 maps 1:1 to the low 48-bit physical
+ * address with no page-table entry. It is used for early/uncached MMIO such
+ * as the console UART. Because there is no page-table entry, pmap_kextract()
+ * and pmap_extract() must translate these VAs directly instead of walking the
+ * page table (which would hit the VIRT_IS_VALID KASSERT in pmap_l1()).
+ */
+#define DMW0_BASE_ADDRESS (0x8000000000000000UL)
+#define VA_IN_DMW0(va) (((va) & 0xffff000000000000UL) == \
+ DMW0_BASE_ADDRESS)
+#define DMW0_TO_PHYS(va) ((vm_paddr_t)((va) & 0x0000ffffffffffffUL))
+
+/* 512 G of user space */
+#define VM_MIN_USER_ADDRESS (0x00000000000UL)
+#define VM_MAX_USER_ADDRESS (0x08000000000UL)
+
+#define VM_MINUSER_ADDRESS (VM_MIN_USER_ADDRESS)
+#define VM_MAXUSER_ADDRESS (VM_MAX_USER_ADDRESS)
+
+#define KERNBASE (VM_MIN_KERNEL_ADDRESS)
+#define SHAREDPAGE (VM_MAX_USER_ADDRESS - PAGE_SIZE)
+#define USRSTACK SHAREDPAGE
+#define PS_STRINGS (USRSTACK - sizeof(struct ps_strings))
+
+#define VM_EARLY_DTB_ADDRESS (VM_MAX_KERNEL_ADDRESS - (2 * L2_SIZE))
+
+/*
+ * How many physical pages per kmem arena virtual page.
+ */
+#ifndef VM_KMEM_SIZE_SCALE
+#define VM_KMEM_SIZE_SCALE (1)
+#endif
+
+/*
+ * Optional ceiling (in bytes) on the size of the kmem arena: 60% of the
+ * kernel map.
+ */
+#ifndef VM_KMEM_SIZE_MAX
+#define VM_KMEM_SIZE_MAX ((VM_MAX_KERNEL_ADDRESS - \
+ VM_MIN_KERNEL_ADDRESS + 1) * 3 / 5)
+#endif
+
+/*
+ * Initial pagein size of beginning of executable file.
+ */
+#ifndef VM_INITIAL_PAGEIN
+#define VM_INITIAL_PAGEIN 16
+#endif
+
+/*
+ * LoongArch has a valid direct map (DMW0 window, see DMAP_MIN_ADDRESS and
+ * PHYS_TO_DMAP in this file), so single-page UMA slabs are served from the
+ * DMAP via the MI uma_small_alloc/free (uma_core.c), matching arm64/riscv.
+ * Without UMA_USE_DMAP, PCPU zones created after booted >= BOOT_KVA fall
+ * through to pcpu_page_alloc, which needs kva/pmap plumbing not ready that
+ * early and hangs the boot (e.g. counter_u64_sysinit at SI_SUB_COUNTER).
+ */
+#define UMA_USE_DMAP
+
+#ifndef LOCORE
+extern vm_paddr_t dmap_phys_base;
+extern vm_paddr_t dmap_phys_max;
+extern vm_offset_t dmap_max_addr;
+extern vm_offset_t vm_max_kernel_address;
+extern vm_offset_t l1_pt_va;
+extern vm_offset_t l2_pt_va;
+extern vm_offset_t dmap_pt_va;
+#endif
+
+#define ZERO_REGION_SIZE (64 * 1024) /* 64KB */
+
+#define DEVMAP_MAX_VADDR VM_MAX_KERNEL_ADDRESS
+
+/*
+ * Helpers needed by subr_devmap.c for 1MB-aligned device mappings.
+ */
+#define L1_S_OFFSET ((1 << 20) - 1)
+#define trunc_1mpage(x) ((x) & ~L1_S_OFFSET)
+#define PMAP_MAPDEV_EARLY_SIZE (L2_SIZE * 8) /* 16M */
+
+/*
+ * No non-transparent large page support in the pmap.
+ */
+#define PMAP_HAS_LARGEPAGES 0
+
+/*
+ * Need a page dump array for minidump.
+ */
+#define MINIDUMP_PAGE_TRACKING 1
+#define MINIDUMP_STARTUP_PAGE_TRACKING 1
+
+#endif /* !_MACHINE_VMPARAM_H_ */
diff --git a/sys/loongarch/include/watch.h b/sys/loongarch/include/watch.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/include/watch.h
@@ -0,0 +1,39 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * LoongArch data-watchpoint debug facility (Phase 1: core).
+ */
+#ifndef _MACHINE_WATCH_H_
+#define _MACHINE_WATCH_H_
+
+#include <sys/_types.h>
+
+/*
+ * Number of hardware data-watchpoint slots tracked per thread. Must match
+ * LA_WATCH_SLOTS (the ptrace struct dbreg slot count) -- asserted in watch.c.
+ */
+#define LA_WATCH_NSLOTS 8
+
+/* One armed (or unused) watchpoint slot: address + DBnCTRL control word. */
+struct la_watch_slot {
+ __uint64_t addr; /* watched VA */
+ __uint32_t ctrl; /* DBnCTRL bits (CSR_DBC_*); 0 = unused */
+ __uint32_t _pad;
+};
+
+/* Per-thread desired watchpoint state, stored in struct mdthread. */
+struct la_watch_image {
+ struct la_watch_slot slot[LA_WATCH_NSLOTS];
+};
+
+struct thread;
+struct trapframe;
+
+void la_watch_init(void);
+void la_watch_load(struct thread *td);
+void la_watch_handler(struct thread *td, struct trapframe *frame);
+void la_watch_kern_handler(struct trapframe *frame);
+
+#endif /* !_MACHINE_WATCH_H_ */
diff --git a/sys/loongarch/loongarch/autoconf.c b/sys/loongarch/loongarch/autoconf.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/autoconf.c
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+/*
+ * Setup the system to run on the current machine.
+ *
+ * Configure() is called at boot time and initializes the vba
+ * device tables and the memory controller monitoring. Available
+ * devices are determined (from possibilities mentioned in ioconf.c),
+ * and the drivers are initialized.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/cons.h>
+
+static void configure_first(void *);
+static void configure(void *);
+static void configure_final(void *);
+
+SYSINIT(configure1, SI_SUB_CONFIGURE, SI_ORDER_FIRST, configure_first, NULL);
+/* SI_ORDER_SECOND is hookable */
+SYSINIT(configure2, SI_SUB_CONFIGURE, SI_ORDER_THIRD, configure, NULL);
+/* SI_ORDER_MIDDLE is hookable */
+SYSINIT(configure3, SI_SUB_CONFIGURE, SI_ORDER_ANY, configure_final, NULL);
+
+/*
+ * Determine i/o configuration for a machine.
+ */
+static void
+configure_first(void *dummy)
+{
+
+ /* nexus0 is the top of the loongarch device tree */
+ device_add_child(root_bus, "nexus", 0);
+}
+
+static void
+configure(void *dummy)
+{
+
+ /* initialize new bus architecture */
+ root_bus_configure();
+}
+
+static void
+configure_final(void *dummy)
+{
+
+ intr_enable();
+
+ cninit_finish();
+
+ if (bootverbose)
+ printf("Device configuration finished.\n");
+
+ cold = 0;
+}
diff --git a/sys/loongarch/loongarch/bus_machdep.c b/sys/loongarch/loongarch/bus_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/bus_machdep.c
@@ -0,0 +1,220 @@
+/*-
+ * Copyright (c) 2014 Andrew Turner
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_platform.h"
+
+#include <sys/param.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+
+uint8_t generic_bs_r_1(void *, bus_space_handle_t, bus_size_t);
+uint16_t generic_bs_r_2(void *, bus_space_handle_t, bus_size_t);
+uint32_t generic_bs_r_4(void *, bus_space_handle_t, bus_size_t);
+uint64_t generic_bs_r_8(void *, bus_space_handle_t, bus_size_t);
+
+void generic_bs_rm_1(void *, bus_space_handle_t, bus_size_t, uint8_t *,
+ bus_size_t);
+void generic_bs_rm_2(void *, bus_space_handle_t, bus_size_t, uint16_t *,
+ bus_size_t);
+void generic_bs_rm_4(void *, bus_space_handle_t, bus_size_t, uint32_t *,
+ bus_size_t);
+void generic_bs_rm_8(void *, bus_space_handle_t, bus_size_t, uint64_t *,
+ bus_size_t);
+
+void generic_bs_rr_1(void *, bus_space_handle_t, bus_size_t, uint8_t *,
+ bus_size_t);
+void generic_bs_rr_2(void *, bus_space_handle_t, bus_size_t, uint16_t *,
+ bus_size_t);
+void generic_bs_rr_4(void *, bus_space_handle_t, bus_size_t, uint32_t *,
+ bus_size_t);
+void generic_bs_rr_8(void *, bus_space_handle_t, bus_size_t, uint64_t *,
+ bus_size_t);
+
+void generic_bs_w_1(void *, bus_space_handle_t, bus_size_t, uint8_t);
+void generic_bs_w_2(void *, bus_space_handle_t, bus_size_t, uint16_t);
+void generic_bs_w_4(void *, bus_space_handle_t, bus_size_t, uint32_t);
+void generic_bs_w_8(void *, bus_space_handle_t, bus_size_t, uint64_t);
+
+void generic_bs_wm_1(void *, bus_space_handle_t, bus_size_t, const uint8_t *,
+ bus_size_t);
+void generic_bs_wm_2(void *, bus_space_handle_t, bus_size_t, const uint16_t *,
+ bus_size_t);
+void generic_bs_wm_4(void *, bus_space_handle_t, bus_size_t, const uint32_t *,
+ bus_size_t);
+void generic_bs_wm_8(void *, bus_space_handle_t, bus_size_t, const uint64_t *,
+ bus_size_t);
+
+void generic_bs_wr_1(void *, bus_space_handle_t, bus_size_t, const uint8_t *,
+ bus_size_t);
+void generic_bs_wr_2(void *, bus_space_handle_t, bus_size_t, const uint16_t *,
+ bus_size_t);
+void generic_bs_wr_4(void *, bus_space_handle_t, bus_size_t, const uint32_t *,
+ bus_size_t);
+void generic_bs_wr_8(void *, bus_space_handle_t, bus_size_t, const uint64_t *,
+ bus_size_t);
+
+static int
+generic_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
+ bus_space_handle_t *bshp)
+{
+ void *va;
+
+ va = pmap_mapdev(bpa, size);
+ if (va == NULL)
+ return (ENOMEM);
+ *bshp = (bus_space_handle_t)va;
+ return (0);
+}
+
+static void
+generic_bs_unmap(void *t, bus_space_handle_t bsh, bus_size_t size)
+{
+
+ pmap_unmapdev((void *)bsh, size);
+}
+
+static void
+generic_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
+ bus_size_t size, int flags)
+{
+}
+
+static int
+generic_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
+ bus_size_t size, bus_space_handle_t *nbshp)
+{
+
+ *nbshp = bsh + offset;
+ return (0);
+}
+
+struct bus_space memmap_bus = {
+ /* cookie */
+ .bs_cookie = NULL,
+
+ /* mapping/unmapping */
+ .bs_map = generic_bs_map,
+ .bs_unmap = generic_bs_unmap,
+ .bs_subregion = generic_bs_subregion,
+
+ /* allocation/deallocation */
+ .bs_alloc = NULL,
+ .bs_free = NULL,
+
+ /* barrier */
+ .bs_barrier = generic_bs_barrier,
+
+ /* read single */
+ .bs_r_1 = generic_bs_r_1,
+ .bs_r_2 = generic_bs_r_2,
+ .bs_r_4 = generic_bs_r_4,
+ .bs_r_8 = generic_bs_r_8,
+
+ /* read multiple */
+ .bs_rm_1 = NULL,
+ .bs_rm_2 = NULL,
+ .bs_rm_4 = NULL,
+ .bs_rm_8 = NULL,
+
+ /* write single */
+ .bs_w_1 = generic_bs_w_1,
+ .bs_w_2 = generic_bs_w_2,
+ .bs_w_4 = generic_bs_w_4,
+ .bs_w_8 = generic_bs_w_8,
+
+ /* write multiple */
+ .bs_wm_1 = NULL,
+ .bs_wm_2 = NULL,
+ .bs_wm_4 = NULL,
+ .bs_wm_8 = NULL,
+
+ /* write region */
+ .bs_wr_1 = NULL,
+ .bs_wr_2 = NULL,
+ .bs_wr_4 = NULL,
+ .bs_wr_8 = NULL,
+
+ /* set multiple */
+ .bs_sm_1 = NULL,
+ .bs_sm_2 = NULL,
+ .bs_sm_4 = NULL,
+ .bs_sm_8 = NULL,
+
+ /* set region */
+ .bs_sr_1 = NULL,
+ .bs_sr_2 = NULL,
+ .bs_sr_4 = NULL,
+ .bs_sr_8 = NULL,
+
+ /* copy */
+ .bs_c_1 = NULL,
+ .bs_c_2 = NULL,
+ .bs_c_4 = NULL,
+ .bs_c_8 = NULL,
+
+ /* read single stream */
+ .bs_r_1_s = NULL,
+ .bs_r_2_s = NULL,
+ .bs_r_4_s = NULL,
+ .bs_r_8_s = NULL,
+
+ /* read multiple stream */
+ .bs_rm_1_s = NULL,
+ .bs_rm_2_s = NULL,
+ .bs_rm_4_s = NULL,
+ .bs_rm_8_s = NULL,
+
+ /* read region stream */
+ .bs_rr_1_s = NULL,
+ .bs_rr_2_s = NULL,
+ .bs_rr_4_s = NULL,
+ .bs_rr_8_s = NULL,
+
+ /* write single stream */
+ .bs_w_1_s = NULL,
+ .bs_w_2_s = NULL,
+ .bs_w_4_s = NULL,
+ .bs_w_8_s = NULL,
+
+ /* write multiple stream */
+ .bs_wm_1_s = NULL,
+ .bs_wm_2_s = NULL,
+ .bs_wm_4_s = NULL,
+ .bs_wm_8_s = NULL,
+
+ /* write region stream */
+ .bs_wr_1_s = NULL,
+ .bs_wr_2_s = NULL,
+ .bs_wr_4_s = NULL,
+ .bs_wr_8_s = NULL,
+};
+
+#ifdef FDT
+bus_space_tag_t fdtbus_bs_tag = &memmap_bus;
+#endif
diff --git a/sys/loongarch/loongarch/bus_space_asm.S b/sys/loongarch/loongarch/bus_space_asm.S
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/bus_space_asm.S
@@ -0,0 +1,74 @@
+/*-
+ * Copyright (c) 2016-2020 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+ENTRY(generic_bs_r_1)
+ add.d $a3, $a1, $a2
+ ld.bu $a0, $a3, 0
+ jr $ra
+END(generic_bs_r_1)
+
+ENTRY(generic_bs_r_2)
+ add.d $a3, $a1, $a2
+ ld.hu $a0, $a3, 0
+ jr $ra
+END(generic_bs_r_2)
+
+ENTRY(generic_bs_r_4)
+ add.d $a3, $a1, $a2
+ ld.wu $a0, $a3, 0
+ jr $ra
+END(generic_bs_r_4)
+
+ENTRY(generic_bs_r_8)
+ add.d $a3, $a1, $a2
+ ld.d $a0, $a3, 0
+ jr $ra
+END(generic_bs_r_8)
+
+ENTRY(generic_bs_w_1)
+ add.d $a4, $a1, $a2
+ st.b $a3, $a4, 0
+ jr $ra
+END(generic_bs_w_1)
+
+ENTRY(generic_bs_w_2)
+ add.d $a4, $a1, $a2
+ st.h $a3, $a4, 0
+ jr $ra
+END(generic_bs_w_2)
+
+ENTRY(generic_bs_w_4)
+ add.d $a4, $a1, $a2
+ st.w $a3, $a4, 0
+ jr $ra
+END(generic_bs_w_4)
+
+ENTRY(generic_bs_w_8)
+ add.d $a4, $a1, $a2
+ st.d $a3, $a4, 0
+ jr $ra
+END(generic_bs_w_8)
diff --git a/sys/loongarch/loongarch/busdma_bounce.c b/sys/loongarch/loongarch/busdma_bounce.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/busdma_bounce.c
@@ -0,0 +1,959 @@
+/*-
+ * Copyright (c) 1997, 1998 Justin T. Gibbs.
+ * Copyright (c) 2015-2016 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by Andrew Turner
+ * under sponsorship of the FreeBSD Foundation.
+ *
+ * Portions of this software were developed by Semihalf
+ * under sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions, and the following disclaimer,
+ * without modification, immediately at the beginning of the file.
+ * 2. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/bus.h>
+#include <sys/interrupt.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/proc.h>
+#include <sys/memdesc.h>
+#include <sys/mutex.h>
+#include <sys/sysctl.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/md_var.h>
+#include <machine/bus_dma_impl.h>
+
+#define MAX_BPAGES 4096
+
+enum {
+ BF_COULD_BOUNCE = 0x01,
+ BF_MIN_ALLOC_COMP = 0x02,
+ BF_KMEM_ALLOC = 0x04,
+ BF_COHERENT = 0x10,
+};
+
+struct bounce_page;
+struct bounce_zone;
+
+struct bus_dma_tag {
+ struct bus_dma_tag_common common;
+ int map_count;
+ int bounce_flags;
+ bus_dma_segment_t *segments;
+ struct bounce_zone *bounce_zone;
+};
+
+static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "Busdma parameters");
+
+struct sync_list {
+ char *vaddr; /* kva of client data */
+ bus_addr_t paddr; /* physical address */
+ vm_page_t pages; /* starting page of client data */
+ bus_size_t datacount; /* client data count */
+};
+
+struct bus_dmamap {
+ STAILQ_HEAD(, bounce_page) bpages;
+ int pagesneeded;
+ int pagesreserved;
+ bus_dma_tag_t dmat;
+ struct memdesc mem;
+ bus_dmamap_callback_t *callback;
+ void *callback_arg;
+ __sbintime_t queued_time;
+ STAILQ_ENTRY(bus_dmamap) links;
+ u_int flags;
+#define DMAMAP_COULD_BOUNCE (1 << 0)
+#define DMAMAP_FROM_DMAMEM (1 << 1)
+ int sync_count;
+ struct sync_list slist[];
+};
+
+static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
+ pmap_t pmap, void *buf, bus_size_t buflen, int flags);
+static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
+ vm_paddr_t buf, bus_size_t buflen, int flags);
+static void bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map);
+
+static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
+
+#define dmat_alignment(dmat) ((dmat)->common.alignment)
+#define dmat_bounce_flags(dmat) ((dmat)->bounce_flags)
+#define dmat_boundary(dmat) ((dmat)->common.boundary)
+#define dmat_flags(dmat) ((dmat)->common.flags)
+#define dmat_highaddr(dmat) ((dmat)->common.highaddr)
+#define dmat_lowaddr(dmat) ((dmat)->common.lowaddr)
+#define dmat_lockfunc(dmat) ((dmat)->common.lockfunc)
+#define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg)
+#define dmat_maxsegsz(dmat) ((dmat)->common.maxsegsz)
+#define dmat_nsegments(dmat) ((dmat)->common.nsegments)
+
+#include "../../kern/subr_busdma_bounce.c"
+
+/*
+ * Allocate a device specific dma_tag.
+ */
+static int
+bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
+ bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
+ bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags,
+ bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat)
+{
+ bus_dma_tag_t newtag;
+ int error;
+
+ *dmat = NULL;
+ error = common_bus_dma_tag_create(parent != NULL ? &parent->common :
+ NULL, alignment, boundary, lowaddr, highaddr, maxsize, nsegments,
+ maxsegsz, flags, lockfunc, lockfuncarg,
+ sizeof (struct bus_dma_tag), (void **)&newtag);
+ if (error != 0)
+ return (error);
+
+ newtag->common.impl = &bus_dma_bounce_impl;
+ newtag->map_count = 0;
+ newtag->segments = NULL;
+
+ if ((flags & BUS_DMA_COHERENT) != 0)
+ newtag->bounce_flags |= BF_COHERENT;
+
+ if (parent != NULL) {
+ if ((parent->bounce_flags & BF_COULD_BOUNCE) != 0)
+ newtag->bounce_flags |= BF_COULD_BOUNCE;
+
+ /* Copy some flags from the parent */
+ newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT;
+ }
+
+ if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) ||
+ newtag->common.alignment > 1)
+ newtag->bounce_flags |= BF_COULD_BOUNCE;
+
+ if (((newtag->bounce_flags & BF_COULD_BOUNCE) != 0) &&
+ (flags & BUS_DMA_ALLOCNOW) != 0) {
+ struct bounce_zone *bz;
+
+ /* Must bounce */
+ if ((error = alloc_bounce_zone(newtag)) != 0) {
+ free(newtag, M_DEVBUF);
+ return (error);
+ }
+ bz = newtag->bounce_zone;
+
+ if (ptoa(bz->total_bpages) < maxsize) {
+ int pages;
+
+ pages = atop(round_page(maxsize)) - bz->total_bpages;
+
+ /* Add pages to our bounce pool */
+ if (alloc_bounce_pages(newtag, pages) < pages)
+ error = ENOMEM;
+ }
+ /* Performed initial allocation */
+ newtag->bounce_flags |= BF_MIN_ALLOC_COMP;
+ } else
+ error = 0;
+
+ if (error != 0)
+ free(newtag, M_DEVBUF);
+ else
+ *dmat = newtag;
+ CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
+ __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
+ error);
+ return (error);
+}
+
+static int
+bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat)
+{
+ int error = 0;
+
+ if (dmat != NULL) {
+ if (dmat->map_count != 0) {
+ error = EBUSY;
+ goto out;
+ }
+ if (dmat->segments != NULL)
+ free(dmat->segments, M_DEVBUF);
+ free(dmat, M_DEVBUF);
+ }
+out:
+ CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error);
+ return (error);
+}
+
+static bus_dmamap_t
+alloc_dmamap(bus_dma_tag_t dmat, int flags)
+{
+ u_long mapsize;
+ bus_dmamap_t map;
+
+ mapsize = sizeof(*map);
+ mapsize += sizeof(struct sync_list) * dmat->common.nsegments;
+ map = malloc(mapsize, M_DEVBUF, flags | M_ZERO);
+ if (map == NULL)
+ return (NULL);
+
+ /* Initialize the new map */
+ STAILQ_INIT(&map->bpages);
+
+ return (map);
+}
+
+/*
+ * Allocate a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static int
+bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
+{
+ struct bounce_zone *bz;
+ int error, maxpages, pages;
+
+ error = 0;
+
+ if (dmat->segments == NULL) {
+ dmat->segments = (bus_dma_segment_t *)malloc(
+ sizeof(bus_dma_segment_t) * dmat->common.nsegments,
+ M_DEVBUF, M_NOWAIT);
+ if (dmat->segments == NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, ENOMEM);
+ return (ENOMEM);
+ }
+ }
+
+ *mapp = alloc_dmamap(dmat, M_NOWAIT);
+ if (*mapp == NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, ENOMEM);
+ return (ENOMEM);
+ }
+
+ /*
+ * Bouncing might be required if the driver asks for an active
+ * exclusion region, a data alignment that is stricter than 1, and/or
+ * an active address boundary.
+ */
+ if (dmat->bounce_flags & BF_COULD_BOUNCE) {
+ /* Must bounce */
+ if (dmat->bounce_zone == NULL) {
+ if ((error = alloc_bounce_zone(dmat)) != 0) {
+ free(*mapp, M_DEVBUF);
+ return (error);
+ }
+ }
+ bz = dmat->bounce_zone;
+
+ (*mapp)->flags = DMAMAP_COULD_BOUNCE;
+
+ /*
+ * Attempt to add pages to our pool on a per-instance
+ * basis up to a sane limit.
+ */
+ if (dmat->common.alignment > 1)
+ maxpages = MAX_BPAGES;
+ else
+ maxpages = MIN(MAX_BPAGES, Maxmem -
+ atop(dmat->common.lowaddr));
+ if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 ||
+ (bz->map_count > 0 && bz->total_bpages < maxpages)) {
+ pages = MAX(atop(dmat->common.maxsize), 1);
+ pages = MIN(maxpages - bz->total_bpages, pages);
+ pages = MAX(pages, 1);
+ if (alloc_bounce_pages(dmat, pages) < pages)
+ error = ENOMEM;
+ if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP)
+ == 0) {
+ if (error == 0) {
+ dmat->bounce_flags |=
+ BF_MIN_ALLOC_COMP;
+ }
+ } else
+ error = 0;
+ }
+ bz->map_count++;
+ }
+ if (error == 0)
+ dmat->map_count++;
+ else
+ free(*mapp, M_DEVBUF);
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, error);
+ return (error);
+}
+
+/*
+ * Destroy a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static int
+bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+
+ /* Check we are destroying the correct map type */
+ if ((map->flags & DMAMAP_FROM_DMAMEM) != 0)
+ panic("bounce_bus_dmamap_destroy: Invalid map freed\n");
+
+ if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY);
+ return (EBUSY);
+ }
+ if (dmat->bounce_zone) {
+ KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0,
+ ("%s: Bounce zone when cannot bounce", __func__));
+ dmat->bounce_zone->map_count--;
+ }
+ free(map, M_DEVBUF);
+ dmat->map_count--;
+ CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
+ return (0);
+}
+
+/*
+ * Allocate a piece of memory that can be efficiently mapped into
+ * bus device space based on the constraints lited in the dma tag.
+ * A dmamap to for use with dmamap_load is also allocated.
+ */
+static int
+bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
+ bus_dmamap_t *mapp)
+{
+ /*
+ * XXX ARM64TODO:
+ * This bus_dma implementation requires IO-Coherent architecutre.
+ * If IO-Coherency is not guaranteed, the BUS_DMA_COHERENT flag has
+ * to be implented using non-cacheable memory.
+ */
+
+ vm_memattr_t attr;
+ int mflags;
+
+ if (flags & BUS_DMA_NOWAIT)
+ mflags = M_NOWAIT;
+ else
+ mflags = M_WAITOK;
+
+ if (dmat->segments == NULL) {
+ dmat->segments = (bus_dma_segment_t *)malloc(
+ sizeof(bus_dma_segment_t) * dmat->common.nsegments,
+ M_DEVBUF, mflags);
+ if (dmat->segments == NULL) {
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, ENOMEM);
+ return (ENOMEM);
+ }
+ }
+ if (flags & BUS_DMA_ZERO)
+ mflags |= M_ZERO;
+ if (flags & BUS_DMA_NOCACHE)
+ attr = VM_MEMATTR_UNCACHEABLE;
+ else if ((flags & BUS_DMA_COHERENT) != 0 &&
+ (dmat->bounce_flags & BF_COHERENT) == 0)
+ /*
+ * If we have a non-coherent tag, and are trying to allocate
+ * a coherent block of memory it needs to be uncached.
+ */
+ attr = VM_MEMATTR_UNCACHEABLE;
+ else
+ attr = VM_MEMATTR_DEFAULT;
+
+ /*
+ * Create the map, but don't set the could bounce flag as
+ * this allocation should never bounce;
+ */
+ *mapp = alloc_dmamap(dmat, mflags);
+ if (*mapp == NULL) {
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, ENOMEM);
+ return (ENOMEM);
+ }
+ (*mapp)->flags = DMAMAP_FROM_DMAMEM;
+
+ /*
+ * Allocate the buffer from the malloc(9) allocator if...
+ * - It's small enough to fit into a single power of two sized bucket.
+ * - The alignment is less than or equal to the maximum size
+ * - The low address requirement is fulfilled.
+ * else allocate non-contiguous pages if...
+ * - The page count that could get allocated doesn't exceed
+ * nsegments also when the maximum segment size is less
+ * than PAGE_SIZE.
+ * - The alignment constraint isn't larger than a page boundary.
+ * - There are no boundary-crossing constraints.
+ * else allocate a block of contiguous pages because one or more of the
+ * constraints is something that only the contig allocator can fulfill.
+ *
+ * NOTE: The (dmat->common.alignment <= dmat->maxsize) check
+ * below is just a quick hack. The exact alignment guarantees
+ * of malloc(9) need to be nailed down, and the code below
+ * should be rewritten to take that into account.
+ *
+ * In the meantime warn the user if malloc gets it wrong.
+ */
+ if ((dmat->common.maxsize <= PAGE_SIZE) &&
+ (dmat->common.alignment <= dmat->common.maxsize) &&
+ dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
+ attr == VM_MEMATTR_DEFAULT) {
+ *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags);
+ } else if (dmat->common.nsegments >=
+ howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) &&
+ dmat->common.alignment <= PAGE_SIZE &&
+ (dmat->common.boundary % PAGE_SIZE) == 0) {
+ /* Page-based multi-segment allocations allowed */
+ *vaddr = kmem_alloc_attr(dmat->common.maxsize, mflags,
+ 0ul, dmat->common.lowaddr, attr);
+ dmat->bounce_flags |= BF_KMEM_ALLOC;
+ } else {
+ *vaddr = kmem_alloc_contig(dmat->common.maxsize, mflags,
+ 0ul, dmat->common.lowaddr, dmat->common.alignment != 0 ?
+ dmat->common.alignment : 1ul, dmat->common.boundary, attr);
+ dmat->bounce_flags |= BF_KMEM_ALLOC;
+ }
+ if (*vaddr == NULL) {
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, ENOMEM);
+ free(*mapp, M_DEVBUF);
+ return (ENOMEM);
+ } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->common.alignment)) {
+ printf("bus_dmamem_alloc failed to align memory properly.\n");
+ }
+ dmat->map_count++;
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, 0);
+ return (0);
+}
+
+/*
+ * Free a piece of memory and it's allociated dmamap, that was allocated
+ * via bus_dmamem_alloc.
+ */
+static void
+bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
+{
+
+ /*
+ * Check the map came from bounce_bus_dmamem_alloc, so the map
+ * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc()
+ * was used and set if kmem_alloc_contig() was used.
+ */
+ if ((map->flags & DMAMAP_FROM_DMAMEM) == 0)
+ panic("bus_dmamem_free: Invalid map freed\n");
+ if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0)
+ free(vaddr, M_DEVBUF);
+ else
+ kmem_free(vaddr, dmat->common.maxsize);
+ free(map, M_DEVBUF);
+ dmat->map_count--;
+ CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat,
+ dmat->bounce_flags);
+}
+
+static void
+_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
+ bus_size_t buflen, int flags)
+{
+ bus_addr_t curaddr;
+ bus_size_t sgsize;
+
+ if ((map->flags & DMAMAP_COULD_BOUNCE) != 0 && map->pagesneeded == 0) {
+ /*
+ * Count the number of bounce pages
+ * needed in order to complete this transfer
+ */
+ curaddr = buf;
+ while (buflen != 0) {
+ sgsize = buflen;
+ if (addr_needs_bounce(dmat, curaddr)) {
+ sgsize = MIN(sgsize,
+ PAGE_SIZE - (curaddr & PAGE_MASK));
+ map->pagesneeded++;
+ }
+ curaddr += sgsize;
+ buflen -= sgsize;
+ }
+ CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
+ }
+}
+
+static void
+_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
+ void *buf, bus_size_t buflen, int flags)
+{
+ vm_offset_t vaddr;
+ vm_offset_t vendaddr;
+ bus_addr_t paddr;
+ bus_size_t sg_len;
+
+ if ((map->flags & DMAMAP_COULD_BOUNCE) != 0 && map->pagesneeded == 0) {
+ CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
+ "alignment= %d", dmat->common.lowaddr,
+ ptoa((vm_paddr_t)Maxmem),
+ dmat->common.boundary, dmat->common.alignment);
+ CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map,
+ map->pagesneeded);
+ /*
+ * Count the number of bounce pages
+ * needed in order to complete this transfer
+ */
+ vaddr = (vm_offset_t)buf;
+ vendaddr = (vm_offset_t)buf + buflen;
+
+ while (vaddr < vendaddr) {
+ sg_len = MIN(vendaddr - vaddr,
+ PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
+ if (pmap == kernel_pmap)
+ paddr = pmap_kextract(vaddr);
+ else
+ paddr = pmap_extract(pmap, vaddr);
+ if (addr_needs_bounce(dmat, paddr)) {
+ sg_len = roundup2(sg_len,
+ dmat->common.alignment);
+ map->pagesneeded++;
+ }
+ vaddr += sg_len;
+ }
+ CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
+ }
+}
+
+/*
+ * Utility function to load a physical buffer. segp contains
+ * the starting segment on entrace, and the ending segment on exit.
+ */
+static int
+bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
+ vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
+ int *segp)
+{
+ struct sync_list *sl;
+ bus_size_t sgsize;
+ bus_addr_t curaddr, sl_end;
+ int error;
+
+ /*
+ * Tolerate reload-without-unload: many drivers reload a dmamap
+ * without bus_dmamap_unload() (amd64's IOMMU tolerates this, so the
+ * bug is widespread). On this bounce layer sync_count only resets on
+ * unload, so a bare reload would accumulate and fail with EFBIG.
+ * Match amd64 tolerance by unloading first.
+ */
+ if (__predict_false(!STAILQ_EMPTY(&map->bpages) ||
+ map->sync_count != 0))
+ bounce_bus_dmamap_unload(dmat, map);
+
+ if (segs == NULL)
+ segs = dmat->segments;
+
+ if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) {
+ _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
+ if (map->pagesneeded != 0) {
+ error = _bus_dmamap_reserve_pages(dmat, map, flags);
+ if (error)
+ return (error);
+ }
+ }
+
+ sl = map->slist + map->sync_count - 1;
+ sl_end = 0;
+
+ while (buflen > 0) {
+ curaddr = buf;
+ sgsize = buflen;
+ if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) &&
+ map->pagesneeded != 0 &&
+ addr_needs_bounce(dmat, curaddr)) {
+ sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
+ curaddr = add_bounce_page(dmat, map, 0, curaddr,
+ sgsize);
+ } else if ((dmat->bounce_flags & BF_COHERENT) == 0) {
+ if (map->sync_count > 0)
+ sl_end = sl->paddr + sl->datacount;
+
+ if (map->sync_count == 0 || curaddr != sl_end) {
+ if (++map->sync_count > dmat->common.nsegments)
+ break;
+ sl++;
+ sl->vaddr = NULL;
+ sl->paddr = curaddr;
+ sl->datacount = sgsize;
+ sl->pages = PHYS_TO_VM_PAGE(curaddr);
+ KASSERT(sl->pages != NULL,
+ ("%s: page at PA:0x%08lx is not in "
+ "vm_page_array", __func__, curaddr));
+ } else
+ sl->datacount += sgsize;
+ }
+ if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
+ segp))
+ break;
+ buf += sgsize;
+ buflen -= sgsize;
+ }
+
+ /*
+ * Did we fit?
+ */
+ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
+}
+
+/*
+ * Utility function to load a linear buffer. segp contains
+ * the starting segment on entrace, and the ending segment on exit.
+ */
+static int
+bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
+ bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
+ int *segp)
+{
+ struct sync_list *sl;
+ bus_size_t sgsize;
+ bus_addr_t curaddr, sl_pend;
+ char *kvaddr, *vaddr, *sl_vend;
+ int error;
+
+ /*
+ * Tolerate reload-without-unload: many drivers reload a dmamap
+ * without bus_dmamap_unload() (amd64's IOMMU tolerates this, so the
+ * bug is widespread). On this bounce layer sync_count only resets on
+ * unload, so a bare reload would accumulate and fail with EFBIG.
+ * Match amd64 tolerance by unloading first.
+ */
+ if (__predict_false(!STAILQ_EMPTY(&map->bpages) ||
+ map->sync_count != 0))
+ bounce_bus_dmamap_unload(dmat, map);
+
+ if (segs == NULL)
+ segs = dmat->segments;
+
+ if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) {
+ _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
+ if (map->pagesneeded != 0) {
+ error = _bus_dmamap_reserve_pages(dmat, map, flags);
+ if (error)
+ return (error);
+ }
+ }
+
+ sl = map->slist + map->sync_count - 1;
+ vaddr = buf;
+ sl_pend = 0;
+ sl_vend = NULL;
+
+ while (buflen > 0) {
+ /*
+ * Get the physical address for this segment.
+ */
+ if (pmap == kernel_pmap) {
+ curaddr = pmap_kextract((vm_offset_t)vaddr);
+ kvaddr = vaddr;
+ } else {
+ curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);
+ kvaddr = NULL;
+ }
+
+ /*
+ * Compute the segment size, and adjust counts.
+ */
+ sgsize = MIN(buflen, PAGE_SIZE - (curaddr & PAGE_MASK));
+ if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) &&
+ map->pagesneeded != 0 &&
+ addr_needs_bounce(dmat, curaddr)) {
+ sgsize = roundup2(sgsize, dmat->common.alignment);
+ sgsize = MIN(sgsize, buflen);
+ curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
+ sgsize);
+ } else if ((dmat->bounce_flags & BF_COHERENT) == 0) {
+ if (map->sync_count > 0) {
+ sl_pend = sl->paddr + sl->datacount;
+ sl_vend = sl->vaddr + sl->datacount;
+ }
+
+ if (map->sync_count == 0 ||
+ (kvaddr != 0 && kvaddr != sl_vend) ||
+ (curaddr != sl_pend)) {
+ if (++map->sync_count > dmat->common.nsegments)
+ goto cleanup;
+ sl++;
+ sl->vaddr = kvaddr;
+ sl->paddr = curaddr;
+ if (kvaddr != NULL) {
+ sl->pages = NULL;
+ } else {
+ sl->pages = PHYS_TO_VM_PAGE(curaddr);
+ KASSERT(sl->pages != NULL,
+ ("%s: page at PA:0x%08lx is not "
+ "in vm_page_array", __func__,
+ curaddr));
+ }
+ sl->datacount = sgsize;
+ } else
+ sl->datacount += sgsize;
+ }
+ if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
+ segp))
+ break;
+ vaddr += sgsize;
+ buflen -= MIN(sgsize, buflen); /* avoid underflow */
+ }
+
+cleanup:
+ /*
+ * Did we fit?
+ */
+ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
+}
+
+static void
+bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
+{
+
+ if ((map->flags & DMAMAP_COULD_BOUNCE) == 0)
+ return;
+ map->mem = *mem;
+ map->dmat = dmat;
+ map->callback = callback;
+ map->callback_arg = callback_arg;
+}
+
+static bus_dma_segment_t *
+bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dma_segment_t *segs, int nsegs, int error)
+{
+
+ if (segs == NULL)
+ segs = dmat->segments;
+ return (segs);
+}
+
+/*
+ * Release the mapping held by map.
+ */
+static void
+bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+ free_bounce_pages(dmat, map);
+ map->sync_count = 0;
+}
+
+static void
+dma_preread_safe(char *va, size_t size)
+{
+ /*
+ * Write back any partial cachelines immediately before and
+ * after the DMA region.
+ */
+ if (!__is_aligned(va, dcache_line_size))
+ cpu_dcache_wb_range(va, 1);
+ if (!__is_aligned(va + size, dcache_line_size))
+ cpu_dcache_wb_range(va + size, 1);
+
+ cpu_dcache_inv_range(va, size);
+}
+
+static void
+dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op)
+{
+ uint32_t len, offset;
+ vm_page_t m;
+ vm_paddr_t pa;
+ char *va, *tempva;
+ bus_size_t size;
+
+ offset = sl->paddr & PAGE_MASK;
+ m = sl->pages;
+ size = sl->datacount;
+ pa = sl->paddr;
+
+ for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) {
+ tempva = 0;
+ if (sl->vaddr == NULL) {
+ len = min(PAGE_SIZE - offset, size);
+ tempva = pmap_quick_enter_page(m);
+ va = tempva + offset;
+ KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset),
+ ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx",
+ VM_PAGE_TO_PHYS(m) | offset, pa));
+ } else {
+ len = sl->datacount;
+ va = sl->vaddr;
+ }
+
+ switch (op) {
+ case BUS_DMASYNC_PREWRITE:
+ case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
+ cpu_dcache_wb_range(va, len);
+ break;
+ case BUS_DMASYNC_PREREAD:
+ /*
+ * An mbuf may start in the middle of a cacheline. There
+ * will be no cpu writes to the beginning of that line
+ * (which contains the mbuf header) while dma is in
+ * progress. Handle that case by doing a writeback of
+ * just the first cacheline before invalidating the
+ * overall buffer. Any mbuf in a chain may have this
+ * misalignment. Buffers which are not mbufs bounce if
+ * they are not aligned to a cacheline.
+ */
+ dma_preread_safe(va, len);
+ break;
+ case BUS_DMASYNC_POSTREAD:
+ case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
+ cpu_dcache_inv_range(va, len);
+ break;
+ default:
+ panic("unsupported combination of sync operations: "
+ "0x%08x\n", op);
+ }
+
+ if (tempva != NULL)
+ pmap_quick_remove_page(tempva);
+ }
+}
+
+static void
+bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dmasync_op_t op)
+{
+ struct bounce_page *bpage;
+ struct sync_list *sl, *end;
+ char *datavaddr, *tempvaddr;
+
+ if (op == BUS_DMASYNC_POSTWRITE)
+ return;
+
+ if ((op & BUS_DMASYNC_POSTREAD) != 0) {
+ /*
+ * Wait for any DMA operations to complete before the bcopy.
+ */
+ fence();
+ }
+
+ if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
+ "performing bounce", __func__, dmat, dmat->common.flags,
+ op);
+
+ if ((op & BUS_DMASYNC_PREWRITE) != 0) {
+ while (bpage != NULL) {
+ tempvaddr = NULL;
+ datavaddr = bpage->datavaddr;
+ if (datavaddr == NULL) {
+ tempvaddr = pmap_quick_enter_page(
+ bpage->datapage);
+ datavaddr = tempvaddr + bpage->dataoffs;
+ }
+
+ bcopy(datavaddr, bpage->vaddr, bpage->datacount);
+ if (tempvaddr != NULL)
+ pmap_quick_remove_page(tempvaddr);
+ if ((dmat->bounce_flags & BF_COHERENT) == 0)
+ cpu_dcache_wb_range(bpage->vaddr,
+ bpage->datacount);
+ bpage = STAILQ_NEXT(bpage, links);
+ }
+ dmat->bounce_zone->total_bounced++;
+ } else if ((op & BUS_DMASYNC_PREREAD) != 0) {
+ while (bpage != NULL) {
+ if ((dmat->bounce_flags & BF_COHERENT) == 0)
+ cpu_dcache_wbinv_range(bpage->vaddr,
+ bpage->datacount);
+ bpage = STAILQ_NEXT(bpage, links);
+ }
+ }
+
+ if ((op & BUS_DMASYNC_POSTREAD) != 0) {
+ while (bpage != NULL) {
+ if ((dmat->bounce_flags & BF_COHERENT) == 0)
+ cpu_dcache_inv_range(bpage->vaddr,
+ bpage->datacount);
+ tempvaddr = NULL;
+ datavaddr = bpage->datavaddr;
+ if (datavaddr == NULL) {
+ tempvaddr = pmap_quick_enter_page(
+ bpage->datapage);
+ datavaddr = tempvaddr + bpage->dataoffs;
+ }
+
+ bcopy(bpage->vaddr, datavaddr, bpage->datacount);
+
+ if (tempvaddr != NULL)
+ pmap_quick_remove_page(tempvaddr);
+ bpage = STAILQ_NEXT(bpage, links);
+ }
+ dmat->bounce_zone->total_bounced++;
+ }
+ }
+
+ /*
+ * Cache maintenance for normal (non-COHERENT non-bounce) buffers.
+ */
+ if (map->sync_count != 0) {
+ sl = &map->slist[0];
+ end = &map->slist[map->sync_count];
+ CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x "
+ "performing sync", __func__, dmat, op);
+
+ for ( ; sl != end; ++sl)
+ dma_dcache_sync(sl, op);
+ }
+
+ if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) {
+ /*
+ * Wait for the bcopy to complete before any DMA operations.
+ */
+ fence();
+ }
+}
+
+struct bus_dma_impl bus_dma_bounce_impl = {
+ .tag_create = bounce_bus_dma_tag_create,
+ .tag_destroy = bounce_bus_dma_tag_destroy,
+ .map_create = bounce_bus_dmamap_create,
+ .map_destroy = bounce_bus_dmamap_destroy,
+ .mem_alloc = bounce_bus_dmamem_alloc,
+ .mem_free = bounce_bus_dmamem_free,
+ .load_phys = bounce_bus_dmamap_load_phys,
+ .load_buffer = bounce_bus_dmamap_load_buffer,
+ .load_ma = bus_dmamap_load_ma_triv,
+ .map_waitok = bounce_bus_dmamap_waitok,
+ .map_complete = bounce_bus_dmamap_complete,
+ .map_unload = bounce_bus_dmamap_unload,
+ .map_sync = bounce_bus_dmamap_sync
+};
diff --git a/sys/loongarch/loongarch/busdma_machdep.c b/sys/loongarch/loongarch/busdma_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/busdma_machdep.c
@@ -0,0 +1,177 @@
+/*-
+ * Copyright (c) 1997, 1998 Justin T. Gibbs.
+ * Copyright (c) 2013, 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Portions of this software were developed by Semihalf
+ * under sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions, and the following disclaimer,
+ * without modification, immediately at the beginning of the file.
+ * 2. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/memdesc.h>
+#include <sys/mutex.h>
+#include <sys/uio.h>
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+#include <machine/bus_dma_impl.h>
+
+int
+common_bus_dma_tag_create(struct bus_dma_tag_common *parent,
+ bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr,
+ bus_addr_t highaddr, bus_size_t maxsize, int nsegments,
+ bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
+ void *lockfuncarg, size_t sz, void **dmat)
+{
+ void *newtag;
+ struct bus_dma_tag_common *common;
+
+ KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz"));
+ /* Return a NULL tag on failure */
+ *dmat = NULL;
+ /* Basic sanity checking */
+ if (boundary != 0 && boundary < maxsegsz)
+ maxsegsz = boundary;
+ if (maxsegsz == 0)
+ return (EINVAL);
+
+ newtag = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT);
+ if (newtag == NULL) {
+ CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
+ __func__, newtag, 0, ENOMEM);
+ return (ENOMEM);
+ }
+
+ common = newtag;
+ common->impl = &bus_dma_bounce_impl;
+ common->alignment = alignment;
+ common->boundary = boundary;
+ common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
+ common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
+ common->maxsize = maxsize;
+ common->nsegments = nsegments;
+ common->maxsegsz = maxsegsz;
+ common->flags = flags;
+ if (lockfunc != NULL) {
+ common->lockfunc = lockfunc;
+ common->lockfuncarg = lockfuncarg;
+ } else {
+ common->lockfunc = _busdma_dflt_lock;
+ common->lockfuncarg = NULL;
+ }
+
+ /* Take into account any restrictions imposed by our parent tag */
+ if (parent != NULL) {
+ common->impl = parent->impl;
+ common->lowaddr = MIN(parent->lowaddr, common->lowaddr);
+ common->highaddr = MAX(parent->highaddr, common->highaddr);
+ if (common->boundary == 0)
+ common->boundary = parent->boundary;
+ else if (parent->boundary != 0) {
+ common->boundary = MIN(parent->boundary,
+ common->boundary);
+ }
+ }
+ *dmat = common;
+ return (0);
+}
+
+/*
+ * Allocate a device specific dma_tag.
+ */
+int
+bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
+ bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
+ bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
+ int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
+ void *lockfuncarg, bus_dma_tag_t *dmat)
+{
+ struct bus_dma_tag_common *tc;
+ int error;
+
+ /* Filters are no longer supported. */
+ if (filter != NULL || filterarg != NULL)
+ return (EINVAL);
+
+ if (parent == NULL) {
+ error = bus_dma_bounce_impl.tag_create(parent, alignment,
+ boundary, lowaddr, highaddr, maxsize, nsegments, maxsegsz,
+ flags, lockfunc, lockfuncarg, dmat);
+ } else {
+ tc = (struct bus_dma_tag_common *)parent;
+ error = tc->impl->tag_create(parent, alignment,
+ boundary, lowaddr, highaddr, maxsize, nsegments, maxsegsz,
+ flags, lockfunc, lockfuncarg, dmat);
+ }
+ return (error);
+}
+
+void
+bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat)
+{
+ struct bus_dma_tag_common *common;
+
+ if (t == NULL || dmat == NULL)
+ return;
+
+ common = (struct bus_dma_tag_common *)dmat;
+
+ t->alignment = common->alignment;
+ t->boundary = common->boundary;
+ t->lowaddr = common->lowaddr;
+ t->highaddr = common->highaddr;
+ t->maxsize = common->maxsize;
+ t->nsegments = common->nsegments;
+ t->maxsegsize = common->maxsegsz;
+ t->flags = common->flags;
+ t->lockfunc = common->lockfunc;
+ t->lockfuncarg = common->lockfuncarg;
+}
+
+int
+bus_dma_tag_destroy(bus_dma_tag_t dmat)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->tag_destroy(dmat));
+}
+
+int
+bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain)
+{
+
+ return (0);
+}
diff --git a/sys/loongarch/loongarch/clock.c b/sys/loongarch/loongarch/clock.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/clock.c
@@ -0,0 +1,60 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+
+void
+cpu_initclocks(void)
+{
+#ifdef EARLY_AP_STARTUP
+ struct thread *td;
+ int i;
+
+ td = curthread;
+ cpu_initclocks_bsp();
+ CPU_FOREACH(i) {
+ if (i == 0)
+ continue;
+ thread_lock(td);
+ sched_bind(td, i);
+ thread_unlock(td);
+ cpu_initclocks_ap();
+ }
+ thread_lock(td);
+ if (sched_is_bound(td))
+ sched_unbind(td);
+ thread_unlock(td);
+#else
+ cpu_initclocks_bsp();
+#endif
+}
diff --git a/sys/loongarch/loongarch/copyinout.S b/sys/loongarch/loongarch/copyinout.S
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/copyinout.S
@@ -0,0 +1,178 @@
+/*-
+ * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2019 Mitchell Horne
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+#include <machine/loongarchreg.h>
+#include <sys/errno.h>
+
+#include "assym.inc"
+
+/*
+ * Fault handler for the copy{in,out} functions below.
+ */
+ENTRY(copyio_fault)
+ SET_FAULT_HANDLER($zero, $a1) /* Clear the handler */
+ EXIT_USER_ACCESS($a1)
+copyio_fault_nopcb:
+ li.d $a0, EFAULT
+ jirl $zero, $ra, 0
+END(copyio_fault)
+
+/*
+ * copycommon - common copy routine
+ *
+ * a0 - Source address
+ * a1 - Destination address
+ * a2 - Size of copy
+ */
+ .macro copycommon
+ la.pcrel $a6, _C_LABEL(copyio_fault) /* Get the handler address */
+ SET_FAULT_HANDLER($a6, $a7) /* Set the handler */
+ ENTER_USER_ACCESS($a7)
+
+ li.d $t2, GRLEN_BYTES
+ blt $a2, $t2, 4f /* Byte-copy if len < GRLEN_BYTES */
+
+ /*
+ * Compare lower bits of src and dest.
+ * If they are aligned with each other, we can do word copy.
+ */
+ li.d $t2, (GRLEN_BYTES-1)
+ and $t0, $a0, $t2 /* Low bits of src */
+ and $t1, $a1, $t2 /* Low bits of dest */
+ bne $t0, $t1, 4f /* Misaligned. Go to byte copy */
+ li.d $t2, GRLEN_BYTES
+ beqz $t0, 2f /* Already word-aligned, skip ahead */
+
+ /* Byte copy until the first word-aligned address */
+1: ld.b $a4, $a0, 0 /* Load byte from src */
+ addi.d $a0, $a0, 1
+ st.b $a4, $a1, 0 /* Store byte in dest */
+ addi.d $a1, $a1, 1
+ addi.d $a2, $a2, -1 /* len-- */
+ li.d $t1, (GRLEN_BYTES-1)
+ and $t0, $a0, $t1
+ bnez $t0, $1b
+ la.pcrel $t3, 3f
+ jr $t3
+
+ /* Copy words */
+2: ld.d $a4, $a0, 0 /* Load word from src */
+ addi.d $a0, $a0, GRLEN_BYTES
+ st.d $a4, $a1, 0 /* Store word in dest */
+ addi.d $a1, $a1, GRLEN_BYTES
+ addi.d $a2, $a2, -GRLEN_BYTES /* len -= GRLEN_BYTES */
+3: bgeu $a2, $t2, 2b /* Again if len >= GRLEN_BYTES */
+
+ /* Check if we're finished */
+ beqz $a2, 5f
+
+ /* Copy any remaining bytes */
+4: ld.b $a4, $a0, 0 /* Load byte from src */
+ addi.d $a0, $a0, 1
+ st.b $a4, $a1, 0 /* Store byte in dest */
+ addi.d $a1, $a1, 1
+ addi.d $a2, $a2, -1 /* len-- */
+ bnez $a2, 4b
+
+5: EXIT_USER_ACCESS($a7)
+ SET_FAULT_HANDLER($zero, $a7) /* Clear the handler */
+ .endm
+
+/*
+ * Copies from a kernel to user address
+ *
+ * int copyout(const void *kaddr, void *udaddr, size_t len)
+ */
+ENTRY(copyout)
+ beqz $a2, copyout_end /* If len == 0 then skip loop */
+ add.d $a3, $a1, $a2
+ li.d $a4, VM_MAXUSER_ADDRESS
+ bgeu $a3, $a4, copyio_fault_nopcb
+
+ copycommon
+
+copyout_end:
+ li.d $a0, 0 /* return 0 */
+ jirl $zero, $ra, 0
+END(copyout)
+
+/*
+ * Copies from a user to kernel address
+ *
+ * int copyin(const void *uaddr, void *kaddr, size_t len)
+ */
+ENTRY(copyin)
+ beqz $a2, copyin_end /* If len == 0 then skip loop */
+ add.d $a3, $a0, $a2
+ li.d $a4, VM_MAXUSER_ADDRESS
+ bgeu $a3, $a4, copyio_fault_nopcb
+
+ copycommon
+
+copyin_end:
+ li.d $a0, 0 /* return 0 */
+ jirl $zero, $ra, 0
+END(copyin)
+
+/*
+ * Copies a string from a user to kernel address
+ *
+ * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
+ */
+ENTRY(copyinstr)
+ move $a5, $zero /* count = 0 */
+ beqz $a2, 3f /* If len == 0 then skip loop */
+
+ la.pcrel $a6, _C_LABEL(copyio_fault) /* Get the handler address */
+ SET_FAULT_HANDLER($a6, $a7) /* Set the handler */
+ ENTER_USER_ACCESS($a7)
+
+ li.d $t0, VM_MAXUSER_ADDRESS
+1: bgeu $a0, $t0, copyio_fault
+ ld.b $a4, $a0, 0 /* Load from uaddr */
+ addi.d $a0, $a0, 1
+ st.b $a4, $a1, 0 /* Store in kaddr */
+ addi.d $a1, $a1, 1
+ beqz $a4, 2f
+ addi.d $a2, $a2, -1 /* len-- */
+ addi.d $a5, $a5, 1 /* count++ */
+ bnez $a2, 1b
+
+2: EXIT_USER_ACCESS($a7)
+ SET_FAULT_HANDLER($zero, $a7) /* Clear the handler */
+
+3: beqz $a3, 4f /* Check if done != NULL */
+ addi.d $a5, $a5, 1 /* count++ */
+ st.d $a5, $a3, 0 /* done = count */
+
+4: move $a0, $zero /* return 0 */
+ beqz $a4, 5f
+ li.d $a0, ENAMETOOLONG
+5:
+ jirl $zero, $ra, 0
+END(copyinstr)
diff --git a/sys/loongarch/loongarch/cpufunc_asm.S b/sys/loongarch/loongarch/cpufunc_asm.S
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/cpufunc_asm.S
@@ -0,0 +1,33 @@
+/*-
+ * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+ .text
+ .align 2
+
+ENTRY(loongarch_nullop)
+ jr $ra
+END(loongarch_nullop)
diff --git a/sys/loongarch/loongarch/cpuic.c b/sys/loongarch/loongarch/cpuic.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/cpuic.c
@@ -0,0 +1,400 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * This code implements the CPU interrupt controller (CPUINTC) for LoongArch,
+ * serving as the root PIC in the INTRNG interrupt framework.
+ *
+ * The CPUINTC manages 14 internal interrupt lines via CSR registers:
+ * - CSR_ECFG (0x4): Exception Config — 14-bit interrupt mask
+ * - CSR_ESTAT (0x5): Exception Status — 14-bit pending status
+ *
+ * Hardware interrupt vector layout:
+ * 0: SWI0, 1: SWI1, 2-9: HWI0-7, 10: PCOV, 11: Timer, 12: IPI
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/proc.h>
+#include <sys/cpuset.h>
+#include <sys/interrupt.h>
+#include <sys/smp.h>
+
+#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#include <machine/frame.h>
+#include <machine/intr.h>
+#include <machine/loongarchreg.h>
+#include <machine/machdep.h> /* loongarch_efi_acpi_rsdp() */
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include "pic_if.h"
+#include "intc.h"
+
+struct cpuic_irqsrc {
+ struct intr_irqsrc isrc;
+ u_int irq;
+};
+
+struct cpuic_softc {
+ device_t dev;
+ struct cpuic_irqsrc isrcs[INTC_NIRQS];
+};
+
+static struct cpuic_softc *cpuic_sc; /* for the CPU-local intr helpers */
+
+/*
+ * Sentinel xref used when the cpuic is directly instantiated (ACPI, no FDT
+ * node) -- it is the root PIC and is reached via cpuic_setup_cpu_intr().
+ */
+#define CPUIC_INTR_XREF ((phandle_t)1)
+
+static int cpuic_intr(void *arg);
+
+static int
+cpuic_probe(device_t dev)
+{
+ /*
+ * Two attach paths, kept strictly separate by whether the device has
+ * an OFW node:
+ *
+ * - FDT path (ofwbus child, node != -1): match ONLY by the
+ * "loongson,cpu-interrupt-controller" compatible. Do NOT fall back
+ * to the device name here -- ofwbus may hand several FDT children
+ * the same name, and matching by name would make cpuic claim every
+ * one of them (the cpuic1 spam / "board broken" regression).
+ *
+ * - nexus/ACPI path (direct child, node == -1): match by name
+ * "cpuic", the name nexus_add_child() gives it.
+ */
+
+ if (ofw_bus_get_node(dev) != -1) {
+ /*
+ * ACPI is the preferred path: ignore the FDT node when ACPI is
+ * present, otherwise the ofwbus enumeration duplicates the
+ * nexus (ACPI) instance (cpuic0 from nexus vs cpuic1 from
+ * ofwbus). The FDT node is only used when ACPI is absent
+ * (pure-FDT systems).
+ */
+ if (loongarch_efi_acpi_rsdp() != 0)
+ return (ENXIO);
+ if (!ofw_bus_is_compatible(dev,
+ "loongson,cpu-interrupt-controller"))
+ return (ENXIO);
+ } else {
+ if (strcmp(device_get_name(dev), "cpuic") != 0)
+ return (ENXIO);
+ }
+
+ device_set_desc(dev, "LoongArch CPU Interrupt Controller");
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+cpuic_attach(device_t dev)
+{
+ struct cpuic_irqsrc *isrcs;
+ struct cpuic_softc *sc;
+ struct intr_pic *pic;
+ const char *name;
+ phandle_t node, xref;
+ u_int flags;
+ uint32_t ecfg;
+ int i, error;
+
+ /*
+ * Single instance: the FDT and direct/nexus paths must not both
+ * attach.
+ */
+ if (cpuic_sc != NULL)
+ return (ENXIO);
+
+ sc = device_get_softc(dev);
+ sc->dev = dev;
+
+ /* Mask all interrupts — clear stale firmware IM bits in ECFG. */
+ ecfg = csr_read32(LOONGARCH_CSR_ECFG);
+ ecfg &= ~CSR_ECFG_IM;
+ csr_write32(ecfg, LOONGARCH_CSR_ECFG);
+
+ name = device_get_nameunit(dev);
+ /*
+ * FDT devices carry an OFW node (use its xref); a directly-instantiated
+ * nexus child has none, so use a fixed sentinel xref. Devices that
+ * chain to the cpuic (timer, eioic) reach it via
+ * cpuic_setup_cpu_intr(), not via the OFW
+ * interrupt-parent/xref lookup, so the sentinel is fine.
+ */
+ node = ofw_bus_get_node(dev);
+ xref = (node > 0) ? OF_xref_from_node(node) : CPUIC_INTR_XREF;
+
+ isrcs = sc->isrcs;
+ for (i = 0; i < INTC_NIRQS; i++) {
+ isrcs[i].irq = i;
+ flags = 0;
+ if (i == IRQ_IPI)
+ flags |= INTR_ISRCF_IPI;
+ /*
+ * Only per-CPU interrupts (timer, PMC, IPI) are PPIs.
+ * HWI0-7 are shared — e.g., HWI0 carries all EIOINTC
+ * device interrupts and must NOT be dispatched on
+ * multiple CPUs simultaneously.
+ */
+ if (i == IRQ_TI || i == IRQ_PCOV)
+ flags |= INTR_ISRCF_PPI;
+ error = intr_isrc_register(&isrcs[i].isrc, sc->dev, flags,
+ "%s,%u", name, i);
+ if (error != 0) {
+ device_printf(dev, "Can't register interrupt %d\n", i);
+ return (error);
+ }
+ }
+
+ cpuic_sc = sc;
+
+ pic = intr_pic_register(sc->dev, xref);
+ if (pic == NULL)
+ return (ENXIO);
+
+ return (intr_pic_claim_root(sc->dev, xref, cpuic_intr, sc,
+ INTR_ROOT_IRQ));
+}
+
+static void
+cpuic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ u_int irq;
+ uint32_t val;
+
+ irq = ((struct cpuic_irqsrc *)isrc)->irq;
+ if (irq >= INTC_NIRQS)
+ panic("%s: Unsupported IRQ %u", __func__, irq);
+
+ val = csr_read32(LOONGARCH_CSR_ECFG);
+ val &= ~ECFGF(irq);
+ csr_write32(val, LOONGARCH_CSR_ECFG);
+}
+
+static void
+cpuic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ u_int irq;
+ uint32_t val;
+
+ irq = ((struct cpuic_irqsrc *)isrc)->irq;
+ if (irq >= INTC_NIRQS)
+ panic("%s: Unsupported IRQ %u", __func__, irq);
+
+ val = csr_read32(LOONGARCH_CSR_ECFG);
+ val |= ECFGF(irq);
+ csr_write32(val, LOONGARCH_CSR_ECFG);
+}
+
+static int
+cpuic_map_intr(device_t dev, struct intr_map_data *data,
+ struct intr_irqsrc **isrcp)
+{
+ struct intr_map_data_fdt *daf;
+ struct cpuic_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ if (data->type != INTR_MAP_DATA_FDT)
+ return (ENOTSUP);
+
+ daf = (struct intr_map_data_fdt *)data;
+ if (daf->ncells != 1 || daf->cells[0] >= INTC_NIRQS)
+ return (EINVAL);
+
+ *isrcp = &sc->isrcs[daf->cells[0]].isrc;
+
+ return (0);
+}
+
+static int
+cpuic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
+ struct resource *res, struct intr_map_data *data)
+{
+
+ if (isrc->isrc_flags & INTR_ISRCF_PPI)
+ CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
+
+ return (0);
+}
+
+static void
+cpuic_post_filter(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ /* HWI lines are never masked during dispatch; nothing to do. */
+}
+
+static void
+cpuic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ cpuic_disable_intr(dev, isrc);
+}
+
+static void
+cpuic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ cpuic_enable_intr(dev, isrc);
+}
+
+#ifdef SMP
+static void
+cpuic_init_secondary(device_t dev, uint32_t rootnum)
+{
+ struct cpuic_softc *sc;
+ struct intr_irqsrc *isrc;
+ u_int cpu, irq;
+
+ sc = device_get_softc(dev);
+ cpu = PCPU_GET(cpuid);
+
+ /*
+ * Unmask interrupts that have active handlers on this CPU.
+ * Only enable IRQs with registered handlers to avoid
+ * interrupt storms from unhandled sources (e.g. HWI0).
+ */
+ for (irq = 0; irq < INTC_NIRQS; irq++) {
+ isrc = &sc->isrcs[irq].isrc;
+ if (isrc->isrc_handlers > 0 &&
+ intr_isrc_init_on_cpu(isrc, cpu))
+ cpuic_enable_intr(dev, isrc);
+ }
+}
+#endif
+
+static int
+cpuic_intr(void *arg)
+{
+ struct trapframe *frame;
+ struct cpuic_softc *sc;
+ uint32_t estatus, is;
+ int bit;
+ struct cpuic_irqsrc *src;
+
+ sc = arg;
+ frame = curthread->td_intr_frame;
+
+ estatus = csr_read32(LOONGARCH_CSR_ESTAT);
+ is = estatus & CSR_ESTAT_IS;
+
+ while ((bit = ffs(is)) != 0) {
+ bit--;
+ is &= ~(1U << bit);
+
+ if (bit >= INTC_NIRQS)
+ continue;
+
+ src = &sc->isrcs[bit];
+ if (intr_isrc_dispatch(&src->isrc, frame) != 0) {
+ cpuic_disable_intr(sc->dev, &src->isrc);
+ device_printf(sc->dev, "Stray irq %u disabled\n", bit);
+ }
+ }
+
+ return (FILTER_HANDLED);
+}
+
+#ifdef SMP
+/*
+ * Register the IPI receive filter on the CPU-local IRQ_IPI line and unmask it
+ * in ECFG. The filter itself (which processes the per-CPU pending IPI action
+ * bitmap) lives in mp_machdep.c; this attaches it to the cpuic IRQ_IPI ISRC,
+ * replacing the old loongarch_setup_ipihandler() helper in intr_machdep.c.
+ */
+void
+cpuic_setup_ipi(driver_filter_t filt)
+{
+ struct intr_irqsrc *isrc;
+
+ if (cpuic_sc == NULL)
+ panic("%s: cpuic not initialized", __func__);
+
+ isrc = &cpuic_sc->isrcs[IRQ_IPI].isrc;
+ if (isrc->isrc_event == NULL) {
+ if (intr_event_create(&isrc->isrc_event, isrc, 0, IRQ_IPI,
+ NULL, NULL, NULL, NULL, "ipi") != 0)
+ panic("%s: intr_event_create failed", __func__);
+ cpuic_enable_intr(cpuic_sc->dev, isrc);
+ }
+ if (intr_event_add_handler(isrc->isrc_event, "ipi", filt, NULL, NULL,
+ intr_priority(INTR_TYPE_MISC), INTR_TYPE_MISC, NULL) != 0)
+ panic("%s: intr_event_add_handler failed", __func__);
+}
+#endif /* SMP */
+
+/*
+ * CPU-local interrupt setup for devices without an FDT interrupt node
+ * (e.g. the CPU timer). Registers a handler directly on the cpuic ISRC
+ * for the given ESTAT line and unmasks it in ECFG.
+ */
+int
+cpuic_setup_cpu_intr(const char *name, driver_filter_t *filt,
+ void (*handler)(void *), void *arg, u_int irq, int flags,
+ void **cookiep)
+{
+ struct intr_irqsrc *isrc;
+ int error;
+
+ if (cpuic_sc == NULL || irq >= INTC_NIRQS)
+ panic("%s: cpuic not initialized or bad irq %u", __func__, irq);
+
+ isrc = &cpuic_sc->isrcs[irq].isrc;
+ if (isrc->isrc_event == NULL) {
+ error = intr_event_create(&isrc->isrc_event, isrc, 0, irq,
+ NULL, NULL, NULL, NULL, "int%u", irq);
+ if (error != 0)
+ return (error);
+ cpuic_enable_intr(cpuic_sc->dev, isrc);
+ }
+
+ error = intr_event_add_handler(isrc->isrc_event, name, filt, handler,
+ arg, intr_priority(flags), flags, cookiep);
+ return (error);
+}
+
+static device_method_t cpuic_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_probe, cpuic_probe),
+ DEVMETHOD(device_attach, cpuic_attach),
+
+ /* Interrupt controller interface */
+ DEVMETHOD(pic_disable_intr, cpuic_disable_intr),
+ DEVMETHOD(pic_enable_intr, cpuic_enable_intr),
+ DEVMETHOD(pic_map_intr, cpuic_map_intr),
+ DEVMETHOD(pic_setup_intr, cpuic_setup_intr),
+ DEVMETHOD(pic_post_filter, cpuic_post_filter),
+ DEVMETHOD(pic_pre_ithread, cpuic_pre_ithread),
+ DEVMETHOD(pic_post_ithread, cpuic_post_ithread),
+#ifdef SMP
+ DEVMETHOD(pic_init_secondary, cpuic_init_secondary),
+#endif
+
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_0(cpuic, cpuic_driver, cpuic_methods,
+ sizeof(struct cpuic_softc));
+EARLY_DRIVER_MODULE(cpuic, ofwbus, cpuic_driver, 0, 0,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_FIRST);
+/*
+ * Also register on nexus so the cpuic can be directly instantiated on ACPI
+ * systems (no FDT/ofwbus node). The ACPI guard in nexus_attach() ensures no
+ * nexus "cpuic" child exists on FDT systems, so QEMU's ofwbus path is used.
+ */
+EARLY_DRIVER_MODULE(cpuic, nexus, cpuic_driver, 0, 0,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_FIRST);
diff --git a/sys/loongarch/loongarch/db_disasm.c b/sys/loongarch/loongarch/db_disasm.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/db_disasm.c
@@ -0,0 +1,354 @@
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * LoongArch DDB disassembler.
+ * Instruction encodings verified against QEMU target/loongarch/insns.decode.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <ddb/ddb.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+
+static const char *rname[] = {
+ "zero", "ra", "tp", "sp", "a0", "a1", "a2", "a3",
+ "a4", "a5", "a6", "a7", "t0", "t1", "t2", "t3",
+ "t4", "t5", "t6", "t7", "t8", "r21","fp", "s0",
+ "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8",
+};
+
+#define RD(insn) (((insn) >> 0) & 0x1f)
+#define RJ(insn) (((insn) >> 5) & 0x1f)
+#define RK(insn) (((insn) >> 10) & 0x1f)
+
+static int32_t sext(uint32_t v, int bits)
+{
+ uint32_t s = 1u << (bits - 1);
+ return ((int32_t)(v ^ s) - (int32_t)s);
+}
+
+#define SI12(insn) sext(((insn) >> 10) & 0xfff, 12)
+#define SI14(insn) sext(((insn) >> 10) & 0x3fff, 14)
+#define SI16(insn) sext(((insn) >> 10) & 0xffff, 16)
+#define SI20(insn) sext(((insn) >> 5) & 0xfffff, 20)
+#define UI5(insn) (((insn) >> 0) & 0x1f)
+#define UI12(insn) (((insn) >> 10) & 0xfff)
+#define UI15(insn) (((insn) >> 0) & 0x7fff)
+#define CSR(insn) (((insn) >> 10) & 0x3fff)
+
+/* offs26: (bits[25:10] | bits[9:0]<<16) << 2, 28-bit signed result */
+static int32_t offs26(uint32_t insn)
+{
+ int32_t v = ((insn >> 10) & 0xffff) | (insn & 0x3ff) << 16;
+ return sext(v, 26) << 2;
+}
+/* offs21: (bits[25:10] | bits[4:0]<<16) << 2, 23-bit signed result */
+static int32_t offs21(uint32_t insn)
+{
+ int32_t v = ((insn >> 10) & 0xffff) | (insn & 0x1f) << 16;
+ return sext(v, 21) << 2;
+}
+/* offs16: bits[25:10] << 2, 18-bit signed result */
+static int32_t offs16(uint32_t insn)
+{
+ return sext((insn >> 10) & 0xffff, 16) << 2;
+}
+static void pr_target(vm_offset_t loc, int32_t o)
+{
+ db_printf("%#lx", loc + o);
+}
+
+struct op {
+ const char *name;
+ uint32_t match;
+ uint32_t mask;
+ void (*pr)(struct op *op, vm_offset_t loc, uint32_t insn);
+};
+static void pr_rrr(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %s", op->name,
+ rname[RD(insn)], rname[RJ(insn)], rname[RK(insn)]);
+}
+static void pr_rr_i12(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %d", op->name,
+ rname[RD(insn)], rname[RJ(insn)], SI12(insn));
+}
+static void pr_rr_ui12(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %u", op->name,
+ rname[RD(insn)], rname[RJ(insn)], UI12(insn));
+}
+static void pr_r_i20(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %d", op->name, rname[RD(insn)], SI20(insn));
+}
+static void pr_r_i14s2(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %d", op->name,
+ rname[RD(insn)], rname[RJ(insn)], SI14(insn) * 4);
+}
+static void pr_rr_i16s2(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %d", op->name,
+ rname[RD(insn)], rname[RJ(insn)], SI16(insn) * 4);
+}
+static void pr_r_csr(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %d", op->name, rname[RD(insn)], CSR(insn));
+}
+static void pr_rr(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s", op->name,
+ rname[RD(insn)], rname[RJ(insn)]);
+}
+static void pr_r(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s", op->name);
+}
+static void pr_i15(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%d", op->name, UI15(insn));
+}
+static void pr_brz(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, ", op->name, rname[RJ(insn)]);
+ pr_target(loc, offs21(insn));
+}
+static void pr_brr(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, ", op->name,
+ rname[RJ(insn)], rname[RD(insn)]);
+ pr_target(loc, offs16(insn));
+}
+static void pr_j(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t", op->name);
+ pr_target(loc, offs26(insn));
+}
+static void pr_hint(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%d", op->name, UI15(insn));
+}
+static void pr_move(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s", "move",
+ rname[RD(insn)], rname[RJ(insn)]);
+}
+static void pr_rr_sh5(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %d", op->name,
+ rname[RD(insn)], rname[RJ(insn)], (insn >> 10) & 0x1f);
+}
+static void pr_rr_sh6(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %d", op->name,
+ rname[RD(insn)], rname[RJ(insn)], (insn >> 10) & 0x3f);
+}
+static void pr_bstr_w(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %d, %d", op->name, rname[RD(insn)],
+ rname[RJ(insn)], (insn >> 16) & 0x1f, (insn >> 10) & 0x1f);
+}
+static void pr_bstr_d(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %d, %d", op->name, rname[RD(insn)],
+ rname[RJ(insn)], (insn >> 16) & 0x3f, (insn >> 10) & 0x3f);
+}
+static void pr_invtlb(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%d, %s, %s", op->name,
+ RD(insn), rname[RJ(insn)], rname[RK(insn)]);
+}
+static void pr_am(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%s, %s, %s", op->name,
+ rname[RD(insn)], rname[RK(insn)], rname[RJ(insn)]);
+}
+static void pr_cacop(struct op *op, vm_offset_t loc, uint32_t insn)
+{
+ db_printf("%-9s\t%d, %s, %d", op->name,
+ insn & 0x1f, rname[RJ(insn)], SI12(insn));
+}
+
+/*
+ * Opcode table: match/mask/print-function.
+ *
+ * Format encodings:
+ * @rrr: match bits[31:15], mask 0xffff8000
+ * @rr_i12: match bits[31:22], mask 0xffc00000
+ * @r_i20: match bits[31:25], mask 0xfe000000
+ * @r_i14s2:match bits[31:24], mask 0xff000000
+ * @rr_i16s2:match bits[31:26], mask 0xfc000000
+ * branch: match bits[31:26], mask 0xfc000000
+ *
+ * More specific patterns must come first.
+ */
+static const struct op ops[] = {
+ /*
+ * Move alias: or rd, rj, $zero.
+ * Must be before generic 'or' in the table.
+ */
+ { "move", 0x00150000, 0xfffffc00, pr_move },
+
+ /* Ret alias: jirl $zero, $ra, 0 */
+ { "ret", 0x4c000020, 0xffffffff, pr_r },
+
+ /* Loads/stores: @rr_i12, bits[31:22] */
+ { "ld.b", 0x28000000, 0xffc00000, pr_rr_i12 },
+ { "ld.h", 0x28400000, 0xffc00000, pr_rr_i12 },
+ { "ld.w", 0x28800000, 0xffc00000, pr_rr_i12 },
+ { "ld.d", 0x28c00000, 0xffc00000, pr_rr_i12 },
+ { "st.b", 0x29000000, 0xffc00000, pr_rr_i12 },
+ { "st.h", 0x29400000, 0xffc00000, pr_rr_i12 },
+ { "st.w", 0x29800000, 0xffc00000, pr_rr_i12 },
+ { "st.d", 0x29c00000, 0xffc00000, pr_rr_i12 },
+ { "ld.bu",0x2a000000, 0xffc00000, pr_rr_i12 },
+ { "ld.hu",0x2a400000, 0xffc00000, pr_rr_i12 },
+ { "ld.wu",0x2a800000, 0xffc00000, pr_rr_i12 },
+
+ /* LL/SC: @r_i14s2, bits[31:24] */
+ { "ll.w", 0x20000000, 0xff000000, pr_r_i14s2 },
+ { "sc.w", 0x21000000, 0xff000000, pr_r_i14s2 },
+ { "ll.d", 0x22000000, 0xff000000, pr_r_i14s2 },
+ { "sc.d", 0x23000000, 0xff000000, pr_r_i14s2 },
+
+ /* ALU reg-imm: @rr_i12, bits[31:22] */
+ { "slti", 0x02000000, 0xffc00000, pr_rr_i12 },
+ { "sltui", 0x02400000, 0xffc00000, pr_rr_i12 },
+ { "addi.w",0x02800000, 0xffc00000, pr_rr_i12 },
+ { "addi.d",0x02c00000, 0xffc00000, pr_rr_i12 },
+ { "andi", 0x03400000, 0xffc00000, pr_rr_ui12 },
+ { "ori", 0x03800000, 0xffc00000, pr_rr_ui12 },
+ { "xori", 0x03c00000, 0xffc00000, pr_rr_ui12 },
+
+ /* lui-type: @r_i20, bits[31:25] */
+ { "lu12i.w", 0x14000000, 0xfe000000, pr_r_i20 },
+ { "lu32i.d", 0x16000000, 0xfe000000, pr_r_i20 },
+ { "pcaddi", 0x18000000, 0xfe000000, pr_r_i20 },
+ { "pcalau12i", 0x1a000000, 0xfe000000, pr_r_i20 },
+ { "pcaddu12i", 0x1c000000, 0xfe000000, pr_r_i20 },
+ { "pcaddu18i", 0x1e000000, 0xfe000000, pr_r_i20 },
+
+ /* ALU reg-reg @rrr, bits[31:15] — opcodes from QEMU patterns */
+ #define RRR(m) { NULL, (m), 0xffff8000, pr_rrr }
+ { "add.w", 0x00100000, 0xffff8000, pr_rrr },
+ { "add.d", 0x00108000, 0xffff8000, pr_rrr },
+ { "sub.w", 0x00110000, 0xffff8000, pr_rrr },
+ { "sub.d", 0x00118000, 0xffff8000, pr_rrr },
+ { "slt", 0x00120000, 0xffff8000, pr_rrr },
+ { "sltu", 0x00128000, 0xffff8000, pr_rrr },
+ { "nor", 0x00140000, 0xffff8000, pr_rrr },
+ { "and", 0x00148000, 0xffff8000, pr_rrr },
+ { "or", 0x00150000, 0xffff8000, pr_rrr },
+ { "xor", 0x00158000, 0xffff8000, pr_rrr },
+ { "orn", 0x00160000, 0xffff8000, pr_rrr },
+ { "andn", 0x00168000, 0xffff8000, pr_rrr },
+ { "maskeqz", 0x00130000, 0xffff8000, pr_rrr },
+ { "masknez", 0x00138000, 0xffff8000, pr_rrr },
+ { "sll.w", 0x00170000, 0xffff8000, pr_rrr },
+ { "srl.w", 0x00178000, 0xffff8000, pr_rrr },
+ { "sra.w", 0x00180000, 0xffff8000, pr_rrr },
+ { "sll.d", 0x00188000, 0xffff8000, pr_rrr },
+ { "srl.d", 0x00190000, 0xffff8000, pr_rrr },
+ { "sra.d", 0x00198000, 0xffff8000, pr_rrr },
+ { "mul.w", 0x001c0000, 0xffff8000, pr_rrr },
+ { "mul.d", 0x001d8000, 0xffff8000, pr_rrr },
+ { "mulh.d", 0x001e0000, 0xffff8000, pr_rrr },
+ { "div.d", 0x00220000, 0xffff8000, pr_rrr },
+ { "mod.d", 0x00228000, 0xffff8000, pr_rrr },
+
+ /* Sign extension @rr */
+ { "ext.w.b", 0x00005c00, 0xfffffc00, pr_rr },
+ { "ext.w.h", 0x00005800, 0xfffffc00, pr_rr },
+
+ /* Shifts by immediate: rd, rj, imm5 [14:10] / imm6 [15:10] */
+ { "slli.w", 0x00408000, 0xffff8000, pr_rr_sh5 },
+ { "slli.d", 0x00410000, 0xffff0000, pr_rr_sh6 },
+ { "srli.w", 0x00448000, 0xffff8000, pr_rr_sh5 },
+ { "srli.d", 0x00450000, 0xffff0000, pr_rr_sh6 },
+ { "srai.w", 0x00488000, 0xffff8000, pr_rr_sh5 },
+ { "srai.d", 0x00490000, 0xffff0000, pr_rr_sh6 },
+
+ /* Bitfield insert/pick: rd, rj, msb, lsb */
+ { "bstrins.w", 0x00600000, 0xffe08000, pr_bstr_w },
+ { "bstrins.d", 0x00800000, 0xffc00000, pr_bstr_d },
+ { "bstrpick.w", 0x00608000, 0xffe08000, pr_bstr_w },
+ { "bstrpick.d", 0x00c00000, 0xffc00000, pr_bstr_d },
+ #undef RRR
+
+ /* Indexed loads/stores @rrr, bits[31:15] */
+ { "ldx.b", 0x38000000, 0xffff8000, pr_rrr },
+ { "ldx.h", 0x38040000, 0xffff8000, pr_rrr },
+ { "ldx.w", 0x38080000, 0xffff8000, pr_rrr },
+ { "ldx.d", 0x380c0000, 0xffff8000, pr_rrr },
+ { "stx.b", 0x38100000, 0xffff8000, pr_rrr },
+ { "stx.h", 0x38140000, 0xffff8000, pr_rrr },
+ { "stx.w", 0x38180000, 0xffff8000, pr_rrr },
+ { "stx.d", 0x381c0000, 0xffff8000, pr_rrr },
+
+ /* Atomics @rrr, bits[31:15] — operand order is rd, rk, rj */
+ { "amswap.w", 0x38600000, 0xffff8000, pr_am },
+ { "amswap.d", 0x38608000, 0xffff8000, pr_am },
+ { "amadd.w", 0x38610000, 0xffff8000, pr_am },
+ { "amadd.d", 0x38618000, 0xffff8000, pr_am },
+ { "amand.w", 0x38620000, 0xffff8000, pr_am },
+ { "amand.d", 0x38628000, 0xffff8000, pr_am },
+ { "amor.w", 0x38630000, 0xffff8000, pr_am },
+ { "amor.d", 0x38638000, 0xffff8000, pr_am },
+ { "amxor.w", 0x38640000, 0xffff8000, pr_am },
+ { "amxor.d", 0x38648000, 0xffff8000, pr_am },
+ { "ammax.w", 0x38650000, 0xffff8000, pr_am },
+ { "ammax.d", 0x38658000, 0xffff8000, pr_am },
+ { "ammin.w", 0x38660000, 0xffff8000, pr_am },
+ { "ammin.d", 0x38668000, 0xffff8000, pr_am },
+
+ /* Branches: @r_offs21 / @rr_offs16 / @offs26 */
+ { "beqz", 0x40000000, 0xfc000000, pr_brz },
+ { "bnez", 0x44000000, 0xfc000000, pr_brz },
+ { "jirl", 0x4c000000, 0xfc000000, pr_rr_i16s2 },
+ { "b", 0x50000000, 0xfc000000, pr_j },
+ { "bl", 0x54000000, 0xfc000000, pr_j },
+ { "beq", 0x58000000, 0xfc000000, pr_brr },
+ { "bne", 0x5c000000, 0xfc000000, pr_brr },
+ { "blt", 0x60000000, 0xfc000000, pr_brr },
+ { "bge", 0x64000000, 0xfc000000, pr_brr },
+ { "bltu", 0x68000000, 0xfc000000, pr_brr },
+ { "bgeu", 0x6c000000, 0xfc000000, pr_brr },
+
+ /* Special */
+ { "cacop", 0x06000000, 0xffc00000, pr_cacop },
+ { "dbar", 0x38720000, 0xffff8000, pr_hint },
+ { "syscall", 0x002b0000, 0xffff8000, pr_i15 },
+ { "break", 0x002a0000, 0xffff8000, pr_i15 },
+ { "idle", 0x06488000, 0xffff8000, pr_hint },
+ { "ertn", 0x06483800, 0xffffffff, pr_r },
+ { "invtlb", 0x06498000, 0xffff8000, pr_invtlb },
+ /* csrrd: bits[9:5] must be zero (csrwr=1, csrxchg=reg differ there) */
+ { "csrrd", 0x04000000, 0xff0003e0, pr_r_csr },
+ { NULL, 0, 0, NULL },
+};
+
+vm_offset_t
+db_disasm(vm_offset_t loc, bool altfmt)
+{
+ const struct op *op;
+ uint32_t insn;
+
+ insn = db_get_value(loc, 4, 0);
+
+ for (op = ops; op->name != NULL; op++) {
+ if (((insn ^ op->match) & op->mask) != 0)
+ continue;
+ op->pr((struct op *)op, loc, insn);
+ return (loc + 4);
+ }
+
+ /* Unknown instruction: print raw hex */
+ db_printf("%08x", insn);
+ return (loc + 4);
+}
diff --git a/sys/loongarch/loongarch/db_interface.c b/sys/loongarch/loongarch/db_interface.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/db_interface.c
@@ -0,0 +1,168 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ *
+ * This software was developed by Semihalf under
+ * the sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/proc.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+
+#ifdef KDB
+#include <sys/kdb.h>
+#endif
+
+#include <ddb/ddb.h>
+#include <ddb/db_variables.h>
+
+#include <machine/cpu.h>
+#include <machine/pcb.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static int
+db_frame(struct db_variable *vp, db_expr_t *valuep, int op)
+{
+ long *reg;
+
+ if (kdb_frame == NULL)
+ return (0);
+
+ reg = (long *)((uintptr_t)kdb_frame + (db_expr_t)vp->valuep);
+ if (op == DB_VAR_GET)
+ *valuep = *reg;
+ else
+ *reg = *valuep;
+ return (1);
+}
+
+#define DB_OFFSET(x) (db_expr_t *)offsetof(struct trapframe, x)
+struct db_variable db_regs[] = {
+ { "r0", DB_OFFSET(u.r0), db_frame },
+ { "ra", DB_OFFSET(u.ra), db_frame },
+ { "tp", DB_OFFSET(u.tp), db_frame },
+ { "sp", DB_OFFSET(u.sp), db_frame },
+ { "a0", DB_OFFSET(u.a[0]), db_frame },
+ { "a1", DB_OFFSET(u.a[1]), db_frame },
+ { "a2", DB_OFFSET(u.a[2]), db_frame },
+ { "a3", DB_OFFSET(u.a[3]), db_frame },
+ { "a4", DB_OFFSET(u.a[4]), db_frame },
+ { "a5", DB_OFFSET(u.a[5]), db_frame },
+ { "a6", DB_OFFSET(u.a[6]), db_frame },
+ { "a7", DB_OFFSET(u.a[7]), db_frame },
+ { "t0", DB_OFFSET(u.t[0]), db_frame },
+ { "t1", DB_OFFSET(u.t[1]), db_frame },
+ { "t2", DB_OFFSET(u.t[2]), db_frame },
+ { "t3", DB_OFFSET(u.t[3]), db_frame },
+ { "t4", DB_OFFSET(u.t[4]), db_frame },
+ { "t5", DB_OFFSET(u.t[5]), db_frame },
+ { "t6", DB_OFFSET(u.t[6]), db_frame },
+ { "t7", DB_OFFSET(u.t[7]), db_frame },
+ { "t8", DB_OFFSET(u.t[8]), db_frame },
+ { "r21", DB_OFFSET(u.r21), db_frame },
+ { "fp", DB_OFFSET(u.fp), db_frame },
+ { "s0", DB_OFFSET(u.s[0]), db_frame },
+ { "s1", DB_OFFSET(u.s[1]), db_frame },
+ { "s2", DB_OFFSET(u.s[2]), db_frame },
+ { "s3", DB_OFFSET(u.s[3]), db_frame },
+ { "s4", DB_OFFSET(u.s[4]), db_frame },
+ { "s5", DB_OFFSET(u.s[5]), db_frame },
+ { "s6", DB_OFFSET(u.s[6]), db_frame },
+ { "s7", DB_OFFSET(u.s[7]), db_frame },
+ { "s8", DB_OFFSET(u.s[8]), db_frame },
+ { "crmd", DB_OFFSET(tf_crmd), db_frame },
+ { "prmd", DB_OFFSET(tf_prmd), db_frame },
+ { "euen", DB_OFFSET(tf_euen), db_frame },
+ { "misc", DB_OFFSET(tf_misc), db_frame },
+ { "ecfg", DB_OFFSET(tf_ecfg), db_frame },
+ { "estat", DB_OFFSET(tf_estat), db_frame },
+ { "era", DB_OFFSET(tf_era), db_frame },
+ { "badvaddr", DB_OFFSET(tf_badvaddr), db_frame },
+};
+
+struct db_variable *db_eregs = db_regs + nitems(db_regs);
+
+void
+db_show_mdpcpu(struct pcpu *pc)
+{
+ db_printf("curpmap = %p\n", pc->pc_curpmap);
+ db_printf("pending_ipis = %x\n", pc->pc_pending_ipis);
+ db_printf("cpuid = %u\n", pc->pc_cpuid);
+ db_printf("kernel_sp = 0x%lx\n", pc->pc_kernel_sp);
+ db_printf("asid_value = 0x%lx\n", pc->pc_asid_value);
+}
+
+/*
+ * Read bytes from kernel address space for debugger.
+ */
+int
+db_read_bytes(vm_offset_t addr, size_t size, char *data)
+{
+ jmp_buf jb;
+ void *prev_jb;
+ const char *src;
+ int ret;
+
+ prev_jb = kdb_jmpbuf(jb);
+ ret = setjmp(jb);
+
+ if (ret == 0) {
+ src = (const char *)addr;
+ while (size-- > 0)
+ *data++ = *src++;
+ }
+ (void)kdb_jmpbuf(prev_jb);
+
+ return (ret);
+}
+
+/*
+ * Write bytes to kernel address space for debugger.
+ */
+int
+db_write_bytes(vm_offset_t addr, size_t size, char *data)
+{
+ jmp_buf jb;
+ void *prev_jb;
+ char *dst;
+ int ret;
+
+ prev_jb = kdb_jmpbuf(jb);
+ ret = setjmp(jb);
+ if (ret == 0) {
+ dst = (char *)addr;
+ while (size-- > 0)
+ *dst++ = *data++;
+
+ /* Invalidate I-cache */
+ flush_icache();
+ }
+ (void)kdb_jmpbuf(prev_jb);
+
+ return (ret);
+}
diff --git a/sys/loongarch/loongarch/db_trace.c b/sys/loongarch/loongarch/db_trace.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/db_trace.c
@@ -0,0 +1,133 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ * Copyright (c) 2020 John Baldwin <jhb@FreeBSD.org>
+ *
+ * Portions of this software were developed by Semihalf under
+ * the sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/kdb.h>
+#include <sys/proc.h>
+
+#include <ddb/ddb.h>
+#include <ddb/db_sym.h>
+
+#include <machine/pcb.h>
+#include <machine/loongarchreg.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static void
+db_stack_trace_cmd(struct thread *td, struct unwind_state *frame)
+{
+ const char *name;
+ db_expr_t offset;
+ db_expr_t value;
+ c_db_sym_t sym;
+ uint64_t pc;
+
+ while (1) {
+ pc = frame->pc;
+
+ sym = db_search_symbol(pc, DB_STGY_ANY, &offset);
+ if (sym == C_DB_SYM_NULL) {
+ value = 0;
+ name = "(null)";
+ } else
+ db_symbol_values(sym, &name, &value);
+
+ db_printf("%s() at ", name);
+ db_printsym(frame->pc, DB_STGY_PROC);
+ db_printf("\n");
+
+ if (strcmp(name, "cpu_exception_handler_supervisor") == 0 ||
+ strcmp(name, "cpu_exception_handler_user") == 0) {
+ struct trapframe *tf;
+
+ tf = (struct trapframe *)(uintptr_t)frame->sp;
+ if (!kstack_contains(td, (vm_offset_t)tf,
+ sizeof(*tf))) {
+ db_printf("--- invalid trapframe %p\n", tf);
+ break;
+ }
+
+ if (tf->tf_estat & CSR_ESTAT_IS) {
+ db_printf("--- interrupt %ld\n",
+ (tf->tf_estat & CSR_ESTAT_IS));
+ } else if ((tf->tf_estat & CSR_ESTAT_EXC) ==
+ (EXCCODE_SYS << CSR_ESTAT_EXC_SHIFT)) {
+ db_printf("--- syscall");
+ db_decode_syscall(td, td->td_sa.code);
+ db_printf("\n");
+ } else {
+ db_printf("--- exception %ld, tval = %#lx\n",
+ (tf->tf_estat & CSR_ESTAT_EXC) >>
+ CSR_ESTAT_EXC_SHIFT,
+ tf->tf_badvaddr);
+ }
+ frame->sp = tf->tf_sp;
+ frame->fp = tf->tf_s[0];
+ frame->pc = tf->tf_era;
+ if (!INKERNEL(frame->fp))
+ break;
+ continue;
+ }
+
+ if (strcmp(name, "fork_trampoline") == 0)
+ break;
+
+ if (!unwind_frame(td, frame))
+ break;
+ }
+}
+
+int
+db_trace_thread(struct thread *thr, int count)
+{
+ struct unwind_state frame;
+ struct pcb *ctx;
+
+ ctx = kdb_thr_ctx(thr);
+
+ frame.sp = ctx->pcb_sp;
+ frame.fp = ctx->pcb_s[0];
+ frame.pc = ctx->pcb_ra;
+ db_stack_trace_cmd(thr, &frame);
+ return (0);
+}
+
+void
+db_trace_self(void)
+{
+ struct unwind_state frame;
+
+ frame.sp = (uintptr_t)__builtin_frame_address(1);
+ frame.fp = (uintptr_t)__builtin_frame_address(0);
+ frame.pc = (uintptr_t)db_trace_self;
+ db_stack_trace_cmd(curthread, &frame);
+}
diff --git a/sys/loongarch/loongarch/dump_machdep.c b/sys/loongarch/loongarch/dump_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/dump_machdep.c
@@ -0,0 +1,46 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include "opt_watchdog.h"
+
+#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/kerneldump.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+int do_minidump = 1;
+SYSCTL_INT(_debug, OID_AUTO, minidump, CTLFLAG_RWTUN, &do_minidump, 0,
+ "Enable mini crash dumps");
+
+void
+dumpsys_map_chunk(vm_paddr_t pa, size_t chunk, void **va)
+{
+ printf("dumpsys_map_chunk\n");
+}
diff --git a/sys/loongarch/loongarch/eioic.h b/sys/loongarch/loongarch/eioic.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/eioic.h
@@ -0,0 +1,41 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * Shared definitions for the EIOINTC driver, used by child PICs
+ * (PCH-PIC, PCH-MSI) that need to reference EIOINTC ISRCs directly.
+ */
+
+#ifndef _LOONGARCH_EIOIC_H_
+#define _LOONGARCH_EIOIC_H_
+
+#include <sys/bus.h>
+#include <sys/interrupt.h>
+#include <sys/mutex.h>
+#include <sys/types.h>
+
+#define EIOINTC_MAX_IRQS 256
+
+struct eioic_irqsrc {
+ struct intr_irqsrc isrc;
+ u_int vec;
+ struct intr_irqsrc *child; /* pch_pic ISRC for INTx vectors;
+ * NULL for MSI */
+};
+
+struct eioic_softc {
+ device_t dev;
+ void *intrhand;
+ struct eioic_irqsrc isrcs[EIOINTC_MAX_IRQS];
+ u_int vec_count;
+ struct mtx mtx;
+};
+
+/*
+ * Look up the EIOINTC softc (NULL until the eioic driver has attached).
+ * Used by child PICs (PCH-PIC) that reference EIOINTC ISRCs, and by AP init.
+ */
+struct eioic_softc *eioic_get_softc(void);
+
+#endif /* !_LOONGARCH_EIOIC_H_ */
diff --git a/sys/loongarch/loongarch/eioic.c b/sys/loongarch/loongarch/eioic.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/eioic.c
@@ -0,0 +1,461 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * This code implements the Extended I/O Interrupt Controller (EIOINTC) for
+ * LoongArch. The EIOINTC is the main external interrupt controller, routing
+ * up to 256 device interrupt vectors to CPU HWI lines.
+ *
+ * Register layout (all accessed via IOCSR):
+ * NODEMAP (0x14a0): Node map — inter-node routing
+ * IPMAP (0x14c0): IP map — which CPU interrupt pin each group routes to
+ * ENABLE (0x1600): Enable mask per vector (64-bit x 4)
+ * BOUNCE (0x1680): Bounce / edge-trigger re-assert control
+ * ISR (0x1800): Interrupt Status Register (64-bit x 4)
+ * ROUTE (0x1c00): Routing per vector (8-bit per vector, 4 vectors per reg)
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/types.h>
+#include <sys/smp.h>
+#include <sys/rman.h>
+#include <sys/interrupt.h>
+
+#include <machine/intr.h>
+#include <machine/cpufunc.h>
+#include <machine/loongarchreg.h>
+#include <machine/machdep.h> /* loongarch_efi_acpi_rsdp() */
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include "pic_if.h"
+#include "eioic.h"
+
+#define VEC_REG_COUNT 4
+#define VEC_COUNT_PER_REG 64
+#define VEC_COUNT (VEC_REG_COUNT * VEC_COUNT_PER_REG)
+
+/*
+ * Sentinel xref used when eioic is directly instantiated (ACPI, no FDT
+ * node).
+ */
+#define EIOIC_INTR_XREF ((phandle_t)2)
+
+static device_t eioic_dev; /* single-instance guard */
+
+struct eioic_softc *
+eioic_get_softc(void)
+{
+ device_t d;
+
+ d = devclass_get_device(devclass_find("eioic"), 0);
+ return (d == NULL ? NULL : device_get_softc(d));
+}
+
+void
+eiointc_enable_extioi(void)
+{
+ uint64_t misc;
+
+ misc = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
+ misc |= IOCSR_MISC_FUNC_EXT_IOI_EN;
+ iocsr_write64(misc, LOONGARCH_IOCSR_MISC_FUNC);
+}
+
+static void
+eioic_init_router(int vec_count)
+{
+ int i, ngroups;
+ uint32_t bit, data;
+
+ ngroups = vec_count / 32;
+
+ /*
+ * The previous code wrote 0 to NODEMAP and ROUTE, which selects
+ * no node / no core -- so every device interrupt was dropped before
+ * reaching a CPU and nothing via the EIOINTC ever fired (timer/IPI
+ * use the cpuic directly). QEMU happened to ignore the empty ROUTE;
+ * real 3A6000 does not.
+ */
+
+ /*
+ * Node map: route EVERY group to Node-0 (bit 0 in the low half).
+ * A per-group pattern spreads groups across nodes (group i -> node
+ * 2i) for multi-node load balancing; on a single node BSP that drops
+ * every group except 0 -- only vector 22 (group 0) ever fired.
+ * Empirically group 0's value (0x00020001) delivers to the BSP, so
+ * replicate it for all groups.
+ */
+ for (i = 0; i < ngroups; i++)
+ iocsr_write32(0x00020001,
+ LOONGARCH_IOCSR_EXTIOI_NODEMAP_BASE + i * 4);
+
+ /*
+ * IP map: route every group to the EIOINTC's CPU line. We chain on
+ * HWI1 (ESTAT bit 3); INT_HWI0 is ESTAT bit 2, so the IP bit is
+ * BIT(3 - 2) = BIT(1) = 0x02, replicated across the 4 byte fields.
+ */
+ bit = (1u << 1);
+ data = bit | (bit << 8) | (bit << 16) | (bit << 24);
+ for (i = 0; i < ngroups / 4; i++)
+ iocsr_write32(data,
+ LOONGARCH_IOCSR_EXTIOI_IPMAP_BASE + i * 4);
+
+ /*
+ * Route: deliver every vector to Node-0 Core-0. The low nibble of
+ * each byte is a core bitmap (BIT(core)), so 0 means no core -- use
+ * BIT(0). (pic_bind_intr can re-route individual vectors later.)
+ */
+ bit = (1u << 0);
+ data = bit | (bit << 8) | (bit << 16) | (bit << 24);
+ for (i = 0; i < vec_count / 4; i++)
+ iocsr_write32(data,
+ LOONGARCH_IOCSR_EXTIOI_ROUTE_BASE + i * 4);
+
+ /*
+ * Enable all vectors. Leave BOUNCE clear: with BOUNCE armed the
+ * EIOINTC treats an HTMSI source as level (ISR re-sets after clear
+ * while HTMSI held), which storms on any PCH-PIC input held asserted
+ * (e.g. OHCI). With BOUNCE=0 the ISR is edge (one bit per HTMSI
+ * rising edge), so a held source sets the ISR once. MSI is
+ * unaffected (each MSI write is a discrete rising edge).
+ */
+ for (i = 0; i < ngroups; i++) {
+ iocsr_write32(0xffffffff,
+ LOONGARCH_IOCSR_EXTIOI_EN_BASE + i * 4);
+ iocsr_write32(0,
+ LOONGARCH_IOCSR_EXTIOI_BOUNCE_BASE + i * 4);
+ }
+}
+
+void
+eiointc_enable_vector(device_t dev, u_int vector)
+{
+ struct eioic_softc *sc = device_get_softc(dev);
+ uint32_t reg, bit, val;
+
+ if (vector >= sc->vec_count)
+ return;
+
+ mtx_lock_spin(&sc->mtx);
+ /*
+ * Set the ENABLE bit only. BOUNCE is left untouched: it is
+ * initialized once in eioic_init_router() and should not be
+ * toggled. Clearing BOUNCE on disable would discard edge-
+ * triggered interrupts that arrived while the vector was
+ * masked; setting it on enable would force edge-triggered
+ * behavior on level-triggered sources.
+ */
+ reg = LOONGARCH_IOCSR_EXTIOI_EN_BASE + (vector / 32) * 4;
+ bit = vector % 32;
+ val = iocsr_read32(reg);
+ val |= (1U << bit);
+ iocsr_write32(val, reg);
+ mtx_unlock_spin(&sc->mtx);
+}
+
+void
+eiointc_disable_vector(device_t dev, u_int vector)
+{
+ struct eioic_softc *sc = device_get_softc(dev);
+ uint32_t reg, bit, val;
+
+ if (vector >= sc->vec_count)
+ return;
+
+ mtx_lock_spin(&sc->mtx);
+ /* Clear only ENABLE; BOUNCE is left untouched. */
+ reg = LOONGARCH_IOCSR_EXTIOI_EN_BASE + (vector / 32) * 4;
+ bit = vector % 32;
+ val = iocsr_read32(reg);
+ val &= ~(1U << bit);
+ iocsr_write32(val, reg);
+ mtx_unlock_spin(&sc->mtx);
+}
+
+static int
+eioic_intr(void *arg)
+{
+ struct eioic_softc *sc = arg;
+ struct trapframe *tf;
+ int i, bit;
+ uint64_t isr;
+
+ tf = curthread->td_intr_frame;
+
+ for (i = 0; i < VEC_REG_COUNT; i++) {
+ isr = iocsr_read64(
+ LOONGARCH_IOCSR_EXTIOI_ISR_BASE + (i << 3));
+ if (!isr)
+ continue;
+
+ /*
+ * Clear each pending bit individually after dispatch.
+ * Clearing the whole group at once would lose vectors
+ * that arrive between the clear and the end of the
+ * while loop.
+ */
+ while (isr) {
+ bit = ffsll(isr) - 1;
+ isr &= ~(1UL << bit);
+
+ int irq = bit + VEC_COUNT_PER_REG * i;
+ if (irq >= EIOINTC_MAX_IRQS)
+ continue;
+
+ /* Clear this vector's ISR bit in hardware. */
+ iocsr_write64(1ULL << bit,
+ LOONGARCH_IOCSR_EXTIOI_ISR_BASE + (i << 3));
+
+ /*
+ * Do NOT mask the EIOINTC vector here. The ISR clear
+ * above consumes the edge for MSI sources (nvme,
+ * XHCI, re0); a new MSI sets a fresh ISR bit on the
+ * next pass. Level-triggered INTx (OHCI/EHCI via
+ * pch_pic) is masked/re-armed at the PCH-PIC child
+ * (pre/post_ithread CLR), not here, so the EIOINTC
+ * vector stays enabled.
+ */
+ {
+ struct intr_irqsrc *isrc = sc->isrcs[irq].child;
+
+ if (isrc == NULL)
+ isrc = &sc->isrcs[irq].isrc;
+ /*
+ * Suppress the "Stray vector" printf: with
+ * all vectors enabled at boot, unclaimed
+ * PCH-PIC inputs (no driver) assert during
+ * probe and would flood the console.
+ * intr_isrc_dispatch still counts each
+ * stray internally (vmstat -i).
+ */
+ (void)intr_isrc_dispatch(isrc, tf);
+ }
+ }
+ }
+
+ return (FILTER_HANDLED);
+}
+
+static int
+eioic_probe(device_t dev)
+{
+
+ /*
+ * Two paths. FDT (ofwbus) node: used only when ACPI is absent (FDT
+ * fallback) -- otherwise ofwbus duplicates the nexus (ACPI) instance.
+ * nexus/ACPI direct child (no OFW node): match by name "eioic".
+ */
+ if (ofw_bus_get_node(dev) != -1) {
+ if (loongarch_efi_acpi_rsdp() != 0)
+ return (ENXIO);
+ if (!ofw_bus_is_compatible(dev, "loongson,ls2k2000-eiointc") &&
+ !ofw_bus_is_compatible(dev, "loongson,eiointc"))
+ return (ENXIO);
+ } else {
+ if (strcmp(device_get_name(dev), "eioic") != 0)
+ return (ENXIO);
+ }
+
+ device_set_desc(dev, "LoongArch Extended I/O Interrupt Controller");
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+eioic_attach(device_t dev)
+{
+ struct eioic_softc *sc;
+ struct eioic_irqsrc *isrcs;
+ const char *name;
+ phandle_t node, xref;
+ int irq, error;
+
+ /*
+ * Single instance: the FDT and direct/nexus paths must not both
+ * attach.
+ */
+ if (eioic_dev != NULL)
+ return (ENXIO);
+ eioic_dev = dev;
+
+ sc = device_get_softc(dev);
+ sc->dev = dev;
+ mtx_init(&sc->mtx, "eiointc", NULL, MTX_SPIN);
+
+ sc->vec_count = VEC_COUNT;
+
+ eiointc_enable_extioi();
+ eioic_init_router(sc->vec_count);
+
+ isrcs = sc->isrcs;
+ name = device_get_nameunit(dev);
+ for (irq = 0; irq < EIOINTC_MAX_IRQS; irq++) {
+ isrcs[irq].vec = irq;
+ error = intr_isrc_register(&isrcs[irq].isrc, dev, 0,
+ "%s,%u", name, irq);
+ if (error != 0) {
+ device_printf(dev, "Cannot register isrc %d\n", irq);
+ goto cleanup_isrcs;
+ }
+ }
+
+ /* FDT devices carry an OFW node (use its xref); a directly-instantiated
+ * nexus child has none, so use a fixed sentinel xref. */
+ node = ofw_bus_get_node(dev);
+ xref = (node > 0) ? OF_xref_from_node(node) : EIOIC_INTR_XREF;
+ if (intr_pic_register(dev, xref) == NULL) {
+ device_printf(dev, "Cannot register PIC\n");
+ error = ENXIO;
+ goto cleanup_isrcs;
+ }
+
+ /*
+ * Register our dispatch filter on the cpuic HWI1 ISRC (the EIOINTC is
+ * chained to ESTAT bit 3 via the IPMAP). We register through the
+ * cpuic driver helper because the FDT-to-INTRNG-IRQ resource path
+ * through nexus is not wired for ofwbus/simplebus children, so
+ * bus_setup_intr cannot resolve the HWI1 line to a cpuic ISRC yet.
+ */
+ error = cpuic_setup_cpu_intr(device_get_nameunit(dev),
+ eioic_intr, NULL, sc, IRQ_HWI1, INTR_TYPE_MISC | INTR_MPSAFE,
+ &sc->intrhand);
+ if (error) {
+ device_printf(dev, "could not setup irq handler: %d\n",
+ error);
+ goto cleanup_isrcs;
+ }
+
+ OF_device_register_xref(xref, dev);
+
+ return (0);
+
+cleanup_isrcs:
+ for (irq = 0; irq < EIOINTC_MAX_IRQS; irq++)
+ intr_isrc_deregister(&sc->isrcs[irq].isrc);
+ return (error);
+}
+
+static void
+eioic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ eiointc_disable_vector(dev,
+ ((struct eioic_irqsrc *)isrc)->vec);
+}
+
+static void
+eioic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ eiointc_enable_vector(dev,
+ ((struct eioic_irqsrc *)isrc)->vec);
+}
+
+static int
+eioic_map_intr(device_t dev, struct intr_map_data *data,
+ struct intr_irqsrc **isrcp)
+{
+ struct intr_map_data_fdt *daf;
+ struct eioic_softc *sc;
+ u_int vec;
+
+ sc = device_get_softc(dev);
+
+ if (data->type != INTR_MAP_DATA_FDT)
+ return (ENOTSUP);
+
+ daf = (struct intr_map_data_fdt *)data;
+ if (daf->ncells != 1)
+ return (EINVAL);
+
+ vec = daf->cells[0];
+ if (vec >= sc->vec_count)
+ return (EINVAL);
+
+ *isrcp = &sc->isrcs[vec].isrc;
+ return (0);
+}
+
+static int
+eioic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
+ struct resource *res, struct intr_map_data *data)
+{
+
+ return (0);
+}
+
+static void
+eioic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+ /*
+ * No mask. The ISR clear in eioic_intr() consumes the edge; new MSIs
+ * during ithread processing set a fresh ISR bit and are delivered on
+ * the next HWI1 pass.
+ */
+}
+
+static void
+eioic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ eioic_enable_intr(dev, isrc);
+}
+
+static void
+eioic_post_filter(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ eioic_enable_intr(dev, isrc);
+}
+
+void
+eiointc_init_cpu(void)
+{
+ /*
+ * APs don't touch the EIOINTC. EXTIOI was enabled + the router
+ * programmed once in eioic_attach() (the BSP, before device probe).
+ * Re-enabling EXTIOI on an AP resets the global ROUTE to 0 (dropping
+ * all device interrupts). eiointc_enable() + eiointc_router_init()
+ * are called only on the first CPU of each node; other CPUs just
+ * enable the ECFG pin (done by cpuic_init_secondary()).
+ */
+}
+
+static device_method_t eioic_methods[] = {
+ DEVMETHOD(device_probe, eioic_probe),
+ DEVMETHOD(device_attach, eioic_attach),
+
+ /* PIC interface */
+ DEVMETHOD(pic_disable_intr, eioic_disable_intr),
+ DEVMETHOD(pic_enable_intr, eioic_enable_intr),
+ DEVMETHOD(pic_map_intr, eioic_map_intr),
+ DEVMETHOD(pic_setup_intr, eioic_setup_intr),
+ DEVMETHOD(pic_pre_ithread, eioic_pre_ithread),
+ DEVMETHOD(pic_post_ithread, eioic_post_ithread),
+ DEVMETHOD(pic_post_filter, eioic_post_filter),
+
+ DEVMETHOD_END
+};
+
+static driver_t eioic_driver = {
+ "eioic",
+ eioic_methods,
+ sizeof(struct eioic_softc),
+};
+
+EARLY_DRIVER_MODULE(eioic, simplebus, eioic_driver, 0, 0,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
+/* Also register on nexus for direct instantiation on ACPI systems. */
+EARLY_DRIVER_MODULE(eioic, nexus, eioic_driver, 0, 0,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
diff --git a/sys/loongarch/loongarch/elf_machdep.c b/sys/loongarch/loongarch/elf_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/elf_machdep.c
@@ -0,0 +1,1156 @@
+/*-
+ * Copyright 1996-1998 John D. Polstra.
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2016 Yukishige Shibata <y-shibat@mtd.biglobe.ne.jp>
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/exec.h>
+#include <sys/imgact.h>
+#include <sys/linker.h>
+#include <sys/proc.h>
+#include <sys/reg.h>
+#include <sys/sysctl.h>
+#include <sys/sysent.h>
+#include <sys/imgact_elf.h>
+#include <sys/syscall.h>
+#include <sys/signalvar.h>
+#include <sys/vnode.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_param.h>
+
+#include <machine/elf.h>
+#include <machine/frame.h>
+#include <machine/loongarchreg.h>
+#include <machine/md_var.h>
+#include <machine/pcb.h>
+#include <machine/reg.h>
+
+u_long elf_hwcap;
+
+static struct sysentvec elf64_freebsd_sysvec = {
+ .sv_size = SYS_MAXSYSCALL,
+ .sv_table = sysent,
+ .sv_fixup = __elfN(freebsd_fixup),
+ .sv_sendsig = sendsig,
+ .sv_sigcode = sigcode,
+ .sv_szsigcode = &szsigcode,
+ .sv_name = "FreeBSD ELF64",
+ .sv_coredump = __elfN(coredump),
+ .sv_elf_core_osabi = ELFOSABI_FREEBSD,
+ .sv_elf_core_abi_vendor = FREEBSD_ABI_VENDOR,
+ .sv_elf_core_prepare_notes = __elfN(prepare_notes),
+ .sv_minsigstksz = MINSIGSTKSZ,
+ .sv_minuser = VM_MIN_ADDRESS,
+ .sv_maxuser = 0, /* Filled in during boot. */
+ .sv_usrstack = 0, /* Filled in during boot. */
+ .sv_psstrings = 0, /* Filled in during boot. */
+ .sv_psstringssz = sizeof(struct ps_strings),
+ .sv_stackprot = VM_PROT_READ | VM_PROT_WRITE,
+ .sv_copyout_auxargs = __elfN(freebsd_copyout_auxargs),
+ .sv_copyout_strings = exec_copyout_strings,
+ .sv_setregs = exec_setregs,
+ .sv_fixlimit = NULL,
+ .sv_maxssiz = NULL,
+ .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_TIMEKEEP |
+ SV_ASLR | SV_RNG_SEED_VER,
+ .sv_set_syscall_retval = cpu_set_syscall_retval,
+ .sv_fetch_syscall_args = cpu_fetch_syscall_args,
+ .sv_syscallnames = syscallnames,
+ .sv_shared_page_base = 0, /* Filled in during boot. */
+ .sv_shared_page_len = PAGE_SIZE,
+ .sv_schedtail = NULL,
+ .sv_thread_detach = NULL,
+ .sv_trap = NULL,
+ .sv_hwcap = &elf_hwcap,
+ .sv_onexec_old = exec_onexec_old,
+ .sv_onexit = exit_onexit,
+ .sv_regset_begin = SET_BEGIN(__elfN(regset)),
+ .sv_regset_end = SET_LIMIT(__elfN(regset)),
+};
+INIT_SYSENTVEC(elf64_sysvec, &elf64_freebsd_sysvec);
+
+static Elf64_Brandinfo freebsd_brand_info = {
+ .brand = ELFOSABI_FREEBSD,
+ .machine = EM_LOONGARCH,
+ .compat_3_brand = "FreeBSD",
+ .interp_path = "/libexec/ld-elf.so.1",
+ .sysvec = &elf64_freebsd_sysvec,
+ .interp_newpath = NULL,
+ .brand_note = &elf64_freebsd_brandnote,
+ .flags = BI_CAN_EXEC_DYN | BI_BRAND_NOTE
+};
+SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_FIRST,
+ (sysinit_cfunc_t)elf64_insert_brand_entry, &freebsd_brand_info);
+
+static void
+elf64_register_sysvec(void *arg)
+{
+ struct sysentvec *sv;
+
+ sv = arg;
+ sv->sv_maxuser = VM_MAX_USER_ADDRESS;
+ sv->sv_usrstack = USRSTACK;
+ sv->sv_psstrings = PS_STRINGS;
+ sv->sv_shared_page_base = SHAREDPAGE;
+}
+SYSINIT(elf64_register_sysvec, SI_SUB_VM, SI_ORDER_ANY, elf64_register_sysvec,
+ &elf64_freebsd_sysvec);
+
+static bool debug_kld;
+SYSCTL_BOOL(_debug, OID_AUTO, kld_reloc, CTLFLAG_RW, &debug_kld, 0,
+ "Activate debug prints in elf_reloc_internal()");
+
+void
+elf64_dump_thread(struct thread *td, void *dst, size_t *off)
+{
+
+}
+
+/*
+ * SOP (Stack-Oriented Programming) relocation stack.
+ *
+ * LoongArch SOP relocations use a stack to compute address expressions.
+ * PUSH operations push values onto the stack, arithmetic operations pop
+ * operands and push results, and POP operations pop the final value and
+ * insert it into the instruction's immediate field.
+ *
+ * The stack persists across consecutive calls to elf_reloc_internal()
+ * because FreeBSD processes relocations one at a time.
+ */
+#define RELA_STACK_DEPTH 16
+
+static long rela_stack[RELA_STACK_DEPTH];
+static size_t rela_stack_top;
+static linker_file_t sop_last_lf; /* reset the SOP stack at module
+ * boundary */
+
+static int
+rela_stack_push(long value)
+{
+
+ if (rela_stack_top >= RELA_STACK_DEPTH) {
+ printf("%s: SOP stack overflow\n", __func__);
+ return (-1);
+ }
+ rela_stack[rela_stack_top++] = value;
+ return (0);
+}
+
+static int
+rela_stack_pop(long *value)
+{
+
+ if (rela_stack_top == 0) {
+ printf("%s: SOP stack underflow\n", __func__);
+ return (-1);
+ }
+ *value = rela_stack[--rela_stack_top];
+ return (0);
+}
+
+/*
+ * LoongArch instruction format helpers.
+ *
+ * reg1i20 (pcalau12i, pcaddu12i, lu12i.w):
+ * bits [31:25] = opcode, [24:5] = si20, [4:0] = rd
+ *
+ * reg2i12 (addi.w/d, ld.w/d, st.w/d, andi, ori):
+ * bits [31:22] = opcode, [21:10] = si12/ui12, [9:5] = rj, [4:0] = rd
+ *
+ * reg2i16 (beq, bne, blt, bge):
+ * bits [31:26] = opcode, [25:10] = offs16, [9:5] = rj, [4:0] = rd
+ *
+ * reg1i21 (beqz, bnez):
+ * bits [31:26] = opcode, [25:10] = offs_l[15:0], [9:5] = rj,
+ * [4:0] = offs_h[20:16]
+ *
+ * reg0i26 (b, bl):
+ * bits [31:26] = opcode, [25:10] = offs_l[15:0],
+ * [9:0] = offs_h[25:16]
+ */
+
+/* Insert 20-bit immediate into reg1i20 format (bits [24:5]) */
+static uint32_t
+insn_set_si20(uint32_t insn, int32_t imm)
+{
+
+ insn &= ~(0xfffffU << 5);
+ insn |= ((uint32_t)(imm & 0xfffff) << 5);
+ return (insn);
+}
+
+/* Insert 12-bit immediate into reg2i12 format (bits [21:10]) */
+static uint32_t
+insn_set_imm12(uint32_t insn, int32_t imm)
+{
+
+ insn &= ~(0xfffU << 10);
+ insn |= ((uint32_t)(imm & 0xfff) << 10);
+ return (insn);
+}
+
+/*
+ * Rewrite a reg2i12 instruction into addi.d, preserving rj and rd
+ * (bits [9:0]). addi.d's opcode occupies bits [31:22] (0x02c00000); the
+ * immediate field is left for the caller to fill via insn_set_imm12().
+ *
+ * Kernel modules have no GOT, so a GOT pointer load emitted as
+ * "pcalau12i + ld.d" must be collapsed into a direct address computation
+ * "pcalau12i + addi.d" -- the ld.d is rewritten to addi.d here. This is
+ * the same transform a relaxing linker applies to a R_LARCH_GOT_PC pair.
+ */
+static uint32_t
+insn_to_addi_d(uint32_t insn)
+{
+
+ return ((insn & 0x3ffU) | 0x02c00000U);
+}
+
+/* Insert 16-bit immediate into reg2i16 format (bits [25:10]) */
+static uint32_t
+insn_set_si16(uint32_t insn, int32_t imm)
+{
+
+ insn &= ~(0xffffU << 10);
+ insn |= ((uint32_t)(imm & 0xffff) << 10);
+ return (insn);
+}
+
+/*
+ * Insert 26-bit offset into reg0i26 format (b, bl).
+ * The offset is in bytes; stored in the instruction shifted right by 2.
+ */
+static uint32_t
+insn_set_offs26(uint32_t insn, int64_t offs)
+{
+ uint32_t imm_l, imm_h;
+
+ imm_l = ((uint32_t)offs >> 2) & 0xffff;
+ imm_h = ((uint32_t)(offs >> 2) >> 16) & 0x3ff;
+ insn &= ~((0xffffU << 10) | 0x3ffU);
+ insn |= (imm_l << 10) | imm_h;
+ return (insn);
+}
+
+/*
+ * Insert 21-bit offset into reg1i21 format (beqz, bnez).
+ * The offset is in bytes; stored in the instruction shifted right by 2.
+ */
+static uint32_t
+insn_set_offs21(uint32_t insn, int64_t offs)
+{
+ uint32_t imm_l, imm_h;
+
+ imm_l = ((uint32_t)offs >> 2) & 0xffff;
+ imm_h = ((uint32_t)(offs >> 2) >> 16) & 0x1f;
+ insn &= ~((0xffffU << 10) | 0x1fU);
+ insn |= (imm_l << 10) | imm_h;
+ return (insn);
+}
+
+/*
+ * Insert 16-bit offset into reg2i16 format (conditional branches) for
+ * B16 relocs. The offset is in bytes; stored shifted right by 2,
+ * into bits [25:10].
+ */
+static uint32_t
+insn_set_offs16(uint32_t insn, int64_t offs)
+{
+
+ return (insn_set_si16(insn, offs >> 2));
+}
+
+struct type2str_ent {
+ int type;
+ const char *str;
+};
+
+static const struct type2str_ent t2s[] = {
+ { R_LARCH_NONE, "R_LARCH_NONE" },
+ { R_LARCH_32, "R_LARCH_32" },
+ { R_LARCH_64, "R_LARCH_64" },
+ { R_LARCH_RELATIVE, "R_LARCH_RELATIVE" },
+ { R_LARCH_COPY, "R_LARCH_COPY" },
+ { R_LARCH_JUMP_SLOT, "R_LARCH_JUMP_SLOT" },
+ { R_LARCH_IRELATIVE, "R_LARCH_IRELATIVE" },
+ { R_LARCH_MARK_LA, "R_LARCH_MARK_LA" },
+ { R_LARCH_MARK_PCREL, "R_LARCH_MARK_PCREL" },
+ { R_LARCH_SOP_PUSH_PCREL, "R_LARCH_SOP_PUSH_PCREL" },
+ { R_LARCH_SOP_PUSH_ABSOLUTE, "R_LARCH_SOP_PUSH_ABSOLUTE" },
+ { R_LARCH_SOP_PUSH_DUP, "R_LARCH_SOP_PUSH_DUP" },
+ { R_LARCH_SOP_PUSH_PLT_PCREL, "R_LARCH_SOP_PUSH_PLT_PCREL" },
+ { R_LARCH_SOP_SUB, "R_LARCH_SOP_SUB" },
+ { R_LARCH_SOP_SL, "R_LARCH_SOP_SL" },
+ { R_LARCH_SOP_SR, "R_LARCH_SOP_SR" },
+ { R_LARCH_SOP_ADD, "R_LARCH_SOP_ADD" },
+ { R_LARCH_SOP_AND, "R_LARCH_SOP_AND" },
+ { R_LARCH_SOP_IF_ELSE, "R_LARCH_SOP_IF_ELSE" },
+ { R_LARCH_SOP_POP_32_S_10_5, "R_LARCH_SOP_POP_32_S_10_5" },
+ { R_LARCH_SOP_POP_32_U_10_12, "R_LARCH_SOP_POP_32_U_10_12" },
+ { R_LARCH_SOP_POP_32_S_10_12, "R_LARCH_SOP_POP_32_S_10_12" },
+ { R_LARCH_SOP_POP_32_S_10_16, "R_LARCH_SOP_POP_32_S_10_16" },
+ { R_LARCH_SOP_POP_32_S_10_16_S2,"R_LARCH_SOP_POP_32_S_10_16_S2" },
+ { R_LARCH_SOP_POP_32_S_5_20, "R_LARCH_SOP_POP_32_S_5_20" },
+ { R_LARCH_SOP_POP_32_S_0_5_10_16_S2,
+ "R_LARCH_SOP_POP_32_S_0_5_10_16_S2" },
+ { R_LARCH_SOP_POP_32_S_0_10_10_16_S2,
+ "R_LARCH_SOP_POP_32_S_0_10_10_16_S2" },
+ { R_LARCH_SOP_POP_32_U, "R_LARCH_SOP_POP_32_U" },
+ { R_LARCH_ADD32, "R_LARCH_ADD32" },
+ { R_LARCH_ADD64, "R_LARCH_ADD64" },
+ { R_LARCH_SUB32, "R_LARCH_SUB32" },
+ { R_LARCH_SUB64, "R_LARCH_SUB64" },
+ { R_LARCH_B26, "R_LARCH_B26" },
+ { R_LARCH_PCALA_HI20, "R_LARCH_PCALA_HI20" },
+ { R_LARCH_PCALA_LO12, "R_LARCH_PCALA_LO12" },
+ { R_LARCH_PCALA64_LO20, "R_LARCH_PCALA64_LO20" },
+ { R_LARCH_PCALA64_HI12, "R_LARCH_PCALA64_HI12" },
+ { R_LARCH_GOT_PC_HI20, "R_LARCH_GOT_PC_HI20" },
+ { R_LARCH_GOT_PC_LO12, "R_LARCH_GOT_PC_LO12" },
+ { R_LARCH_64_PCREL, "R_LARCH_64_PCREL" },
+ { R_LARCH_PCADD_HI20, "R_LARCH_PCADD_HI20" },
+ { R_LARCH_PCADD_LO12, "R_LARCH_PCADD_LO12" },
+ { R_LARCH_GOT_PCADD_HI20, "R_LARCH_GOT_PCADD_HI20" },
+ { R_LARCH_GOT_PCADD_LO12, "R_LARCH_GOT_PCADD_LO12" },
+};
+
+static const char *
+reloctype_to_str(int type)
+{
+
+ for (int i = 0; i < nitems(t2s); ++i)
+ if (type == t2s[i].type)
+ return (t2s[i].str);
+ return ("*unknown*");
+}
+
+bool
+elf_is_ifunc_reloc(Elf_Size r_info)
+{
+
+ return (ELF_R_TYPE(r_info) == R_LARCH_IRELATIVE);
+}
+
+/*
+ * Compute the HI20 value for pcalau12i.
+ *
+ * pcalau12i rd, si20 computes: rd = (PC & ~0xfff) + SignExtend(si20 << 12)
+ *
+ * The +0x800 adjustment accounts for sign extension of the LO12 immediate:
+ * if the lower 12 bits of the target are >= 0x800, sign extension of the
+ * 12-bit immediate produces a negative value, so the HI20 page must be
+ * one higher to compensate.
+ */
+static int32_t
+pcala_hi20_offset(Elf_Addr val, Elf_Addr location)
+{
+ int32_t off;
+
+ off = (int32_t)(((val + 0x800) & ~0xfffUL) - (location & ~0xfffUL));
+ return (off >> 12);
+}
+
+/*
+ * Compute the HI20 value for pcaddi/pcaddu12i.
+ *
+ * pcaddi rd, si20 computes: rd = PC + SignExtend(si20 << 12)
+ *
+ * Unlike pcalau12i, pcaddi does NOT clear the lower 12 bits of PC.
+ */
+static int32_t
+pcadd_hi20_offset(Elf_Addr val, Elf_Addr location)
+{
+ int32_t off;
+
+ off = (int32_t)((val + 0x800) - location);
+ return (off >> 12);
+}
+
+/*
+ * Validate a PC-relative branch/call offset: it must be 4-byte aligned and
+ * fit in 'bits' signed bits (the immediate field width per the psABI).
+ * Without this an out-of-range branch is silently truncated and jumps to a
+ * wrong address. Ranges match lld's checkInt()/checkAlignment().
+ */
+static int
+check_pcrel_range(u_long rtype, int64_t val, int bits)
+{
+ int64_t lim = (int64_t)1 << (bits - 1);
+
+ if ((val & 3) != 0) {
+ printf("%s: reloc %lu offset %ld not 4-byte aligned\n",
+ __func__, rtype, (long)val);
+ return (-1);
+ }
+ if (val < -lim || val >= lim) {
+ printf("%s: reloc %lu offset %ld out of %d-bit range\n",
+ __func__, rtype, (long)val, bits);
+ return (-1);
+ }
+ return (0);
+}
+
+static int
+elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
+ int type, int local, elf_lookup_fn lookup)
+{
+ Elf_Size rtype, symidx;
+ const Elf_Rela *rela;
+ Elf_Addr val, addr;
+ Elf64_Addr *where;
+ Elf_Addr addend;
+ uint32_t before32;
+ uint64_t before64;
+ uint32_t *insn32p;
+ int error;
+
+ /*
+ * The SOP relocation stack is global and must persist across the
+ * per-reloc calls that make up one SOP patch sequence, but a sequence
+ * that aborts mid-way would leak stale entries into the next module's
+ * SOP processing. Reset the stack when relocation of a different
+ * module begins. (SOP relocs are legacy psABI v1; modern v2 toolchains
+ * don't emit them, so this is defensive.)
+ */
+ if (lf != sop_last_lf) {
+ rela_stack_top = 0;
+ sop_last_lf = lf;
+ }
+
+ switch (type) {
+ case ELF_RELOC_RELA:
+ rela = (const Elf_Rela *)data;
+ where = (Elf_Addr *)(relocbase + rela->r_offset);
+ insn32p = (uint32_t *)where;
+ addend = rela->r_addend;
+ rtype = ELF_R_TYPE(rela->r_info);
+ symidx = ELF_R_SYM(rela->r_info);
+ break;
+ default:
+ printf("%s:%d unknown reloc type %d\n",
+ __FUNCTION__, __LINE__, type);
+ return (-1);
+ }
+
+ switch (rtype) {
+
+ /* ---- Basic data relocations ---- */
+
+ case R_LARCH_NONE:
+ case R_LARCH_MARK_LA:
+ case R_LARCH_MARK_PCREL:
+ /*
+ * Hint relocations paired with a primary relocation (e.g.
+ * R_LARCH_PCALA_*). The kernel module linker does not perform
+ * linker relaxation, so treat them as no-ops; the paired
+ * relocation is handled on its own entry.
+ */
+ case R_LARCH_RELAX:
+ case R_LARCH_ALIGN:
+ break;
+
+ case R_LARCH_32:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ before32 = *(uint32_t *)where;
+ *(uint32_t *)where = (uint32_t)(addr + addend);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *(uint32_t *)where);
+ break;
+
+ case R_LARCH_32_PCREL:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ before32 = *(uint32_t *)where;
+ *(uint32_t *)where = (uint32_t)(addr + addend -
+ (Elf_Addr)where);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *(uint32_t *)where);
+ break;
+
+ case R_LARCH_64:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ before64 = *where;
+ *where = addr + addend;
+ if (debug_kld)
+ printf("%p %c %-24s %016lx -> %016lx\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before64, *where);
+ break;
+
+ case R_LARCH_RELATIVE:
+ before64 = *where;
+ *where = elf_relocaddr(lf, relocbase + addend);
+ if (debug_kld)
+ printf("%p %c %-24s %016lx -> %016lx\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before64, *where);
+ break;
+
+ case R_LARCH_JUMP_SLOT:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ before64 = *where;
+ *where = addr;
+ if (debug_kld)
+ printf("%p %c %-24s %016lx -> %016lx\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before64, *where);
+ break;
+
+ case R_LARCH_IRELATIVE:
+ addr = relocbase + addend;
+ before64 = *where;
+ *where = ((Elf_Addr (*)(void))addr)();
+ if (debug_kld)
+ printf("%p %c %-24s %016lx -> %016lx\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before64, *where);
+ break;
+
+ /* ---- SOP stack operations ---- */
+
+ case R_LARCH_SOP_PUSH_PCREL:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend - (Elf_Addr)where;
+ return (rela_stack_push(val));
+
+ case R_LARCH_SOP_PUSH_ABSOLUTE:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ return (rela_stack_push(addr + addend));
+
+ case R_LARCH_SOP_PUSH_DUP: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ error = rela_stack_push(opr);
+ if (error != 0)
+ return (-1);
+ return (rela_stack_push(opr));
+ }
+
+ case R_LARCH_SOP_PUSH_PLT_PCREL:
+ /* No PLT in kernel modules; resolve directly. */
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend - (Elf_Addr)where;
+ return (rela_stack_push(val));
+
+ /* SOP arithmetic */
+ case R_LARCH_SOP_SUB:
+ case R_LARCH_SOP_SL:
+ case R_LARCH_SOP_SR:
+ case R_LARCH_SOP_ADD:
+ case R_LARCH_SOP_AND:
+ case R_LARCH_SOP_IF_ELSE: {
+ long opr1, opr2, opr3, result;
+
+ if (rtype == R_LARCH_SOP_IF_ELSE) {
+ error = rela_stack_pop(&opr3);
+ if (error != 0)
+ return (-1);
+ }
+ error = rela_stack_pop(&opr2);
+ if (error != 0)
+ return (-1);
+ error = rela_stack_pop(&opr1);
+ if (error != 0)
+ return (-1);
+
+ switch (rtype) {
+ case R_LARCH_SOP_AND: result = opr1 & opr2; break;
+ case R_LARCH_SOP_ADD: result = opr1 + opr2; break;
+ case R_LARCH_SOP_SUB: result = opr1 - opr2; break;
+ case R_LARCH_SOP_SL: result = opr1 << opr2; break;
+ case R_LARCH_SOP_SR: result = opr1 >> opr2; break;
+ case R_LARCH_SOP_IF_ELSE: result = opr1 ? opr2 : opr3; break;
+ default:
+ return (-1);
+ }
+ return (rela_stack_push(result));
+ }
+
+ /* SOP pop into instruction immediate fields */
+ case R_LARCH_SOP_POP_32_U_10_12:
+ case R_LARCH_SOP_POP_32_S_10_12: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_imm12(*insn32p, opr);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+ }
+
+ case R_LARCH_SOP_POP_32_S_10_16: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_si16(*insn32p, opr);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+ }
+
+ case R_LARCH_SOP_POP_32_S_10_16_S2: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_si16(*insn32p, opr >> 2);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+ }
+
+ case R_LARCH_SOP_POP_32_S_5_20: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_si20(*insn32p, opr);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+ }
+
+ case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_offs21(*insn32p, opr);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+ }
+
+ case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_offs26(*insn32p, opr);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+ }
+
+ case R_LARCH_SOP_POP_32_U: {
+ long opr;
+
+ error = rela_stack_pop(&opr);
+ if (error != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = (uint32_t)opr;
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+ }
+
+ /* ---- Add/Sub data relocations ---- */
+
+ case R_LARCH_ADD32:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ *(int32_t *)where += addr + addend;
+ break;
+
+ case R_LARCH_ADD64:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ *(int64_t *)where += addr + addend;
+ break;
+
+ case R_LARCH_SUB32:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ *(int32_t *)where -= addr + addend;
+ break;
+
+ case R_LARCH_SUB64:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ *(int64_t *)where -= addr + addend;
+ break;
+
+ /* ---- Branch relocations ---- */
+
+ case R_LARCH_B26:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend - (Elf_Addr)where;
+ if (check_pcrel_range(R_LARCH_B26, (int64_t)val, 28) != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_offs26(*insn32p, val);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_B16:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend - (Elf_Addr)where;
+ if (check_pcrel_range(R_LARCH_B16, (int64_t)val, 18) != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_offs16(*insn32p, val);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_B21:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend - (Elf_Addr)where;
+ if (check_pcrel_range(R_LARCH_B21, (int64_t)val, 23) != 0)
+ return (-1);
+ before32 = *insn32p;
+ *insn32p = insn_set_offs21(*insn32p, val);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_CALL36: {
+ /*
+ * Adjacent pcaddu18i + jirl pair patched by one relocation.
+ * hi20 goes into pcaddu18i (si20 at [24:5]); lo16 (the low 18
+ * bits with the lowest 2 bits zero, so >> 2) goes into jirl
+ * (si16 at [25:10]). The +0x20000 adjustment accounts for
+ * sign-extension of the jirl immediate.
+ */
+ uint32_t hi20, lo16;
+
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend - (Elf_Addr)where;
+ /* CALL36 reaches +-128 GiB (38-bit signed on val + 0x20000). */
+ if ((val & 3) != 0 ||
+ (int64_t)(val + (1LL << 17)) < -(1LL << 37) ||
+ (int64_t)(val + (1LL << 17)) >= (1LL << 37)) {
+ printf("%s: R_LARCH_CALL36 offset %ld out of range\n",
+ __func__, (long)(int64_t)val);
+ return (-1);
+ }
+ hi20 = (uint32_t)((val + (1LL << 17)) >> 18) & 0xfffff;
+ lo16 = (uint32_t)(val >> 2) & 0xffff;
+ before32 = *insn32p;
+ *insn32p = insn_set_si20(*insn32p, hi20);
+ insn32p[1] = insn_set_si16(insn32p[1], lo16);
+ if (debug_kld)
+ printf("%p %c %-24s %08x %08x -> %08x %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, insn32p[1], *insn32p, insn32p[1]);
+ break;
+ }
+
+ /* ---- PC-relative addressing (pcalau12i + addi.d) ---- */
+
+ case R_LARCH_PCALA_HI20:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_si20(*insn32p, pcala_hi20_offset(val,
+ (Elf_Addr)where));
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_PCALA_LO12:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_imm12(*insn32p, val);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_PCALA64_LO20:
+ /*
+ * lu32i.d sets bits [51:32] of the absolute address. The
+ * paired pcalau12i+addi.d produce the low 32 bits and lu52i.d
+ * the top 12, so this and HI12 carry (S+A) directly (matching
+ * lld's extractBits(val, 51, 32) / (val, 63, 52)). The prior
+ * code derived a page-delta remainder that collapsed to 0/-1.
+ */
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_si20(*insn32p, (val >> 32) & 0xfffff);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_PCALA64_HI12:
+ /* lu52i.d sets bits [63:52] of the absolute address. */
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_imm12(*insn32p, (val >> 52) & 0xfff);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ /* ---- GOT PC-relative (no GOT in kernel, resolve directly) ---- */
+
+ case R_LARCH_GOT_PC_HI20:
+ /* Same as PCALA_HI20: resolve symbol address directly */
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_si20(*insn32p, pcala_hi20_offset(val,
+ (Elf_Addr)where));
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_GOT_PC_LO12:
+ /*
+ * Resolve a GOT indirection directly. The compiler emitted
+ * "pcalau12i + ld.d" to load &sym from a GOT slot; the paired
+ * HI20 already aims pcalau12i at sym, and with no GOT the
+ * pointer load must become an address computation (ld.d ->
+ * addi.d) so the pair yields &sym rather than loading sym's
+ * value. Without this rewrite the next dereference would
+ * follow the loaded value as an address.
+ */
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_imm12(insn_to_addi_d(*insn32p), val);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ /* ---- PCADD relocations (pcaddi/pcaddu12i) ---- */
+
+ case R_LARCH_PCADD_HI20:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_si20(*insn32p, pcadd_hi20_offset(val,
+ (Elf_Addr)where));
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_PCADD_LO12:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_imm12(*insn32p, val);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_GOT_PCADD_HI20:
+ /* No GOT in kernel; resolve directly as PCADD_HI20 */
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_si20(*insn32p, pcadd_hi20_offset(val,
+ (Elf_Addr)where));
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ case R_LARCH_GOT_PCADD_LO12:
+ /*
+ * Same no-GOT relaxation as R_LARCH_GOT_PC_LO12: the
+ * "pcaddu12i + ld.d" pair loaded &sym from a GOT slot, so the
+ * pointer load must become an address computation (ld.d ->
+ * addi.d) for the pair to yield &sym directly. The paired
+ * PCADD-style HI20 already aims pcaddu12i at sym.
+ */
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ val = addr + addend;
+ before32 = *insn32p;
+ *insn32p = insn_set_imm12(insn_to_addi_d(*insn32p), val);
+ if (debug_kld)
+ printf("%p %c %-24s %08x -> %08x\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before32, *insn32p);
+ break;
+
+ /* ---- PC-relative data ---- */
+
+ case R_LARCH_64_PCREL:
+ error = lookup(lf, symidx, 1, &addr);
+ if (error != 0)
+ return (-1);
+ before64 = *where;
+ *where = addr + addend - (Elf_Addr)where;
+ if (debug_kld)
+ printf("%p %c %-24s %016lx -> %016lx\n", where,
+ (local ? 'l' : 'g'), reloctype_to_str(rtype),
+ before64, *where);
+ break;
+
+ default:
+ printf("kldload: unexpected relocation type %ld, "
+ "symbol index %ld\n", rtype, symidx);
+ return (-1);
+ }
+
+ return (0);
+}
+
+int
+elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
+ elf_lookup_fn lookup)
+{
+
+ return (elf_reloc_internal(lf, relocbase, data, type, 0, lookup));
+}
+
+int
+elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, const void *data,
+ int type, elf_lookup_fn lookup)
+{
+
+ return (elf_reloc_internal(lf, relocbase, data, type, 1, lookup));
+}
+
+int
+elf_cpu_load_file(linker_file_t lf __unused)
+{
+
+ return (0);
+}
+
+int
+elf_cpu_unload_file(linker_file_t lf __unused)
+{
+
+ return (0);
+}
+
+int
+elf_cpu_parse_dynamic(caddr_t loadbase __unused, Elf_Dyn *dynamic __unused)
+{
+
+ return (0);
+}
+
+/*
+ * LSX/LASX register sets: expose vector state additively via
+ * PT_GETREGSET/PT_SETREGSET (note numbers NT_LOONGARCH_LSX/LASX) and
+ * core-dump notes. Legacy PT_GETFPREGS / struct fpreg are unchanged.
+ * Vector state lives in the trapframe's tf_vregs (32-byte slots); LSX uses
+ * the low 16 bytes of each slot, LASX the full 32.
+ */
+static bool
+loongarch_get_lsxregset(struct regset *rs, struct thread *td, void *buf,
+ size_t *sizep)
+{
+ struct trapframe *tf;
+ struct lsxreg *lr;
+ int i;
+
+ *sizep = sizeof(struct lsxreg);
+ if (buf == NULL)
+ return ((td->td_pcb->pcb_fpflags & PCB_FP_LSX_STARTED) != 0);
+ if ((td->td_pcb->pcb_fpflags & PCB_FP_LSX_STARTED) != 0) {
+ tf = td->td_frame;
+ lr = buf;
+ for (i = 0; i < 32; i++)
+ memcpy(&lr->xr_regs[i * 2], &tf->tf_vregs[i][0], 16);
+ } else
+ memset(buf, 0, sizeof(struct lsxreg));
+ return (true);
+}
+
+static bool
+loongarch_set_lsxregset(struct regset *rs, struct thread *td, void *buf,
+ size_t size)
+{
+ struct trapframe *tf = td->td_frame;
+ struct lsxreg *lr = buf;
+ int i;
+
+ for (i = 0; i < 32; i++)
+ memcpy(&tf->tf_vregs[i][0], &lr->xr_regs[i * 2], 16);
+ td->td_pcb->pcb_fpflags |= PCB_FP_STARTED | PCB_FP_LSX_STARTED;
+ tf->tf_euen |= CSR_EUEN_FPEN | CSR_EUEN_LSXEN;
+ return (true);
+}
+
+static struct regset loongarch_regset_lsx = {
+ .note = NT_LOONGARCH_LSX,
+ .size = sizeof(struct lsxreg),
+ .get = loongarch_get_lsxregset,
+ .set = loongarch_set_lsxregset,
+};
+ELF_REGSET(loongarch_regset_lsx);
+
+static bool
+loongarch_get_lasxregset(struct regset *rs, struct thread *td, void *buf,
+ size_t *sizep)
+{
+ struct trapframe *tf;
+ struct lasxreg *lr;
+ int i;
+
+ *sizep = sizeof(struct lasxreg);
+ if (buf == NULL)
+ return ((td->td_pcb->pcb_fpflags & PCB_FP_LASX_STARTED) != 0);
+ if ((td->td_pcb->pcb_fpflags & PCB_FP_LASX_STARTED) != 0) {
+ tf = td->td_frame;
+ lr = buf;
+ for (i = 0; i < 32; i++)
+ memcpy(&lr->xr_regs[i * 4], &tf->tf_vregs[i][0], 32);
+ } else
+ memset(buf, 0, sizeof(struct lasxreg));
+ return (true);
+}
+
+static bool
+loongarch_set_lasxregset(struct regset *rs, struct thread *td, void *buf,
+ size_t size)
+{
+ struct trapframe *tf = td->td_frame;
+ struct lasxreg *lr = buf;
+ int i;
+
+ for (i = 0; i < 32; i++)
+ memcpy(&tf->tf_vregs[i][0], &lr->xr_regs[i * 4], 32);
+ td->td_pcb->pcb_fpflags |= PCB_FP_STARTED | PCB_FP_LSX_STARTED |
+ PCB_FP_LASX_STARTED;
+ tf->tf_euen |= CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN;
+ return (true);
+}
+
+static struct regset loongarch_regset_lasx = {
+ .note = NT_LOONGARCH_LASX,
+ .size = sizeof(struct lasxreg),
+ .get = loongarch_get_lasxregset,
+ .set = loongarch_set_lasxregset,
+};
+ELF_REGSET(loongarch_regset_lasx);
diff --git a/sys/loongarch/loongarch/exception.S b/sys/loongarch/loongarch/exception.S
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/exception.S
@@ -0,0 +1,431 @@
+/*-
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+#include "assym.inc"
+
+#include <machine/trap.h>
+#include <machine/loongarchreg.h>
+
+.altmacro
+.macro save_registers mode
+
+.if \mode == 0 /* We came from userspace. */
+
+ move $t0, $sp /* user sp */
+ csrrd $t1, PERCPU_BASE_KS /* percpu */
+ /* Switch to kernel sp. */
+ ld.d $sp, $t1, PC_KERNEL_SP
+ addi.d $sp, $sp, -(TF_SIZE)
+
+ /* Save user sp. */
+ st.d $t0, $sp, TF_SP
+ /* Save user r21. */
+ st.d $r21, $sp, TF_R21
+
+ /* Load r21 from KS4. */
+ move $r21, $t1
+
+.else
+ move $t0, $sp /* already kernel sp */
+ addi.d $sp, $sp, -(TF_SIZE)
+ /* Save kernel sp. */
+ st.d $t0, $sp, TF_SP
+
+ /* Load from KS4. */
+ csrrd $r21, PERCPU_BASE_KS
+ st.d $r21, $sp, TF_R21
+.endif
+
+ /* Restore t0, t1. */
+ csrrd $t0, EXCEPTION_KS0
+ csrrd $t1, EXCEPTION_KS1
+
+ /* Push GP registers. */
+ st.d $zero, $sp, TF_R0
+ st.d $ra, $sp, TF_RA
+ st.d $tp, $sp, TF_TP
+ st.d $a0, $sp, TF_R4
+ st.d $a1, $sp, TF_R5
+ st.d $a2, $sp, TF_R6
+ st.d $a3, $sp, TF_R7
+ st.d $a4, $sp, TF_R8
+ st.d $a5, $sp, TF_R9
+ st.d $a6, $sp, TF_R10
+ st.d $a7, $sp, TF_R11
+
+ st.d $t0, $sp, TF_R12
+ st.d $t1, $sp, TF_R13
+ st.d $t2, $sp, TF_R14
+ st.d $t3, $sp, TF_R15
+ st.d $t4, $sp, TF_R16
+ st.d $t5, $sp, TF_R17
+ st.d $t6, $sp, TF_R18
+ st.d $t7, $sp, TF_R19
+ st.d $t8, $sp, TF_R20
+ st.d $fp, $sp, TF_R22
+
+ /* save static regiter */
+ st.d $s0, $sp, TF_R23
+ st.d $s1, $sp, TF_R24
+ st.d $s2, $sp, TF_R25
+ st.d $s3, $sp, TF_R26
+ st.d $s4, $sp, TF_R27
+ st.d $s5, $sp, TF_R28
+ st.d $s6, $sp, TF_R29
+ st.d $s7, $sp, TF_R30
+ st.d $s8, $sp, TF_R31
+
+ /* Push CSR registers. */
+ csrrd $t0, LOONGARCH_CSR_CRMD
+ st.d $t0, $sp, TF_CRMD
+ csrrd $t0, LOONGARCH_CSR_PRMD
+ st.d $t0, $sp, TF_PRMD
+ csrrd $t0, LOONGARCH_CSR_EUEN
+ st.d $t0, $sp, TF_EUEN
+ csrrd $t0, LOONGARCH_CSR_MISC
+ st.d $t0, $sp, TF_MISC
+ csrrd $t0, LOONGARCH_CSR_ECFG
+ st.d $t0, $sp, TF_ECFG
+ csrrd $t0, LOONGARCH_CSR_ESTAT
+ st.d $t0, $sp, TF_ESTAT
+ csrrd $t0, LOONGARCH_CSR_ERA
+ st.d $t0, $sp, TF_ERA
+ csrrd $t0, LOONGARCH_CSR_BADV
+ st.d $t0, $sp, TF_BADVADDR
+
+ /* Push FP registers. */
+ csrrd $t0, LOONGARCH_CSR_EUEN
+ andi $t0, $t0, CSR_EUEN_FPEN
+ beqz $t0, PushRegDone_\@
+
+ fst.d $fa0, $sp, TF_FR0
+ fst.d $fa1, $sp, TF_FR1
+ fst.d $fa2, $sp, TF_FR2
+ fst.d $fa3, $sp, TF_FR3
+ fst.d $fa4, $sp, TF_FR4
+ fst.d $fa5, $sp, TF_FR5
+ fst.d $fa6, $sp, TF_FR6
+ fst.d $fa7, $sp, TF_FR7
+ fst.d $ft0, $sp, TF_FR8
+ fst.d $ft1, $sp, TF_FR9
+ fst.d $ft2, $sp, TF_FR10
+ fst.d $ft3, $sp, TF_FR11
+ fst.d $ft4, $sp, TF_FR12
+ fst.d $ft5, $sp, TF_FR13
+ fst.d $ft6, $sp, TF_FR14
+ fst.d $ft7, $sp, TF_FR15
+ fst.d $ft8, $sp, TF_FR16
+ fst.d $ft9, $sp, TF_FR17
+ fst.d $ft10, $sp, TF_FR18
+ fst.d $ft11, $sp, TF_FR19
+ fst.d $ft12, $sp, TF_FR20
+ fst.d $ft13, $sp, TF_FR21
+ fst.d $ft14, $sp, TF_FR22
+ fst.d $ft15, $sp, TF_FR23
+ fst.d $fs0, $sp, TF_FR24
+ fst.d $fs1, $sp, TF_FR25
+ fst.d $fs2, $sp, TF_FR26
+ fst.d $fs3, $sp, TF_FR27
+ fst.d $fs4, $sp, TF_FR28
+ fst.d $fs5, $sp, TF_FR29
+ fst.d $fs6, $sp, TF_FR30
+ fst.d $fs7, $sp, TF_FR31
+
+ movfcsr2gr $t3, $fcsr0
+ st.d $t3, $sp, TF_FCSR0 /* Push the FCSR0 register. */
+
+ /* Push the fcc0-fcc7 registers. */
+ movcf2gr $t3, $fcc0
+ or $t2, $t3, $zero
+ movcf2gr $t3, $fcc1
+ bstrins.d $t2, $t3, 0xf, 0x8
+ movcf2gr $t3, $fcc2
+ bstrins.d $t2, $t3, 0x17, 0x10
+ movcf2gr $t3, $fcc3
+ bstrins.d $t2, $t3, 0x1f, 0x18
+ movcf2gr $t3, $fcc4
+ bstrins.d $t2, $t3, 0x27, 0x20
+ movcf2gr $t3, $fcc5
+ bstrins.d $t2, $t3, 0x2f, 0x28
+ movcf2gr $t3, $fcc6
+ bstrins.d $t2, $t3, 0x37, 0x30
+ movcf2gr $t3, $fcc7
+ bstrins.d $t2, $t3, 0x3f, 0x38
+ st.d $t2, $sp, TF_FCC
+
+ /*
+ * Save vector registers when LSX/LASX is enabled. Each tf_vregs
+ * slot is 32 bytes; vst writes the low 16 (LSX view), xvst the
+ * full 32 (LASX view). No-op until the LSXDIS/LASXDIS handlers
+ * enable the units.
+ */
+ csrrd $t0, LOONGARCH_CSR_EUEN
+ andi $t1, $t0, (CSR_EUEN_LSXEN | CSR_EUEN_LASXEN)
+ beqz $t1, PushRegDone_\@
+ andi $t0, $t0, CSR_EUEN_LASXEN
+ bnez $t0, PushVecLasx_\@
+ .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+ vst $vr\i, $sp, (TF_VREGS + \i * 32)
+ .endr
+ b PushRegDone_\@
+PushVecLasx_\@:
+ .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+ xvst $xr\i, $sp, (TF_VREGS + \i * 32)
+ .endr
+PushRegDone_\@:
+
+.endm
+
+.macro load_registers mode
+.if \mode == 1
+ /*
+ * Disable interrupts for supervisor mode
+ */
+ li.d $t0, 1
+ li.d $t1, CSR_CRMD_IE
+ csrxchg $t0, $t1, LOONGARCH_CSR_CRMD
+.endif
+
+ ld.d $t1, $sp, TF_EUEN
+ csrrd $t0, LOONGARCH_CSR_EUEN
+ li.d $t2, (CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN)
+ and $t0, $t0, $t2
+ and $t1, $t1, $t2
+ beq $t0, $t1, PopRegs_\@
+
+ /* Restore saved EUEN to enable/disable FPU as needed */
+ ld.d $t0, $sp, TF_EUEN
+ csrwr $t0, LOONGARCH_CSR_EUEN
+
+PopRegs_\@:
+
+ /* Pop CSR registers: just restore PRMD and ERA. */
+ ld.d $t0, $sp, TF_PRMD
+ csrwr $t0, LOONGARCH_CSR_PRMD
+ ld.d $t0, $sp, TF_ERA
+ csrwr $t0, LOONGARCH_CSR_ERA
+
+ /* If the FPE is not set, only pop the GP registers. */
+ beqz $t1, PopGP_\@
+
+ /* Pop FP registers. */
+ fld.d $fa0, $sp, TF_FR0
+ fld.d $fa1, $sp, TF_FR1
+ fld.d $fa2, $sp, TF_FR2
+ fld.d $fa3, $sp, TF_FR3
+ fld.d $fa4, $sp, TF_FR4
+ fld.d $fa5, $sp, TF_FR5
+ fld.d $fa6, $sp, TF_FR6
+ fld.d $fa7, $sp, TF_FR7
+ fld.d $ft0, $sp, TF_FR8
+ fld.d $ft1, $sp, TF_FR9
+ fld.d $ft2, $sp, TF_FR10
+ fld.d $ft3, $sp, TF_FR11
+ fld.d $ft4, $sp, TF_FR12
+ fld.d $ft5, $sp, TF_FR13
+ fld.d $ft6, $sp, TF_FR14
+ fld.d $ft7, $sp, TF_FR15
+ fld.d $ft8, $sp, TF_FR16
+ fld.d $ft9, $sp, TF_FR17
+ fld.d $ft10, $sp, TF_FR18
+ fld.d $ft11, $sp, TF_FR19
+ fld.d $ft12, $sp, TF_FR20
+ fld.d $ft13, $sp, TF_FR21
+ fld.d $ft14, $sp, TF_FR22
+ fld.d $ft15, $sp, TF_FR23
+ fld.d $fs0, $sp, TF_FR24
+ fld.d $fs1, $sp, TF_FR25
+ fld.d $fs2, $sp, TF_FR26
+ fld.d $fs3, $sp, TF_FR27
+ fld.d $fs4, $sp, TF_FR28
+ fld.d $fs5, $sp, TF_FR29
+ fld.d $fs6, $sp, TF_FR30
+ fld.d $fs7, $sp, TF_FR31
+
+ ld.d $t0, $sp, TF_FCSR0
+ movgr2fcsr $fcsr0, $t0 /* Pop the fcsr0 register. */
+
+ /* Pop the fcc0-fcc7 registers. */
+ ld.d $t0, $sp, TF_FCC
+ bstrpick.d $t1, $t0, 7, 0
+ movgr2cf $fcc0, $t1
+ bstrpick.d $t1, $t0, 15, 8
+ movgr2cf $fcc1, $t1
+ bstrpick.d $t1, $t0, 23, 16
+ movgr2cf $fcc2, $t1
+ bstrpick.d $t1, $t0, 31, 24
+ movgr2cf $fcc3, $t1
+ bstrpick.d $t1, $t0, 39, 32
+ movgr2cf $fcc4, $t1
+ bstrpick.d $t1, $t0, 47, 40
+ movgr2cf $fcc5, $t1
+ bstrpick.d $t1, $t0, 55, 48
+ movgr2cf $fcc6, $t1
+ bstrpick.d $t1, $t0, 63, 56
+ movgr2cf $fcc7, $t1
+
+ /*
+ * Restore vector registers if LSX/LASX was enabled in the saved
+ * EUEN (written to the CSR above, so vld/xvld are legal here).
+ */
+ ld.d $t1, $sp, TF_EUEN
+ andi $t0, $t1, (CSR_EUEN_LSXEN | CSR_EUEN_LASXEN)
+ beqz $t0, PopGP_\@
+ andi $t0, $t1, CSR_EUEN_LASXEN
+ bnez $t0, PopVecLasx_\@
+ .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+ vld $vr\i, $sp, (TF_VREGS + \i * 32)
+ .endr
+ b PopGP_\@
+PopVecLasx_\@:
+ .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+ xvld $xr\i, $sp, (TF_VREGS + \i * 32)
+ .endr
+PopGP_\@:
+ /* Pop GP registers. */
+ ld.d $ra, $sp, TF_RA
+ ld.d $tp, $sp, TF_TP
+ ld.d $a0, $sp, TF_R4
+ ld.d $a1, $sp, TF_R5
+ ld.d $a2, $sp, TF_R6
+ ld.d $a3, $sp, TF_R7
+ ld.d $a4, $sp, TF_R8
+ ld.d $a5, $sp, TF_R9
+ ld.d $a6, $sp, TF_R10
+ ld.d $a7, $sp, TF_R11
+ ld.d $t0, $sp, TF_R12
+ ld.d $t1, $sp, TF_R13
+ ld.d $t2, $sp, TF_R14
+ ld.d $t3, $sp, TF_R15
+ ld.d $t4, $sp, TF_R16
+ ld.d $t5, $sp, TF_R17
+ ld.d $t6, $sp, TF_R18
+ ld.d $t7, $sp, TF_R19
+ ld.d $t8, $sp, TF_R20
+ ld.d $fp, $sp, TF_R22
+ ld.d $s0, $sp, TF_R23
+ ld.d $s1, $sp, TF_R24
+ ld.d $s2, $sp, TF_R25
+ ld.d $s3, $sp, TF_R26
+ ld.d $s4, $sp, TF_R27
+ ld.d $s5, $sp, TF_R28
+ ld.d $s6, $sp, TF_R29
+ ld.d $s7, $sp, TF_R30
+ ld.d $s8, $sp, TF_R31
+
+.if \mode == 0 /* restore userspace r21. */
+ /* save kernel sp */
+ addi.d $t0, $sp, TF_SIZE
+ st.d $t0, $r21, PC_KERNEL_SP
+ /* Fixup t0. */
+ ld.d $t0, $sp, TF_R12
+
+ ld.d $r21, $sp, TF_R21
+ /* restore user sp */
+ ld.d $sp, $sp, TF_SP
+.else
+ csrrd $r21, PERCPU_BASE_KS
+ /* restore kernel sp */
+ ld.d $sp, $sp, TF_SP
+.endif
+.endm
+
+ .align 12
+ENTRY(cpu_exception_handler)
+ /* Backup t0, t1. */
+ csrwr $t0, EXCEPTION_KS0
+ csrwr $t1, EXCEPTION_KS1
+ /*
+ * Stash the raw ERA in KS2 and KS3 before any code that might
+ * trigger a nested exception (TLB refill, etc.) so that
+ * do_trap_supervisor can compare it against frame->tf_era
+ * to detect stack/TLB corruption.
+ */
+ csrrd $t0, EXCEPTION_KS2
+ csrwr $t0, EXCEPTION_KS3
+
+ csrrd $t0, LOONGARCH_CSR_ERA
+ csrwr $t0, EXCEPTION_KS2
+
+ csrrd $t0, LOONGARCH_CSR_PRMD
+ andi $t0, $t0, CSR_PRMD_PPLV
+ beqz $t0, 1f
+ b cpu_exception_handler_user
+1:
+ b cpu_exception_handler_supervisor
+END(cpu_exception_handler)
+
+ENTRY(cpu_exception_handler_supervisor)
+ save_registers 1
+
+ move $a0, $sp
+ bl _C_LABEL(do_trap_supervisor)
+
+ load_registers 1
+ ertn
+END(cpu_exception_handler_supervisor)
+
+ENTRY(cpu_exception_handler_user)
+ save_registers 0
+ move $a0, $sp
+ bl _C_LABEL(do_trap_user)
+
+ /*
+ * Check for pending ASTs (signals, etc.) before returning to
+ * user. do_trap_user may have queued signals via
+ * call_trapsignal() which must be delivered via ast() before
+ * resuming user execution.
+ *
+ * Sp still points to the trapframe after the C call (callee-
+ * saved per the calling convention), so use it directly as
+ * ARM64 does.
+ */
+1:
+ /*
+ * Disable interrupts
+ */
+ li.d $t0, 1
+ li.d $t1, CSR_CRMD_IE
+ csrxchg $t0, $t1, LOONGARCH_CSR_CRMD
+
+ ld.d $t0, $r21, PC_CURTHREAD
+ ld.d $t0, $t0, TD_AST
+ beqz $t0, 2f /* no ASTs pending */
+
+ /*
+ * Enable interrupts
+ */
+ li.d $t0, CSR_CRMD_IE
+ li.d $t1, CSR_CRMD_IE
+ csrxchg $t0, $t1, LOONGARCH_CSR_CRMD
+
+ move $a0, $sp
+ bl _C_LABEL(ast)
+ b 1b /* re-check for new ASTs */
+2:
+ load_registers 0
+ ertn
+END(cpu_exception_handler_user)
diff --git a/sys/loongarch/loongarch/exec_machdep.c b/sys/loongarch/loongarch/exec_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/exec_machdep.c
@@ -0,0 +1,539 @@
+/*-
+ * Copyright (c) 2014 Andrew Turner
+ * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/exec.h>
+#include <sys/imgact.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/ptrace.h>
+#include <sys/reg.h>
+#include <sys/rwlock.h>
+#include <sys/sched.h>
+#include <sys/signalvar.h>
+#include <sys/syscallsubr.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
+#include <sys/ucontext.h>
+
+#include <machine/cpu.h>
+#include <machine/fpe.h>
+#include <machine/kdb.h>
+#include <machine/pcb.h>
+#include <machine/pte.h>
+#include <machine/loongarchreg.h>
+#include <machine/trap.h>
+#include <machine/watch.h>
+#include <machine/vmparam.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+
+static void get_fpcontext(struct thread *td, mcontext_t *mcp);
+static void set_fpcontext(struct thread *td, mcontext_t *mcp);
+
+int
+fill_regs(struct thread *td, struct reg *regs)
+{
+ struct trapframe *frame;
+
+ frame = td->td_frame;
+ regs->r0 = frame->r0;
+ regs->ra = frame->tf_ra;
+ regs->tp = frame->tf_tp;
+ regs->sp = frame->tf_sp;
+ memcpy(regs->a, frame->tf_a, sizeof(regs->a));
+ memcpy(regs->t, frame->tf_t, sizeof(regs->t));
+ regs->r21 = frame->r21;
+ regs->fp = frame->tf_fp;
+ memcpy(regs->s, frame->tf_s, sizeof(regs->s));
+ regs->era = frame->tf_era;
+ regs->prmd = frame->tf_prmd;
+
+ return (0);
+}
+
+int
+set_regs(struct thread *td, struct reg *regs)
+{
+ struct trapframe *frame;
+
+ frame = td->td_frame;
+ frame->r0 = regs->r0;
+ frame->tf_ra = regs->ra;
+ frame->tf_tp = regs->tp;
+ frame->tf_sp = regs->sp;
+ memcpy(frame->tf_a, regs->a, sizeof(frame->tf_a));
+ memcpy(frame->tf_t, regs->t, sizeof(frame->tf_t));
+ frame->r21 = regs->r21;
+ frame->tf_fp = regs->fp;
+ memcpy(frame->tf_s, regs->s, sizeof(frame->tf_s));
+ frame->tf_era = regs->era;
+
+ return (0);
+}
+
+int
+fill_fpregs(struct thread *td, struct fpreg *regs)
+{
+ struct trapframe *tf;
+
+ tf = td->td_frame;
+
+ if ((td->td_pcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
+ memcpy(&regs->fpr[0], tf->fa, sizeof(tf->fa));
+ memcpy(&regs->fpr[8], tf->ft, sizeof(tf->ft));
+ memcpy(&regs->fpr[24], tf->fs, sizeof(tf->fs));
+ regs->fcc = tf->fcc;
+ regs->fcsr = (uint32_t)tf->fcsr0;
+ } else
+ memset(regs, 0, sizeof(*regs));
+
+ return (0);
+}
+
+int
+set_fpregs(struct thread *td, struct fpreg *regs)
+{
+ struct trapframe *tf;
+
+ /*
+ * Write FP state into the trapframe (what load_registers restores on
+ * return to userland), not the PCB -- writing the PCB had no effect on
+ * resume. fcc and fcsr are written from their explicit fields.
+ */
+ tf = td->td_frame;
+
+ memcpy(tf->fa, &regs->fpr[0], sizeof(tf->fa));
+ memcpy(tf->ft, &regs->fpr[8], sizeof(tf->ft));
+ memcpy(tf->fs, &regs->fpr[24], sizeof(tf->fs));
+ tf->fcc = regs->fcc;
+ tf->fcsr0 = regs->fcsr;
+ td->td_pcb->pcb_fpflags |= PCB_FP_STARTED;
+
+ return (0);
+}
+
+int
+fill_dbregs(struct thread *td, struct dbreg *regs)
+{
+ struct la_watch_image *img = &td->td_md.md_watch;
+ unsigned i;
+
+ memset(regs, 0, sizeof(*regs));
+ regs->db_revision = 1;
+ regs->db_nslots = LA_WATCH_SLOTS;
+ for (i = 0; i < LA_WATCH_SLOTS; i++) {
+ regs->db_slots[i].db_addr = img->slot[i].addr;
+ regs->db_slots[i].db_ctrl = img->slot[i].ctrl;
+ }
+ return (0);
+}
+
+int
+set_dbregs(struct thread *td, struct dbreg *regs)
+{
+ struct la_watch_image *img = &td->td_md.md_watch;
+ uint64_t addr, ctrl;
+ unsigned i;
+
+ /*
+ * Validate each requested slot. ptrace may only arm user-mode
+ * (PLV3) watchpoints on user addresses; strip everything else.
+ */
+ for (i = 0; i < LA_WATCH_SLOTS; i++) {
+ ctrl = regs->db_slots[i].db_ctrl;
+ if (ctrl == 0)
+ continue; /* slot unused */
+
+ addr = regs->db_slots[i].db_addr;
+ if (addr >= VM_MAXUSER_ADDRESS)
+ return (EINVAL);
+
+ /* Keep only load/store/size, and force PLV3-only. */
+ ctrl &= CSR_DBC_LOADEN | CSR_DBC_STOREEN | CSR_DBC_SIZE_MASK;
+ ctrl |= CSR_DBC_PLV3;
+ if ((ctrl & (CSR_DBC_LOADEN | CSR_DBC_STOREEN)) == 0)
+ return (EINVAL);
+
+ regs->db_slots[i].db_addr = addr;
+ regs->db_slots[i].db_ctrl = ctrl;
+ }
+
+ /* Commit the image. */
+ for (i = 0; i < LA_WATCH_SLOTS; i++) {
+ img->slot[i].addr = regs->db_slots[i].db_addr;
+ img->slot[i].ctrl = regs->db_slots[i].db_ctrl;
+ }
+
+ /* Reload the hardware now if the target is the current thread. */
+ if (td == curthread)
+ la_watch_load(td);
+
+ return (0);
+}
+
+void
+exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack)
+{
+ struct trapframe *tf;
+ struct pcb *pcb;
+
+ tf = td->td_frame;
+ pcb = td->td_pcb;
+
+ memset(tf, 0, sizeof(struct trapframe));
+
+ tf->tf_a[0] = stack;
+ tf->tf_sp = STACKALIGN(stack);
+ tf->tf_era = imgp->entry_addr;
+ tf->tf_ra = imgp->entry_addr;
+ tf->tf_prmd |= CSR_PRMD_PIE; /* Enable interrupts. */
+ tf->tf_prmd |= PLV_USER << CSR_PRMD_PPLV_SHIFT; /* User mode. */
+ tf->tf_prmd |= CSR_PRMD_PWE; /* Enable watchpoints in user mode. */
+
+ /* Exec drops any inherited watchpoints. */
+ memset(&td->td_md.md_watch, 0, sizeof(td->td_md.md_watch));
+ td->td_md.md_ss_active = 0;
+
+ pcb->pcb_fpflags &= ~PCB_FP_STARTED;
+}
+
+/* Sanity check these are the same size, they will be memcpy'd to and from */
+
+CTASSERT(sizeof(((struct trapframe *)0)->tf_a) ==
+ sizeof((struct gpregs *)0)->gp_a);
+CTASSERT(sizeof(((struct trapframe *)0)->tf_s) ==
+ sizeof((struct gpregs *)0)->gp_s);
+CTASSERT(sizeof(((struct trapframe *)0)->tf_t) ==
+ sizeof((struct gpregs *)0)->gp_t);
+CTASSERT(sizeof(((struct trapframe *)0)->tf_a) ==
+ sizeof((struct reg *)0)->a);
+CTASSERT(sizeof(((struct trapframe *)0)->tf_s) ==
+ sizeof((struct reg *)0)->s);
+CTASSERT(sizeof(((struct trapframe *)0)->tf_t) ==
+ sizeof((struct reg *)0)->t);
+
+int
+get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret)
+{
+ struct trapframe *tf = td->td_frame;
+
+ mcp->mc_gpregs.r0 = tf->r0;
+ mcp->mc_gpregs.ra = tf->tf_ra;
+ mcp->mc_gpregs.tp = tf->tf_tp;
+ mcp->mc_gpregs.sp = tf->tf_sp;
+ memcpy(mcp->mc_gpregs.gp_a, tf->tf_a, sizeof(mcp->mc_gpregs.gp_a));
+ memcpy(mcp->mc_gpregs.gp_t, tf->tf_t, sizeof(mcp->mc_gpregs.gp_t));
+ mcp->mc_gpregs.r21 = tf->r21;
+ mcp->mc_gpregs.fp = tf->tf_fp;
+ memcpy(mcp->mc_gpregs.gp_s, tf->tf_s, sizeof(mcp->mc_gpregs.gp_s));
+
+ if (clear_ret & GET_MC_CLEAR_RET) {
+ mcp->mc_gpregs.gp_a[0] = 0;
+ mcp->mc_gpregs.gp_t[0] = 0; /* clear syscall error */
+ }
+
+ mcp->mc_gpregs.gp_era = tf->tf_era;
+ mcp->mc_gpregs.gp_prmd = tf->tf_prmd;
+ get_fpcontext(td, mcp);
+
+ return (0);
+}
+
+int
+set_mcontext(struct thread *td, mcontext_t *mcp)
+{
+ struct trapframe *tf;
+
+ tf = td->td_frame;
+
+ /*
+ * PRMD supplies the privilege level and the interrupt state for the
+ * ertn return to user mode. PPLV must remain PLV3; writes to the
+ * PIE and PWE bits are permitted, the remaining bits are read-only.
+ */
+ if ((mcp->mc_gpregs.gp_prmd & CSR_PRMD_PPLV) != PLV_USER ||
+ ((mcp->mc_gpregs.gp_prmd ^ tf->tf_prmd) &
+ ~(CSR_PRMD_PIE | CSR_PRMD_PWE)) != 0)
+ return (EINVAL);
+
+ tf->r0 = mcp->mc_gpregs.r0;
+ tf->tf_ra = mcp->mc_gpregs.ra;
+ tf->tf_tp = mcp->mc_gpregs.tp;
+ tf->tf_sp = mcp->mc_gpregs.sp;
+ memcpy(tf->tf_a, mcp->mc_gpregs.gp_a, sizeof(tf->tf_a));
+ memcpy(tf->tf_t, mcp->mc_gpregs.gp_t, sizeof(tf->tf_t));
+ tf->r21 = mcp->mc_gpregs.r21;
+ tf->tf_fp = mcp->mc_gpregs.fp;
+ memcpy(tf->tf_s, mcp->mc_gpregs.gp_s, sizeof(tf->tf_s));
+
+ tf->tf_era = mcp->mc_gpregs.gp_era;
+ tf->tf_prmd = mcp->mc_gpregs.gp_prmd;
+ /*
+ * EUEN is not part of the mcontext. Re-enable FP for the resumed
+ * thread iff the saved context carries FP state.
+ * sigreturn re-enables LSX/LASX below from the SIMD block.
+ */
+ tf->tf_euen = (mcp->mc_flags & _MC_FP_VALID) != 0 ? CSR_EUEN_FPEN : 0;
+ set_fpcontext(td, mcp);
+
+ return (0);
+}
+
+static void
+get_fpcontext(struct thread *td, mcontext_t *mcp)
+{
+ struct pcb *curpcb;
+
+ critical_enter();
+
+ curpcb = curthread->td_pcb;
+
+ KASSERT(td->td_pcb == curpcb, ("Invalid fpe pcb"));
+
+ if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
+ /*
+ * If we have just been running FPE instructions we will
+ * need to save the state to memcpy it below.
+ */
+ fpe_state_save(td);
+
+ KASSERT((curpcb->pcb_fpflags & ~(PCB_FP_USERMASK |
+ PCB_FP_LSX_STARTED | PCB_FP_LASX_STARTED)) == 0,
+ ("Non-userspace FPE flags set in get_fpcontext"));
+ memcpy(&mcp->mc_fpregs.fp_regs[0], curpcb->fa,
+ sizeof(curpcb->fa));
+ memcpy(&mcp->mc_fpregs.fp_regs[8], curpcb->ft,
+ sizeof(curpcb->ft));
+ memcpy(&mcp->mc_fpregs.fp_regs[24], curpcb->fs,
+ sizeof(curpcb->fs));
+ mcp->mc_fpregs.fp_fcsr = curpcb->pcb_fcsr0;
+ /*
+ * fcc0-fcc7 are not stored in the PCB (only FPRs and fcsr0
+ * are), but the trapframe captured them on entry, so read them
+ * from there to round-trip FP condition codes across signals.
+ */
+ mcp->mc_fpregs.fcc = td->td_frame->fcc;
+ mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags;
+ mcp->mc_flags |= _MC_FP_VALID;
+ }
+
+ critical_exit();
+}
+
+static void
+set_fpcontext(struct thread *td, mcontext_t *mcp)
+{
+ struct trapframe *tf;
+
+ critical_enter();
+
+ if ((mcp->mc_flags & _MC_FP_VALID) != 0) {
+ tf = td->td_frame;
+ /*
+ * On return to userland load_registers restores the FP
+ * register set from the trapframe (fa/ft/fs / fcsr0), not
+ * from the PCB. Install the saved FP state into the trapframe
+ * here: the previous code wrote the PCB, which load_registers
+ * ignores, so on sigreturn the signal handler's FPRs/fcsr
+ * resumed and any async signal corrupted the victim's FP state.
+ * The PCB copy is re-synced from hardware on the next
+ * fpe_state_save, so it needs no separate update.
+ */
+ memcpy(tf->fa, &mcp->mc_fpregs.fp_regs[0], sizeof(tf->fa));
+ memcpy(tf->ft, &mcp->mc_fpregs.fp_regs[8], sizeof(tf->ft));
+ memcpy(tf->fs, &mcp->mc_fpregs.fp_regs[24], sizeof(tf->fs));
+ tf->fcsr0 = mcp->mc_fpregs.fp_fcsr;
+ tf->fcc = mcp->mc_fpregs.fcc;
+ td->td_pcb->pcb_fpflags =
+ mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK;
+ }
+
+ critical_exit();
+}
+
+int
+sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
+{
+ ucontext_t uc;
+ int error;
+
+ if (copyin(uap->sigcntxp, &uc, sizeof(uc)))
+ return (EFAULT);
+
+ error = set_mcontext(td, &uc.uc_mcontext);
+ if (error != 0)
+ return (error);
+
+ /*
+ * Restore SIMD vector state if the signal frame carried it.
+ * EUEN is not part of the mcontext, so re-enable the SIMD units here
+ * (mirroring the LSX/LASX regsets in elf_machdep.c), re-set the
+ * STARTED latches (USERMASK) and put the saved vectors back into the
+ * trapframe so load_registers reinstates them on return to user.
+ */
+ if ((uc.uc_mcontext.mc_flags & _MC_FP_SIMD) != 0) {
+ struct loongarch_simd_ctx sc;
+ struct trapframe *tf = td->td_frame;
+
+ error = copyin((void *)(uintptr_t)uc.uc_mcontext.mc_spare[0],
+ &sc, sizeof(sc));
+ if (error != 0)
+ return (EFAULT);
+ if (sc.d_size != sizeof(sc) ||
+ (sc.d_magic != LOONGARCH_SIMD_MAGIC_LSX &&
+ sc.d_magic != LOONGARCH_SIMD_MAGIC_LASX))
+ return (EINVAL);
+ if (sc.d_magic == LOONGARCH_SIMD_MAGIC_LASX) {
+ tf->tf_euen |= CSR_EUEN_FPEN | CSR_EUEN_LSXEN |
+ CSR_EUEN_LASXEN;
+ for (int i = 0; i < 32; i++)
+ memcpy(tf->tf_vregs[i], sc.d_regs[i], 32);
+ td->td_pcb->pcb_fpflags |= PCB_FP_STARTED |
+ PCB_FP_LSX_STARTED | PCB_FP_LASX_STARTED;
+ } else {
+ tf->tf_euen |= CSR_EUEN_FPEN | CSR_EUEN_LSXEN;
+ for (int i = 0; i < 32; i++)
+ memcpy(tf->tf_vregs[i], sc.d_regs[i], 16);
+ td->td_pcb->pcb_fpflags |=
+ PCB_FP_STARTED | PCB_FP_LSX_STARTED;
+ }
+ }
+
+ /* Restore signal mask. */
+ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);
+
+ return (EJUSTRETURN);
+}
+
+void
+sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
+{
+ struct sigframe *fp, frame;
+ struct sysentvec *sysent;
+ struct trapframe *tf;
+ struct sigacts *psp;
+ struct thread *td;
+ struct proc *p;
+ int onstack;
+ int sig;
+
+ td = curthread;
+ p = td->td_proc;
+ PROC_LOCK_ASSERT(p, MA_OWNED);
+
+ sig = ksi->ksi_signo;
+ psp = p->p_sigacts;
+ mtx_assert(&psp->ps_mtx, MA_OWNED);
+
+ tf = td->td_frame;
+ onstack = sigonstack(tf->tf_sp);
+
+ CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm,
+ catcher, sig);
+
+ /* Allocate and validate space for the signal handler context. */
+ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack &&
+ SIGISMEMBER(psp->ps_sigonstack, sig)) {
+ fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp +
+ td->td_sigstk.ss_size);
+ } else {
+ fp = (struct sigframe *)td->td_frame->tf_sp;
+ }
+
+ /* Make room, keeping the stack aligned */
+ fp--;
+ fp = (struct sigframe *)STACKALIGN(fp);
+
+ /* Fill in the frame to copy out */
+ bzero(&frame, sizeof(frame));
+ get_mcontext(td, &frame.sf_uc.uc_mcontext, 0);
+ /*
+ * If the thread has LSX/LASX active, snapshot its vector state into
+ * the sigframe's SIMD block and record it in the mcontext; sigreturn
+ * restores it so a handler that uses SIMD cannot clobber the preempted
+ * thread's vectors. The block rides along with the frame copyout.
+ */
+ if ((td->td_pcb->pcb_fpflags &
+ (PCB_FP_LSX_STARTED | PCB_FP_LASX_STARTED)) != 0) {
+ frame.sf_simd.d_size = sizeof(frame.sf_simd);
+ frame.sf_simd.d_magic =
+ (td->td_pcb->pcb_fpflags & PCB_FP_LASX_STARTED) != 0 ?
+ LOONGARCH_SIMD_MAGIC_LASX : LOONGARCH_SIMD_MAGIC_LSX;
+ for (int i = 0; i < 32; i++)
+ memcpy(frame.sf_simd.d_regs[i], tf->tf_vregs[i], 32);
+ frame.sf_uc.uc_mcontext.mc_flags |= _MC_FP_SIMD;
+ frame.sf_uc.uc_mcontext.mc_spare[0] =
+ (uint64_t)(uintptr_t)&fp->sf_simd;
+ }
+ frame.sf_si = ksi->ksi_info;
+ frame.sf_uc.uc_sigmask = *mask;
+ frame.sf_uc.uc_stack = td->td_sigstk;
+ frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) != 0 ?
+ (onstack ? SS_ONSTACK : 0) : SS_DISABLE;
+ mtx_unlock(&psp->ps_mtx);
+ PROC_UNLOCK(td->td_proc);
+
+ /* Copy the sigframe out to the user's stack. */
+ if (copyout(&frame, fp, sizeof(*fp)) != 0) {
+ /* Process has trashed its stack. Kill it. */
+ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp);
+ PROC_LOCK(p);
+ sigexit(td, SIGILL);
+ }
+
+ tf->tf_a[0] = sig;
+ tf->tf_a[1] = (register_t)&fp->sf_si;
+ tf->tf_a[2] = (register_t)&fp->sf_uc;
+
+ tf->tf_sp = (register_t)fp;
+
+ sysent = p->p_sysent;
+ if (PROC_HAS_SHP(p))
+ tf->tf_ra = (register_t)PROC_SIGCODE(p);
+ else
+ tf->tf_ra = (register_t)(PROC_PS_STRINGS(p) -
+ *(sysent->sv_szsigcode));
+
+ tf->tf_era = (register_t)catcher;
+
+ CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_era,
+ tf->tf_sp);
+
+ PROC_LOCK(p);
+ mtx_lock(&psp->ps_mtx);
+}
diff --git a/sys/loongarch/loongarch/gdb_machdep.c b/sys/loongarch/loongarch/gdb_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/gdb_machdep.c
@@ -0,0 +1,158 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Mitchell Horne <mhorne@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * LoongArch GDB register numbering follows the architectural order r0..r31,
+ * which is also the layout order of the general-purpose registers in both
+ * struct trapframe and struct pcb, followed by the PC (tf_era). GDB_NREGS is
+ * therefore 33: 32 GPRs + PC.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/proc.h>
+#include <sys/signal.h>
+
+#include <machine/frame.h>
+#include <machine/gdb_machdep.h>
+#include <machine/pcb.h>
+#include <machine/loongarchreg.h>
+
+#include <gdb/gdb.h>
+
+/*
+ * Map a GDB GPR register number onto the matching named field of a
+ * trapframe. Returns NULL if regnum is out of range.
+ */
+static uint64_t *
+tf_gpr(struct trapframe *tf, int regnum)
+{
+
+ switch (regnum) {
+ case GDB_REG_ZERO:
+ return (&tf->r0);
+ case GDB_REG_RA:
+ return (&tf->tf_ra);
+ case GDB_REG_TP:
+ return (&tf->tf_tp);
+ case GDB_REG_SP:
+ return (&tf->tf_sp);
+ case GDB_REG_FP:
+ return (&tf->tf_fp);
+ default:
+ if (regnum >= GDB_REG_A0 && regnum < GDB_REG_T0)
+ return (&tf->tf_a[regnum - GDB_REG_A0]);
+ if (regnum >= GDB_REG_T0 && regnum < GDB_REG_R21)
+ return (&tf->tf_t[regnum - GDB_REG_T0]);
+ if (regnum == GDB_REG_R21)
+ return (&tf->r21);
+ if (regnum >= GDB_REG_S0 && regnum < GDB_REG_PC)
+ return (&tf->tf_s[regnum - GDB_REG_S0]);
+ return (NULL);
+ }
+}
+
+/*
+ * For other threads only the callee-saved registers are valid in the
+ * saved pcb (see savectx): ra, tp, sp, fp, and s0-s8.
+ */
+static uint64_t *
+pcb_gpr(struct pcb *pcb, int regnum)
+{
+
+ switch (regnum) {
+ case GDB_REG_RA:
+ return (&pcb->ra);
+ case GDB_REG_TP:
+ return (&pcb->tp);
+ case GDB_REG_SP:
+ return (&pcb->sp);
+ case GDB_REG_FP:
+ return (&pcb->fp);
+ default:
+ if (regnum >= GDB_REG_S0 && regnum < GDB_REG_PC)
+ return (&pcb->s[regnum - GDB_REG_S0]);
+ return (NULL);
+ }
+}
+
+void *
+gdb_cpu_getreg(int regnum, size_t *regsz)
+{
+ *regsz = gdb_cpu_regsz(regnum);
+
+ if (kdb_thread == curthread) {
+ if (regnum >= 0 && regnum < GDB_REG_PC)
+ return (tf_gpr(kdb_frame, regnum));
+ if (regnum == GDB_REG_PC)
+ return (&kdb_frame->tf_era);
+ return (NULL);
+ }
+
+ if (regnum == GDB_REG_PC) /* Best effort: the resume PC is ra. */
+ return (&kdb_thrctx->ra);
+ return (pcb_gpr(kdb_thrctx, regnum));
+}
+
+void
+gdb_cpu_setreg(int regnum, void *val)
+{
+ register_t regval = *(register_t *)val;
+ uint64_t *reg;
+
+ if (kdb_thread == curthread) {
+ /* $zero (r0) is hardwired to zero by the architecture */
+ if (regnum == GDB_REG_ZERO)
+ return;
+ reg = (regnum > 0 && regnum < GDB_REG_PC) ?
+ tf_gpr(kdb_frame, regnum) : NULL;
+ if (reg != NULL)
+ *reg = regval;
+ else if (regnum == GDB_REG_PC)
+ kdb_frame->tf_era = regval;
+ }
+
+ /* Keep the saved pcb context in sync for the callee-saved registers. */
+ reg = pcb_gpr(kdb_thrctx, regnum);
+ if (reg != NULL)
+ *reg = regval;
+ else if (regnum == GDB_REG_PC)
+ kdb_thrctx->ra = regval;
+}
+
+int
+gdb_cpu_signal(int type, int code)
+{
+
+ if (type == EXCCODE_BP)
+ return (SIGTRAP);
+
+ return (SIGEMT);
+}
diff --git a/sys/loongarch/loongarch/genassym.c b/sys/loongarch/loongarch/genassym.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/genassym.c
@@ -0,0 +1,180 @@
+/*-
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/assym.h>
+#include <sys/proc.h>
+#include <sys/mbuf.h>
+#include <sys/vmmeter.h>
+#include <sys/bus.h>
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+
+#include <machine/loongarchreg.h>
+#include <machine/frame.h>
+#include <machine/pcb.h>
+#include <machine/cpu.h>
+#include <machine/proc.h>
+#include <machine/cpufunc.h>
+#include <machine/pte.h>
+#include <machine/intr.h>
+#include <machine/machdep.h>
+#include <machine/vmparam.h>
+
+ASSYM(KERNBASE, KERNBASE);
+ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
+ASSYM(VM_MAX_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS);
+ASSYM(VM_EARLY_DTB_ADDRESS, VM_EARLY_DTB_ADDRESS);
+ASSYM(PMAP_MAPDEV_EARLY_SIZE, PMAP_MAPDEV_EARLY_SIZE);
+
+ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault));
+ASSYM(PCB_SIZE, sizeof(struct pcb));
+ASSYM(PCB_A0, offsetof(struct pcb, pcb_a0));
+ASSYM(PCB_S, offsetof(struct pcb, s[0]));
+ASSYM(PCB_RA, offsetof(struct pcb, ra));
+ASSYM(PCB_SP, offsetof(struct pcb, sp));
+ASSYM(PCB_TP, offsetof(struct pcb, tp));
+ASSYM(PCB_FP, offsetof(struct pcb, fp));
+ASSYM(PCB_F, offsetof(struct pcb, fa));
+
+ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
+
+ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
+ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
+ASSYM(PC_KERNEL_SP, offsetof(struct pcpu, pc_kernel_sp));
+
+ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
+ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
+ASSYM(TD_AST, offsetof(struct thread, td_ast));
+ASSYM(TD_PROC, offsetof(struct thread, td_proc));
+ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
+ASSYM(TD_MD, offsetof(struct thread, td_md));
+ASSYM(TD_LOCK, offsetof(struct thread, td_lock));
+
+ASSYM(TF_SIZE, roundup2(sizeof(struct trapframe), STACKALIGNBYTES + 1));
+ASSYM(TF_R0, offsetof(struct trapframe, r0));
+ASSYM(TF_R1, offsetof(struct trapframe, ra));
+ASSYM(TF_R2, offsetof(struct trapframe, tp));
+ASSYM(TF_R3, offsetof(struct trapframe, sp));
+ASSYM(TF_R4, offsetof(struct trapframe, a[0]));
+ASSYM(TF_R5, offsetof(struct trapframe, a[1]));
+ASSYM(TF_R6, offsetof(struct trapframe, a[2]));
+ASSYM(TF_R7, offsetof(struct trapframe, a[3]));
+ASSYM(TF_R8, offsetof(struct trapframe, a[4]));
+ASSYM(TF_R9, offsetof(struct trapframe, a[5]));
+ASSYM(TF_R10, offsetof(struct trapframe, a[6]));
+ASSYM(TF_R11, offsetof(struct trapframe, a[7]));
+ASSYM(TF_R12, offsetof(struct trapframe, t[0]));
+ASSYM(TF_R13, offsetof(struct trapframe, t[1]));
+ASSYM(TF_R14, offsetof(struct trapframe, t[2]));
+ASSYM(TF_R15, offsetof(struct trapframe, t[3]));
+ASSYM(TF_R16, offsetof(struct trapframe, t[4]));
+ASSYM(TF_R17, offsetof(struct trapframe, t[5]));
+ASSYM(TF_R18, offsetof(struct trapframe, t[6]));
+ASSYM(TF_R19, offsetof(struct trapframe, t[7]));
+ASSYM(TF_R20, offsetof(struct trapframe, t[8]));
+ASSYM(TF_R21, offsetof(struct trapframe, r21));
+ASSYM(TF_R22, offsetof(struct trapframe, fp));
+ASSYM(TF_R23, offsetof(struct trapframe, s[0]));
+ASSYM(TF_R24, offsetof(struct trapframe, s[1]));
+ASSYM(TF_R25, offsetof(struct trapframe, s[2]));
+ASSYM(TF_R26, offsetof(struct trapframe, s[3]));
+ASSYM(TF_R27, offsetof(struct trapframe, s[4]));
+ASSYM(TF_R28, offsetof(struct trapframe, s[5]));
+ASSYM(TF_R29, offsetof(struct trapframe, s[6]));
+ASSYM(TF_R30, offsetof(struct trapframe, s[7]));
+ASSYM(TF_R31, offsetof(struct trapframe, s[8]));
+
+ASSYM(TF_A, offsetof(struct trapframe, a[0]));
+ASSYM(TF_T, offsetof(struct trapframe, t[0]));
+ASSYM(TF_S, offsetof(struct trapframe, s[0]));
+ASSYM(TF_RA, offsetof(struct trapframe, ra));
+ASSYM(TF_TP, offsetof(struct trapframe, tp));
+ASSYM(TF_SP, offsetof(struct trapframe, sp));
+
+ASSYM(TF_A0, offsetof(struct trapframe, tf_a0));
+ASSYM(TF_ERA, offsetof(struct trapframe, tf_era));
+ASSYM(TF_BADVADDR, offsetof(struct trapframe, tf_badvaddr));
+ASSYM(TF_CRMD, offsetof(struct trapframe, tf_crmd));
+ASSYM(TF_PRMD, offsetof(struct trapframe, tf_prmd));
+ASSYM(TF_EUEN, offsetof(struct trapframe, tf_euen));
+ASSYM(TF_MISC, offsetof(struct trapframe, tf_misc));
+ASSYM(TF_ECFG, offsetof(struct trapframe, tf_ecfg));
+ASSYM(TF_ESTAT, offsetof(struct trapframe, tf_estat));
+
+ASSYM(TF_FR0, offsetof(struct trapframe, fa[0]));
+ASSYM(TF_FR1, offsetof(struct trapframe, fa[1]));
+ASSYM(TF_FR2, offsetof(struct trapframe, fa[2]));
+ASSYM(TF_FR3, offsetof(struct trapframe, fa[3]));
+ASSYM(TF_FR4, offsetof(struct trapframe, fa[4]));
+ASSYM(TF_FR5, offsetof(struct trapframe, fa[5]));
+ASSYM(TF_FR6, offsetof(struct trapframe, fa[6]));
+ASSYM(TF_FR7, offsetof(struct trapframe, fa[7]));
+ASSYM(TF_FR8, offsetof(struct trapframe, ft[0]));
+ASSYM(TF_FR9, offsetof(struct trapframe, ft[1]));
+ASSYM(TF_FR10, offsetof(struct trapframe, ft[2]));
+ASSYM(TF_FR11, offsetof(struct trapframe, ft[3]));
+ASSYM(TF_FR12, offsetof(struct trapframe, ft[4]));
+ASSYM(TF_FR13, offsetof(struct trapframe, ft[5]));
+ASSYM(TF_FR14, offsetof(struct trapframe, ft[6]));
+ASSYM(TF_FR15, offsetof(struct trapframe, ft[7]));
+ASSYM(TF_FR16, offsetof(struct trapframe, ft[8]));
+ASSYM(TF_FR17, offsetof(struct trapframe, ft[9]));
+ASSYM(TF_FR18, offsetof(struct trapframe, ft[10]));
+ASSYM(TF_FR19, offsetof(struct trapframe, ft[11]));
+ASSYM(TF_FR20, offsetof(struct trapframe, ft[12]));
+ASSYM(TF_FR21, offsetof(struct trapframe, ft[13]));
+ASSYM(TF_FR22, offsetof(struct trapframe, ft[14]));
+ASSYM(TF_FR23, offsetof(struct trapframe, ft[15]));
+ASSYM(TF_FR24, offsetof(struct trapframe, fs[0]));
+ASSYM(TF_FR25, offsetof(struct trapframe, fs[1]));
+ASSYM(TF_FR26, offsetof(struct trapframe, fs[2]));
+ASSYM(TF_FR27, offsetof(struct trapframe, fs[3]));
+ASSYM(TF_FR28, offsetof(struct trapframe, fs[4]));
+ASSYM(TF_FR29, offsetof(struct trapframe, fs[5]));
+ASSYM(TF_FR30, offsetof(struct trapframe, fs[6]));
+ASSYM(TF_FR31, offsetof(struct trapframe, fs[7]));
+ASSYM(TF_FCSR0, offsetof(struct trapframe, fcsr0));
+ASSYM(TF_FCC, offsetof(struct trapframe, fcc));
+ASSYM(TF_VREGS, offsetof(struct trapframe, tf_vregs));
+
+ASSYM(LOONGARCH_BOOTPARAMS_SIZE, sizeof(struct loongarch_bootparams));
+ASSYM(LOONGARCH_BOOTPARAMS_KERN_PGD,
+ offsetof(struct loongarch_bootparams, kern_pgd));
+ASSYM(LOONGARCH_BOOTPARAMS_KERN_PHYS,
+ offsetof(struct loongarch_bootparams, kern_phys));
+ASSYM(LOONGARCH_BOOTPARAMS_KERN_STACK, offsetof(struct loongarch_bootparams,
+ kern_stack));
+ASSYM(LOONGARCH_BOOTPARAMS_DTBP_VIRT,
+ offsetof(struct loongarch_bootparams, dtbp_virt));
+ASSYM(LOONGARCH_BOOTPARAMS_DTBP_PHYS,
+ offsetof(struct loongarch_bootparams, dtbp_phys));
+ASSYM(LOONGARCH_BOOTPARAMS_MODULEP,
+ offsetof(struct loongarch_bootparams, modulep));
diff --git a/sys/loongarch/loongarch/identcpu.c b/sys/loongarch/loongarch/identcpu.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/identcpu.c
@@ -0,0 +1,328 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ * Copyright (c) 2022 Mitchell Horne <mhorne@FreeBSD.org>
+ * Copyright (c) 2023 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Mitchell Horne
+ * <mhorne@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/ctype.h>
+#include <sys/kernel.h>
+#include <sys/pcpu.h>
+#include <sys/sysctl.h>
+
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#include <machine/elf.h>
+#include <machine/md_var.h>
+
+#ifdef FDT
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#endif
+
+const char machine[] = "loongarch";
+
+SYSCTL_CONST_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD | CTLFLAG_CAPRD,
+ machine, "Machine class");
+
+/* Hardware implementation info. These values may be empty. */
+register_t mvendorid; /* The CPU's JEDEC vendor ID */
+register_t marchid; /* The architecture ID */
+register_t mimpid; /* The implementation ID */
+
+/* Supervisor-mode extension support. */
+bool __read_frequently has_sstc;
+bool __read_frequently has_sscofpmf;
+
+struct cpu_desc cpu_desc[MAXCPU];
+
+/*
+ * Micro-architecture tables.
+ */
+struct marchid_entry {
+ register_t march_id;
+ const char *march_name;
+};
+
+#define MARCHID_LA32 CPUCFG1_ISGR32
+#define MARCHID_LA64 CPUCFG1_ISGR64
+#define MARCHID_END { -1ul, NULL }
+
+#define MVENDORID_UNIMPL 0
+#define MVENDORID_LOONGSON PRID_COMP_LOONGSON
+
+
+static const struct marchid_entry loongson_marchids[] = {
+ { MARCHID_LA32, "Loongson-32bit" },
+ { MARCHID_LA64, "Loongson-64bit" },
+ MARCHID_END,
+};
+
+/*
+ * Known CPU vendor/manufacturer table.
+ */
+static const struct {
+ register_t mvendor_id;
+ const char *mvendor_name;
+ const struct marchid_entry *marchid_table;
+} mvendor_ids[] = {
+ { MVENDORID_LOONGSON, "Loongson", loongson_marchids },
+ { MVENDORID_UNIMPL, "Unspecified", NULL },
+};
+
+static void cpu_probe_common(struct cpuinfo_loongarch *c)
+{
+ unsigned int config;
+ unsigned long asid_mask;
+
+ c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR |
+ LOONGARCH_CPU_TLB | LOONGARCH_CPU_VINT | LOONGARCH_CPU_WATCH;
+
+ c->hwcap = HWCAP_LOONGARCH_CPUCFG;
+
+ config = read_cpucfg(LOONGARCH_CPUCFG1);
+ if (config & CPUCFG1_UAL) {
+ c->options |= LOONGARCH_CPU_UAL;
+ c->hwcap |= HWCAP_LOONGARCH_UAL;
+ }
+ if (config & CPUCFG1_CRC32) {
+ c->options |= LOONGARCH_CPU_CRC32;
+ c->hwcap |= HWCAP_LOONGARCH_CRC32;
+ }
+
+ config = read_cpucfg(LOONGARCH_CPUCFG2);
+ if (config & CPUCFG2_LAM) {
+ c->options |= LOONGARCH_CPU_LAM;
+ c->hwcap |= HWCAP_LOONGARCH_LAM;
+ }
+ if (config & CPUCFG2_FP) {
+ c->options |= LOONGARCH_CPU_FPU;
+ c->hwcap |= HWCAP_LOONGARCH_FPU;
+ }
+#ifdef CONFIG_CPU_HAS_LSX
+ if (config & CPUCFG2_LSX) {
+ c->options |= LOONGARCH_CPU_LSX;
+ c->hwcap |= HWCAP_LOONGARCH_LSX;
+ }
+#endif
+#ifdef CONFIG_CPU_HAS_LASX
+ if (config & CPUCFG2_LASX) {
+ c->options |= LOONGARCH_CPU_LASX;
+ c->hwcap |= HWCAP_LOONGARCH_LASX;
+ }
+#endif
+ if (config & CPUCFG2_COMPLEX) {
+ c->options |= LOONGARCH_CPU_COMPLEX;
+ c->hwcap |= HWCAP_LOONGARCH_COMPLEX;
+ }
+ if (config & CPUCFG2_CRYPTO) {
+ c->options |= LOONGARCH_CPU_CRYPTO;
+ c->hwcap |= HWCAP_LOONGARCH_CRYPTO;
+ }
+ if (config & CPUCFG2_PTW) {
+ c->options |= LOONGARCH_CPU_PTW;
+ c->hwcap |= HWCAP_LOONGARCH_PTW;
+ }
+ if (config & CPUCFG2_LVZP) {
+ c->options |= LOONGARCH_CPU_LVZ;
+ c->hwcap |= HWCAP_LOONGARCH_LVZ;
+ }
+#ifdef CONFIG_CPU_HAS_LBT
+ if (config & CPUCFG2_X86BT) {
+ c->options |= LOONGARCH_CPU_LBT_X86;
+ c->hwcap |= HWCAP_LOONGARCH_LBT_X86;
+ }
+ if (config & CPUCFG2_ARMBT) {
+ c->options |= LOONGARCH_CPU_LBT_ARM;
+ c->hwcap |= HWCAP_LOONGARCH_LBT_ARM;
+ }
+ if (config & CPUCFG2_MIPSBT) {
+ c->options |= LOONGARCH_CPU_LBT_MIPS;
+ c->hwcap |= HWCAP_LOONGARCH_LBT_MIPS;
+ }
+#endif
+
+ config = read_cpucfg(LOONGARCH_CPUCFG6);
+ if (config & CPUCFG6_PMP)
+ c->options |= LOONGARCH_CPU_PMP;
+
+ config = iocsr_read32(LOONGARCH_IOCSR_FEATURES);
+ if (config & IOCSRF_CSRIPI)
+ c->options |= LOONGARCH_CPU_CSRIPI;
+ if (config & IOCSRF_EXTIOI)
+ c->options |= LOONGARCH_CPU_EXTIOI;
+ if (config & IOCSRF_FREQSCALE)
+ c->options |= LOONGARCH_CPU_SCALEFREQ;
+ if (config & IOCSRF_FLATMODE)
+ c->options |= LOONGARCH_CPU_FLATMODE;
+ if (config & IOCSRF_EIODECODE)
+ c->options |= LOONGARCH_CPU_EIODECODE;
+ if (config & IOCSRF_VM)
+ c->options |= LOONGARCH_CPU_HYPERVISOR;
+
+ config = csr_read32(LOONGARCH_CSR_ASID);
+ config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT;
+ asid_mask = _GENMASK(config - 1, 0);
+ c->asid_mask = asid_mask;
+
+ config = read_csr_prcfg1();
+ c->ksave_mask = _GENMASK((config & CSR_CONF1_KSNUM) - 1, 0);
+ c->ksave_mask &= ~(EXC_KSAVE_MASK | PERCPU_KSAVE_MASK);
+
+ config = read_csr_prcfg3();
+ switch (config & CSR_CONF3_TLBTYPE) {
+ case 0:
+ c->tlbsizemtlb = 0;
+ c->tlbsizestlbsets = 0;
+ c->tlbsizestlbways = 0;
+ c->tlbsize = 0;
+ break;
+ case 1:
+ c->tlbsizemtlb =
+ ((config & CSR_CONF3_MTLBSIZE) >>
+ CSR_CONF3_MTLBSIZE_SHIFT) + 1;
+ c->tlbsizestlbsets = 0;
+ c->tlbsizestlbways = 0;
+ c->tlbsize = c->tlbsizemtlb +
+ c->tlbsizestlbsets * c->tlbsizestlbways;
+ break;
+ case 2:
+ c->tlbsizemtlb =
+ ((config & CSR_CONF3_MTLBSIZE) >>
+ CSR_CONF3_MTLBSIZE_SHIFT) + 1;
+ c->tlbsizestlbsets = 1 <<
+ ((config & CSR_CONF3_STLBIDX) >>
+ CSR_CONF3_STLBIDX_SHIFT);
+ c->tlbsizestlbways =
+ ((config & CSR_CONF3_STLBWAYS) >>
+ CSR_CONF3_STLBWAYS_SHIFT) + 1;
+ c->tlbsize = c->tlbsizemtlb +
+ c->tlbsizestlbsets * c->tlbsizestlbways;
+ break;
+ default:
+ break;
+ }
+}
+
+static void
+identify_cpu_features(u_int cpu, struct cpu_desc *desc)
+{
+ cpu_probe_common(&desc->cpuinfo);
+}
+
+/*
+ * Update kernel/user global state based on the feature parsing results, stored
+ * in desc.
+ *
+ * We keep only the subset of values common to all CPUs.
+ */
+static void
+update_global_capabilities(u_int cpu, struct cpu_desc *desc)
+{
+#define UPDATE_CAP(t, v) \
+ do { \
+ if (cpu == 0) { \
+ (t) = (v); \
+ } else { \
+ (t) &= (v); \
+ } \
+ } while (0)
+
+ /* Update the capabilities exposed to userspace via AT_HWCAP. */
+ UPDATE_CAP(elf_hwcap, (u_long)desc->cpuinfo.hwcap);
+#undef UPDATE_CAP
+}
+
+static void
+identify_cpu_ids(struct cpu_desc *desc)
+{
+ const struct marchid_entry *table = NULL;
+ int i;
+
+ desc->cpu_mvendor_name = "Unknown";
+ desc->cpu_march_name = "Unknown";
+
+ mvendorid = desc->cpuinfo.processor_id & PRID_COMP_MASK;
+ marchid = desc->cpuinfo.arch & LOONGARCH_ARCH_MASK;
+
+ /*
+ * Search for a recognized vendor, and possibly obtain the secondary
+ * table for marchid lookup.
+ */
+ for (i = 0; i < nitems(mvendor_ids); i++) {
+ if (mvendorid == mvendor_ids[i].mvendor_id) {
+ desc->cpu_mvendor_name = mvendor_ids[i].mvendor_name;
+ table = mvendor_ids[i].marchid_table;
+ break;
+ }
+ }
+
+ for (i = 0; table[i].march_name != NULL; i++) {
+ if (marchid == table[i].march_id) {
+ desc->cpu_march_name = table[i].march_name;
+ break;
+ }
+ }
+}
+
+void
+identify_cpu(u_int cpu)
+{
+ struct cpu_desc *desc = &cpu_desc[cpu];
+
+ desc->cpuinfo.processor_id = read_cpucfg(LOONGARCH_CPUCFG0);
+ desc->cpuinfo.arch = read_cpucfg(LOONGARCH_CPUCFG1);
+
+ identify_cpu_ids(desc);
+ identify_cpu_features(cpu, desc);
+
+ update_global_capabilities(cpu, desc);
+}
+
+void
+printcpuinfo(u_int cpu)
+{
+ struct cpu_desc *desc;
+
+ desc = &cpu_desc[cpu];
+
+ /* Always print summary line. */
+ printf("CPU %-3u: Vendor=%s Core=%s \n", cpu,
+ desc->cpu_mvendor_name, desc->cpu_march_name);
+
+ /* These values are global. */
+ if (cpu == 0)
+ printf(" marchid=%#lx\n", marchid);
+}
diff --git a/sys/loongarch/loongarch/intc.h b/sys/loongarch/loongarch/intc.h
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/intc.h
@@ -0,0 +1,23 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * Local header for the CPU interrupt controller (cpuic) and
+ * intr_machdep shared data.
+ */
+
+#ifndef _LOONGARCH_INTC_H_
+#define _LOONGARCH_INTC_H_
+
+#include <sys/intr.h>
+#include <machine/intr.h>
+
+struct intc_irqsrc {
+ struct intr_irqsrc isrc;
+ u_int irq;
+};
+
+/* isrcs moved to cpuic_softc — no global array */
+
+#endif /* !_LOONGARCH_INTC_H_ */
diff --git a/sys/loongarch/loongarch/intr_machdep.c b/sys/loongarch/loongarch/intr_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/intr_machdep.c
@@ -0,0 +1,126 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * This file contains LoongArch IPI/mailbox hardware access primitives
+ * and legacy helpers needed during the transition to the full INTRNG
+ * PIC framework. The interrupt controllers live in cpuic.c (root)
+ * and eioic.c (external).
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/cpuset.h>
+#include <sys/interrupt.h>
+#include <sys/smp.h>
+
+#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#include <machine/frame.h>
+#include <machine/intr.h>
+
+#include "intc.h"
+
+#ifdef SMP
+#include <machine/smp.h>
+
+void
+csr_mail_send(uint64_t data, int cpu, int mailbox)
+{
+ uint64_t val;
+
+ val = IOCSR_MBUF_SEND_BLOCKING;
+ val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox)
+ << IOCSR_MBUF_SEND_BOX_SHIFT);
+ val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
+ val |= (data & IOCSR_MBUF_SEND_H32_MASK);
+ iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND);
+
+ val = IOCSR_MBUF_SEND_BLOCKING;
+ val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox)
+ << IOCSR_MBUF_SEND_BOX_SHIFT);
+ val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
+ val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT);
+ iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND);
+}
+
+uint32_t
+ipi_read_clear(int cpu)
+{
+ uint32_t action;
+
+ action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS);
+ iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR);
+ wbflush();
+ return (action);
+}
+
+void
+ipi_write_action(int cpu, uint32_t action)
+{
+ unsigned int irq;
+
+ while ((irq = ffs(action))) {
+ uint32_t val = IOCSR_IPI_SEND_BLOCKING;
+ val |= (irq - 1);
+ val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT);
+ iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND);
+ action &= ~(1U << (irq - 1));
+ }
+}
+
+static void
+ipi_send(struct pcpu *pc, int ipi)
+{
+ CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi);
+ atomic_set_32(&pc->pc_pending_ipis, ipi);
+ /*
+ * Ensure the pending_ipis store is globally visible before the
+ * IOCSR doorbell write. Without a fence, the target CPU may
+ * take the IPI but see a zero bitmap and return without processing.
+ */
+ fence();
+ ipi_write_action(pc->pc_cpuid, (uint32_t)ipi);
+ CTR1(KTR_SMP, "%s: sent", __func__);
+}
+
+void
+ipi_all_but_self(u_int ipi)
+{
+ cpuset_t other_cpus;
+
+ other_cpus = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &other_cpus);
+ CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
+ ipi_selected(other_cpus, ipi);
+}
+
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+ ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
+void
+ipi_selected(cpuset_t cpus, u_int ipi)
+{
+ struct pcpu *pc;
+
+ CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);
+ STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
+ if (CPU_ISSET(pc->pc_cpuid, &cpus)) {
+ CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n",
+ __func__, pc, ipi);
+ atomic_set_32(&pc->pc_pending_ipis, ipi);
+ ipi_send(pc, (uint32_t)ipi);
+ }
+ }
+}
+
+#endif /* SMP */
diff --git a/sys/loongarch/loongarch/locore.S b/sys/loongarch/loongarch/locore.S
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/locore.S
@@ -0,0 +1,502 @@
+/*-
+ * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "assym.inc"
+
+#include <sys/syscall.h>
+#include <machine/asm.h>
+#include <machine/param.h>
+#include <machine/intr.h>
+#include <machine/loongarchreg.h>
+#include <machine/pte.h>
+
+ .globl kernbase
+ .set kernbase, KERNBASE
+
+ .text
+
+
+.macro setup_page_table bsp
+ /* setup STLB page size */
+ li.w $t0, PAGE_SHIFT
+ csrwr $t0, LOONGARCH_CSR_STLBPGSIZE
+
+ /* set PWCL */
+ li.d $t0, PWCL
+ csrwr $t0, LOONGARCH_CSR_PWCTL0
+
+ /* set PWCH */
+ li.d $t0, PWCH
+ csrwr $t0, LOONGARCH_CSR_PWCTL1
+
+ /* bsp or ap */
+ li.d $t4, \bsp
+
+ /* setup PGD */
+ la.pcrel $t0, pagetable_l0
+ beqz $t4, $2f
+ la.pcrel $t1, pagetable_l1
+ li.d $t2, KERNBASE
+ srli.d $t2, $t2, L0_SHIFT
+ andi $t2, $t2, L0_ADDR_MASK
+ li.d $t3, PTE_SIZE
+ mul.d $t3, $t2, $t3 /* l0 slot */
+ stx.d $t1, $t0, $t3 /* l1 entry */
+2:
+ csrwr $t0, LOONGARCH_CSR_PGDH /* high half */
+
+ beqz $t4, $3f
+ la.pcrel $t0, pagetable_id_l0
+ la.pcrel $t1, pagetable_id_l1
+ csrrd $t2, LOONGARCH_CSR_KS0
+ srli.d $t2, $t2, L0_SHIFT
+ andi $t2, $t2, L0_ADDR_MASK
+ li.d $t3, PTE_SIZE
+ mul.d $t3, $t2, $t3 /* l0 slot */
+ stx.d $t1, $t0, $t3 /* l1 entry */
+ csrwr $t0, LOONGARCH_CSR_PGDL /* low half identity map*/
+3:
+.endm
+
+/*
+ * Main entry point. This routine is marked as the ELF entry, and is where
+ * loader(8) will enter the kernel. Arguments are as follows:
+ * - a0 = modulep
+ * - a1 = kernel load phys address
+ * - a2 = kernel end
+ *
+ * It is expected that only a single CPU will enter here.
+ */
+ .globl _start
+_start:
+ csrwr $a1, LOONGARCH_CSR_KS0
+ /*
+ * Zero a1 to indicate that we have no DTB pointer. It is already
+ * included in the loader(8) metadata.
+ */
+ move $a1, $zero
+
+ /* get kernel size */
+ li.d $t7, KERNBASE
+ sub.d $t7, $a2, $t7
+ li.d $t6, (L2_SIZE - 1) /* L2 Align */
+ add.d $t7, $t7, $t6
+ andn $t7, $t7, $t6
+ li.d $t6, LOCORE_EXTRA_MAP_LEN
+ add.d $t7, $t7, $t6
+
+ /*
+ * a0 - modulep or zero
+ * a1 - zero or dtbp
+ */
+
+ /* disable PG */
+ li.w $t0, 0xa8 # PLV=0, IE=0, PG=0, DA=1
+ csrwr $t0, LOONGARCH_CSR_CRMD
+
+pagetables:
+ /*
+ * map region starting at KERNBASE using L2 hugepage
+ */
+ li.d $t5, PMD_KERN_HUGE
+ csrwr $t5, LOONGARCH_CSR_KS1
+
+ la.pcrel $t0, pagetable_l1
+ la.pcrel $t1, pagetable_l2
+ li.d $t2, KERNBASE
+ csrrd $t3, LOONGARCH_CSR_KS0
+ move $t4, $t7
+ bl create_pagetables_to_l2
+
+ /*
+ * map region starting at 0 using L2 hugepage, Lower half
+ */
+ la.pcrel $t0, pagetable_id_l1
+ la.pcrel $t1, pagetable_id_l2
+ csrrd $t2, LOONGARCH_CSR_KS0 /* virtual load address, id mmap */
+ csrrd $t3, LOONGARCH_CSR_KS0 /* phys load address */
+ move $t4, $t7
+ bl create_pagetables_to_l2
+
+ /*
+ * Map the early devmap window so the console UART is reachable in
+ * initloongarch() before cninit()/pmap_bootstrap. create_pagetables_to_l2
+ * links pagetable_l2_devmap into the kernel pagetable_l1 at the devmap
+ * VA's L1 slot -- the devmap VA shares the kernel's L0 slot, so it must
+ * use the kernel L1 (not a separate one) -- and fills it with 2M huge
+ * pages with SUC (uncached) attributes mapping the Loongson config/UART
+ * region at 0x1fe00000.
+ */
+
+ li.d $t5, PMD_KERN_SUC_HUGE
+ csrwr $t5, LOONGARCH_CSR_KS1
+
+ la.pcrel $t0, pagetable_l1 /* kernel L1 (devmap shares its L0 slot) */
+ la.pcrel $t1, pagetable_l2_devmap /* L2 table to fill */
+ li.d $t2, (VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE) /* VA */
+ li.d $t3, 0x1fe00000 /* PA: Loongson UART config base */
+ li.d $t4, PMAP_MAPDEV_EARLY_SIZE /* size: 16M (8 x 2M pages) */
+ bl create_pagetables_to_l2
+
+ /* setup pagetable settings */
+ setup_page_table 1
+
+ /* setup tlbrentry */
+ la.pcrel $t0, handle_tlbrefill
+ csrwr $t0, LOONGARCH_CSR_TLBRENTRY
+
+ /* setup exception */
+ csrrd $t0, LOONGARCH_CSR_ECFG
+ li.d $t1, 0x70000
+ andn $t0, $t0, $t1
+ /* enable timer and ipi */
+ li.d $t1, 0x1
+ slli.d $t1, $t1, INT_TI
+ li.d $t2, 0x1
+ slli.d $t2, $t2, INT_IPI
+ or $t1, $t1, $t2
+ or $t0, $t0, $t1
+ csrwr $t0, LOONGARCH_CSR_ECFG
+
+ /* flush tlb (global) */
+ invtlb 0, $zero, $zero
+
+ /* record virtdone virt addr*/
+ la.pcrel $t1, Lvirtdone
+ ld.d $t5, $t1, 0
+
+ /* record kernel pgd */
+ la.pcrel $s2, pagetable_l0
+
+ /* Enable PG */
+ li.w $t0, 0xb0 # PLV=0, IE=0, PG=1
+ csrwr $t0, LOONGARCH_CSR_CRMD
+
+ /* jump to virtual address space */
+ jirl $zero, $t5, 0
+
+virtdone:
+
+ li.d $t0, 0x8000000000000001 /* CSR_DMW0_INIT: DMW0 VSEG=0x8000, PLV0 */
+ csrwr $t0, LOONGARCH_CSR_DMWIN0
+
+ la.pcrel $t0, Lbss
+ ld.d $t0, $t0, 0 # clear .bss
+ st.d $zero, $t0, 0
+ la.pcrel $t1, Lend
+ ld.d $t1, $t1, 0
+1:
+ addi.d $t0, $t0, 8
+ st.d $zero, $t0, 0
+ bne $t0, $t1, 1b
+
+ /* KSave4 used for percpu base, initialized as 0 */
+ csrwr $zero, PERCPU_BASE_KS
+ /* GPR21 used for percpu base (runtime), initialized as 0 */
+ move $r21, $zero
+
+ la.pcrel $tp, initstack
+
+ /* Initialize stack pointer */
+ la.pcrel $sp, initstack_end
+
+ /* Allocate space for thread0 PCB and bootparams*/
+ li.d $t0, (-(PCB_SIZE + LOONGARCH_BOOTPARAMS_SIZE) & ~STACKALIGNBYTES)
+ add.d $sp, $sp, $t0
+
+ /* fill bootparams */
+ la.pcrel $t0, initstack
+ st.d $t0, $sp, LOONGARCH_BOOTPARAMS_KERN_STACK
+ csrrd $s1, LOONGARCH_CSR_KS0
+ st.d $s1, $sp, LOONGARCH_BOOTPARAMS_KERN_PHYS
+ st.d $s2, $sp, LOONGARCH_BOOTPARAMS_KERN_PGD
+ st.d $a0, $sp, LOONGARCH_BOOTPARAMS_MODULEP
+
+ /* disable id map */
+ li.d $t0, PAGE_SIZE
+ sub.d $s2, $s2, $t0
+ csrwr $s2, LOONGARCH_CSR_PGDL /* disable low half identity map*/
+
+ move $a0, $sp
+ bl _C_LABEL(initloongarch) /* Off we go */
+ bl _C_LABEL(mi_startup)
+
+ /* We should never reach here, but if so just hang. */
+2:
+ idle 0
+ b 2b
+
+/*
+ * Create L1 pagetables
+ * @parms
+ * t0: l1 address
+ * t1: l2 address
+ * t2: virtual adress
+ */
+create_pagetables_to_l1:
+ /* level 1 */
+ srli.d $a2, $t1, PPNS
+
+ srli.d $a3, $t2, L1_SHIFT
+ andi $a3, $a3, Ln_ADDR_MASK /* L1 index */
+
+ slli.d $t5, $a2, PPNS /* L1 pte */
+
+ li.d $t6, PTE_SIZE
+ mul.d $a3, $a3, $t6 /* L1 slot */
+ stx.d $t5, $t0, $a3 /* save L1 pte */
+ jr $ra
+
+/*
+ * Create L1 and L2 pagetables, L2 is huge page
+ * @parms
+ * t0: l1 address
+ * t1: l2 address
+ * t2: virtual adress
+ * t3: physical address
+ * t4: map size
+ * KS1: memory attribute
+ */
+create_pagetables_to_l2:
+ /* level 1 */
+ srli.d $a2, $t1, PPNS
+
+ srli.d $a3, $t2, L1_SHIFT
+ andi $a3, $a3, Ln_ADDR_MASK /* L1 index */
+
+ slli.d $t5, $a2, PPNS /* L1 pte */
+
+ li.d $t6, PTE_SIZE
+ mul.d $a3, $a3, $t6 /* L1 slot */
+ stx.d $t5, $t0, $a3 /* save L1 pte */
+
+ /* level 2 huge page */
+ srli.d $t5, $t2, L2_SHIFT
+ andi $t5, $t5, Ln_ADDR_MASK /* L2 index */
+ srli.d $t6, $t4, L2_SHIFT /* L2 count */
+ add.d $t6, $t6, $t5 /* t6 => map end idex */
+
+ csrrd $a2, LOONGARCH_CSR_KS1 /* memory atrribute */
+ li.d $a3, PTE_SIZE
+ mul.d $a4, $t5, $a3 /* L2 slot */
+ srli.d $t4, $t3, L2_SHIFT
+1:
+ slli.d $a3, $t4, L2_SHIFT
+ or $a3, $a3, $a2 /* meke L2 pte */
+ stx.d $a3, $t1, $a4 /* save L2 pte */
+
+ addi.d $a4, $a4, PTE_SIZE /* update l2 slot */
+ addi.d $t4, $t4, 1 /* update physical address */
+ addi.d $t5, $t5, 1 /* update l2 index */
+ bltu $t5, $t6, 1b /* build all L2 entries */
+ jr $ra
+
+
+ .align 12
+handle_tlbrefill:
+ csrwr $t0, LOONGARCH_CSR_TLBRSAVE
+ csrrd $t0, LOONGARCH_CSR_PGD
+ lddir $t0, $t0, 3
+ lddir $t0, $t0, 2
+ lddir $t0, $t0, 1
+ ldpte $t0, 0
+ ldpte $t0, 1
+ tlbfill
+ csrrd $t0, LOONGARCH_CSR_TLBRSAVE
+ ertn
+
+ .align 4
+initstack:
+ .space (PAGE_SIZE * KSTACK_PAGES)
+initstack_end:
+
+ENTRY(sigcode)
+ move $a0, $sp
+ addi.d $a0, $a0, SF_UC
+
+1:
+ li.d $a7, SYS_sigreturn
+ syscall 0
+
+ /* sigreturn failed, exit */
+ li.d $a7, SYS_exit
+ syscall 0
+
+ b 1b
+END(sigcode)
+ /* This may be copied to the stack, keep it 16-byte aligned */
+ .align 3
+esigcode:
+
+ .data
+ .align 3
+ .global szsigcode
+szsigcode:
+ .quad esigcode - sigcode
+
+ .align 12
+pagetable_invalid:
+ .space PAGE_SIZE
+pagetable_l0:
+ .space PAGE_SIZE
+pagetable_l1:
+ .space PAGE_SIZE
+pagetable_l2:
+ .space PAGE_SIZE
+pagetable_l2_devmap:
+ .space PAGE_SIZE
+pagetable_id_l0:
+ .space PAGE_SIZE
+pagetable_id_l1:
+ .space PAGE_SIZE
+pagetable_id_l2:
+ .space PAGE_SIZE
+pagetable_dmap:
+ .space PAGE_SIZE
+
+ .align 3
+Lvirtdone:
+ .quad virtdone
+LSvirtdone:
+ .quad svirtdone
+
+Lbss:
+ .quad __bss_start
+Lend:
+ .quad _end
+
+ .globl l1_pt_va
+l1_pt_va:
+ .quad pagetable_l1
+ .globl l2_pt_va
+l2_pt_va:
+ .quad pagetable_l2 /* XXX: Keep page tables VA */
+
+ .globl dmap_pt_va /* for kernel dmap */
+dmap_pt_va:
+ .quad pagetable_dmap
+
+#ifndef SMP
+ENTRY(mpentry)
+1:
+ idle 0
+ j 1b
+END(mpentry)
+#else
+/*
+ * mpentry(unsigned long)
+ *
+ * Called by a core when it is being brought online.
+ */
+ENTRY(mpentry)
+
+ /* disable PG */
+ li.w $t0, 0xa8 # PLV=0, IE=0, PG=0, DA=1
+ csrwr $t0, LOONGARCH_CSR_CRMD
+
+ /* setup kernel pagetable */
+ setup_page_table 0
+
+ /* setup cached DMWIN0 */
+ pcaddi $t0, 0
+ li.d $t1, 0xFFFF000000000000
+ and $t0, $t0, $t1
+ ori $t0, $t0, (1 << 4 | 1)
+ csrwr $t0, LOONGARCH_CSR_DMWIN0
+
+ /* setup tlbrentry */
+ la.pcrel $t0, handle_tlbrefill
+ csrwr $t0, LOONGARCH_CSR_TLBRENTRY
+
+ /*
+ * Configure ECFG to enable timer and IPI interrupts.
+ * Global interrupts are disabled (CRMD.IE=0) so these will not
+ * fire until enabled in init_secondary().
+ */
+ csrrd $t0, LOONGARCH_CSR_ECFG
+ li.d $t1, 0x70000
+ andn $t0, $t0, $t1
+ li.d $t1, 0x1
+ slli.d $t1, $t1, INT_TI
+ li.d $t2, 0x1
+ slli.d $t2, $t2, INT_IPI
+ or $t1, $t1, $t2
+ or $t0, $t0, $t1
+ csrwr $t0, LOONGARCH_CSR_ECFG
+
+ /* flush tlb (global) */
+ invtlb 0, $zero, $zero
+
+ /* record virtdone virt addr*/
+ la.pcrel $t1, LSvirtdone
+ ld.d $t5, $t1, 0
+
+ /* record l0 phy address */
+ la.pcrel $s1, pagetable_l0
+
+ /* Enable PG */
+ li.w $t0, 0xb0 # PLV=0, IE=0, PG=1
+ csrwr $t0, LOONGARCH_CSR_CRMD
+
+ /* jump to virtual address space */
+ jirl $zero, $t5, 0
+
+svirtdone:
+
+ /* remove DMW0 */
+ li.d $t0, 0
+ csrwr $t0, LOONGARCH_CSR_DMWIN0
+
+ /* for uart */
+ li.d $t0, 0x8000000000000001 /* CSR_DMW0_INIT: DMW0 VSEG=0x8000, PLV0 */
+ csrwr $t0, LOONGARCH_CSR_DMWIN0
+
+ /* get cpuid, pass to init_secondary */
+ csrrd $a0, LOONGARCH_CSR_CPUID
+ andi $a0, $a0, 0x3ff
+
+ /*
+ * Calculate the offset to __loongarch_boot_ap
+ * for the current core, cpuid is in a0.
+ */
+ li.w $t1, 4
+ mul.w $t1, $t1, $a0
+ /* Get the pointer */
+ la.pcrel $t0, __loongarch_boot_ap
+ add.d $t0, $t0, $t1
+
+1:
+ /* Wait the kernel to be ready */
+ ld.w $t1, $t0, 0
+ beqz $t1, $1b
+
+ /* Setup stack pointer */
+ la.pcrel $t0, bootstack
+ ld.d $sp, $t0, 0
+
+ bl init_secondary
+END(mpentry)
+#endif
diff --git a/sys/loongarch/loongarch/loongarch_syscon.c b/sys/loongarch/loongarch/loongarch_syscon.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/loongarch_syscon.c
@@ -0,0 +1,80 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2018 Kyle Evans <kevans@FreeBSD.org>
+ * Copyright (c) 2020 Jessica Clarke <jrtc27@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * LoongArch syscon driver. Used by QEMU's virt machine for power/reset control
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <machine/bus.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/syscon/syscon.h>
+#include <dev/syscon/syscon_generic.h>
+
+static struct ofw_compat_data compat_data[] = {
+ {"syscon", 1},
+ {NULL, 0}
+};
+
+static int
+loongarch_syscon_probe(device_t dev)
+{
+
+ if (!ofw_bus_status_okay(dev))
+ return (ENXIO);
+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
+ return (ENXIO);
+
+ device_set_desc(dev, "LoongArch syscon");
+ return (BUS_PROBE_DEFAULT);
+}
+
+static device_method_t loongarch_syscon_methods[] = {
+ DEVMETHOD(device_probe, loongarch_syscon_probe),
+
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_1(loongarch_syscon, loongarch_syscon_driver,
+ loongarch_syscon_methods, sizeof(struct syscon_generic_softc),
+ syscon_generic_driver);
+
+/* loongarch_syscon needs to attach prior to syscon_power */
+EARLY_DRIVER_MODULE(loongarch_syscon, simplebus, loongarch_syscon_driver, 0, 0,
+ BUS_PASS_SCHEDULER + BUS_PASS_ORDER_LAST);
+MODULE_VERSION(loongarch_syscon, 1);
diff --git a/sys/loongarch/loongarch/machdep.c b/sys/loongarch/loongarch/machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/machdep.c
@@ -0,0 +1,982 @@
+/*-
+ * Copyright (c) 2014 Andrew Turner
+ * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/boot.h>
+#include <sys/buf.h>
+#include <sys/bus.h>
+#include <sys/cons.h>
+#include <sys/cpu.h>
+#include <sys/devmap.h>
+#include <sys/efi.h>
+#include <sys/exec.h>
+#include <sys/imgact.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/limits.h>
+#include <sys/linker.h>
+#include <sys/msgbuf.h>
+#include <sys/pcpu.h>
+#include <sys/physmem.h>
+#include <sys/proc.h>
+#include <sys/ptrace.h>
+#include <sys/reboot.h>
+#include <sys/reg.h>
+#include <sys/rwlock.h>
+#include <sys/sched.h>
+#include <sys/signalvar.h>
+#include <sys/syscallsubr.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
+#include <sys/tslog.h>
+#include <sys/ucontext.h>
+#include <sys/uuid.h>
+#include <sys/vmmeter.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <vm/vm_phys.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_pager.h>
+
+#include <machine/cpu.h>
+#include <machine/fpe.h>
+#include <machine/intr.h>
+#include <machine/kdb.h>
+#include <machine/machdep.h>
+#include <machine/metadata.h>
+#include <machine/pcb.h>
+#include <machine/pte.h>
+#include <machine/loongarchreg.h>
+#include <machine/trap.h>
+#include <machine/watch.h>
+#include <machine/vmparam.h>
+
+#ifdef DDB
+#include <ddb/ddb.h>
+#endif
+
+#ifdef FDT
+#include <contrib/libfdt/libfdt.h>
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#endif
+
+struct pcpu __pcpu[MAXCPU];
+
+static struct trapframe proc0_tf;
+
+int early_boot = 1;
+int cold = 1;
+
+#define DTB_SIZE_MAX (1024 * 1024)
+
+struct kva_md_info kmi;
+
+int64_t dcache_line_size; /* The minimum D cache line size */
+int64_t icache_line_size; /* The minimum I cache line size */
+int64_t idcache_line_size; /* The minimum cache line size */
+
+#define BOOT_CPU_INVALID 0xffffffff
+uint32_t boot_cpu = BOOT_CPU_INVALID; /* The cpuid we booted on. */
+
+/*
+ * Physical address of the EFI System Table. Stashed from the metadata hints
+ * passed into the kernel and used by the EFI code to call runtime services.
+ */
+vm_paddr_t efi_systbl_phys;
+static struct efi_map_header *efihdr;
+
+extern int *end;
+extern void cpu_exception_handler(void);
+
+static char static_kenv[PAGE_SIZE];
+
+static void
+cpu_startup(void *dummy)
+{
+
+ printcpuinfo(0);
+
+ printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem),
+ ptoa((uintmax_t)realmem) / (1024 * 1024));
+
+ /*
+ * Display any holes after the first chunk of extended memory.
+ */
+ if (bootverbose) {
+ int indx;
+
+ printf("Physical memory chunk(s):\n");
+ for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
+ vm_paddr_t size;
+
+ size = phys_avail[indx + 1] - phys_avail[indx];
+ printf(
+ "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n",
+ (uintmax_t)phys_avail[indx],
+ (uintmax_t)phys_avail[indx + 1] - 1,
+ (uintmax_t)size, (uintmax_t)size / PAGE_SIZE);
+ }
+ }
+
+ vm_ksubmap_init(&kmi);
+
+ printf("avail memory = %ju (%ju MB)\n",
+ ptoa((uintmax_t)vm_free_count()),
+ ptoa((uintmax_t)vm_free_count()) / (1024 * 1024));
+ if (bootverbose)
+ devmap_print_table();
+
+ bufinit();
+ vm_pager_bufferinit();
+}
+
+SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
+
+int
+cpu_idle_wakeup(int cpu)
+{
+ return (0);
+}
+
+void
+cpu_idle(int busy)
+{
+
+ spinlock_enter();
+ if (!busy)
+ cpu_idleclock();
+ if (!sched_runnable()) {
+ /*
+ * idle 0 only wakes on external interrupts when CRMD.IE=1,
+ * so interrupts must be enabled for the wait. Crucially,
+ * the wait must run OUTSIDE the spinlock critical section:
+ * spinlock_enter() only disables interrupts when the spinlock
+ * count is zero, so an interrupt taken while the count is
+ * non-zero would run with interrupts enabled and a nested
+ * interrupt could recurse on thread_lock (the
+ * "Assertion v != tid" panic in kern_mutex.c). Drop the
+ * section for the wait and re-acquire it afterwards.
+ */
+ spinlock_exit();
+ mb();
+ intr_enable();
+ __asm __volatile("idle 0");
+ spinlock_enter();
+ }
+ if (!busy)
+ cpu_activeclock();
+ spinlock_exit();
+}
+
+/*
+ * Flush the D-cache for non-DMA I/O so that the I-cache can
+ * be made coherent later.
+ */
+void
+cpu_flush_dcache(void *ptr, size_t len)
+{
+
+ /* TBD */
+}
+
+/* Get current clock frequency for the given CPU ID. */
+int
+cpu_est_clockrate(int cpu_id, uint64_t *rate)
+{
+#ifdef FDT
+ phandle_t cpus, child;
+ pcell_t prop;
+#endif
+
+ if (rate == NULL)
+ return (EINVAL);
+
+#ifdef FDT
+ cpus = OF_finddevice("/cpus");
+ if (cpus != -1) {
+ for (child = OF_child(cpus); child != 0;
+ child = OF_peer(child)) {
+ if (OF_getprop(child, "clock-frequency", &prop,
+ sizeof(prop)) > 0) {
+ *rate = prop;
+ return (0);
+ }
+ }
+ }
+#endif
+
+ /* Default to 100MHz (QEMU virt machine). */
+ *rate = 100 * 1000 * 1000;
+ return (0);
+}
+
+void
+cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
+{
+}
+
+void
+spinlock_enter(void)
+{
+ struct thread *td;
+ register_t reg;
+
+ td = curthread;
+ if (td->td_md.md_spinlock_count == 0) {
+ reg = intr_disable();
+ td->td_md.md_spinlock_count = 1;
+ td->td_md.md_saved_crmd_ie = reg;
+ critical_enter();
+ } else
+ td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+ struct thread *td;
+ register_t crmd_ie;
+
+ td = curthread;
+ crmd_ie = td->td_md.md_saved_crmd_ie;
+ td->td_md.md_spinlock_count--;
+ if (td->td_md.md_spinlock_count == 0) {
+ critical_exit();
+ intr_restore(crmd_ie);
+ }
+}
+
+/*
+ * Construct a PCB from a trapframe. This is called from kdb_trap() where
+ * we want to start a backtrace from the function that caused us to enter
+ * the debugger. We have the context in the trapframe, but base the trace
+ * on the PCB. The PCB doesn't have to be perfect, as long as it contains
+ * enough for a backtrace.
+ */
+void
+makectx(struct trapframe *tf, struct pcb *pcb)
+{
+
+ pcb->r0 = tf->r0;
+ pcb->ra = tf->tf_ra;
+ pcb->tp = tf->tf_tp;
+ pcb->sp = tf->tf_sp;
+ memcpy(pcb->a, tf->tf_a, sizeof(pcb->a));
+ memcpy(pcb->t, tf->tf_t, sizeof(pcb->t));
+ pcb->r21 = tf->r21;
+ pcb->fp = tf->tf_fp;
+ memcpy(pcb->s, tf->tf_s, sizeof(pcb->s));
+
+ pcb->pcb_a0 = tf->tf_a0;
+ pcb->pcb_sp = tf->tf_sp;
+
+}
+
+static void
+init_proc0(vm_offset_t kstack)
+{
+ struct pcpu *pcpup;
+
+ pcpup = &__pcpu[0];
+
+ proc_linkup0(&proc0, &thread0);
+ thread0.td_kstack = (char *)kstack;
+ thread0.td_kstack_pages = kstack_pages;
+ thread0.td_pcb = (struct pcb *)(thread0.td_kstack +
+ thread0.td_kstack_pages * PAGE_SIZE) - 1;
+ thread0.td_pcb->pcb_fpflags = 0;
+ thread0.td_frame = &proc0_tf;
+ pcpup->pc_curpcb = thread0.td_pcb;
+}
+
+typedef void (*efi_map_entry_cb)(struct efi_md *, void *argp);
+
+static void
+foreach_efi_map_entry(struct efi_map_header *efihdr, efi_map_entry_cb cb,
+ void *argp)
+{
+ struct efi_md *map, *p;
+ size_t efisz;
+ int ndesc, i;
+
+ /*
+ * Memory map data provided by UEFI via the GetMemoryMap
+ * Boot Services API.
+ */
+ efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
+ map = (struct efi_md *)((uint8_t *)efihdr + efisz);
+
+ if (efihdr->descriptor_size == 0)
+ return;
+ ndesc = efihdr->memory_size / efihdr->descriptor_size;
+
+ for (i = 0, p = map; i < ndesc; i++,
+ p = efi_next_descriptor(p, efihdr->descriptor_size)) {
+ cb(p, argp);
+ }
+}
+
+/*
+ * Handle the EFI memory map list.
+ *
+ * We will make two passes at this, the first (exclude == false) to populate
+ * physmem with valid physical memory ranges from recognized map entry types.
+ * In the second pass we will exclude memory ranges from physmem which must not
+ * be used for general allocations, either because they are used by runtime
+ * firmware or otherwise reserved.
+ *
+ * Adding the runtime-reserved memory ranges to physmem and excluding them
+ * later ensures that they are included in the DMAP, but excluded from
+ * phys_avail[].
+ *
+ * Entry types not explicitly listed here are ignored and not mapped.
+ */
+static void
+handle_efi_map_entry(struct efi_md *p, void *argp)
+{
+ bool exclude = *(bool *)argp;
+
+ switch (p->md_type) {
+ case EFI_MD_TYPE_RECLAIM:
+ /*
+ * The recomended location for ACPI tables. Map into the
+ * DMAP so we can access them from userspace via /dev/mem.
+ */
+ case EFI_MD_TYPE_RT_CODE:
+ /*
+ * Some UEFI implementations put the system table in the
+ * runtime code section. Include it in the DMAP, but will
+ * be excluded from phys_avail.
+ */
+ case EFI_MD_TYPE_RT_DATA:
+ /*
+ * Runtime data will be excluded after the DMAP
+ * region is created to stop it from being added
+ * to phys_avail.
+ */
+ if (exclude) {
+ physmem_exclude_region(p->md_phys,
+ p->md_pages * EFI_PAGE_SIZE, EXFLAG_NOALLOC);
+ break;
+ }
+ /* FALLTHROUGH */
+ case EFI_MD_TYPE_CODE:
+ case EFI_MD_TYPE_DATA:
+ case EFI_MD_TYPE_BS_CODE:
+ case EFI_MD_TYPE_BS_DATA:
+ case EFI_MD_TYPE_FREE:
+ /*
+ * We're allowed to use any entry with these types.
+ */
+ if (!exclude)
+ physmem_hardware_region(p->md_phys,
+ p->md_pages * EFI_PAGE_SIZE);
+ break;
+ default:
+ /* Other types shall not be handled by physmem. */
+ break;
+ }
+}
+
+static void
+add_efi_map_entries(struct efi_map_header *efihdr)
+{
+ bool exclude = false;
+ foreach_efi_map_entry(efihdr, handle_efi_map_entry, &exclude);
+}
+
+static void
+exclude_efi_map_entries(struct efi_map_header *efihdr)
+{
+ bool exclude = true;
+ foreach_efi_map_entry(efihdr, handle_efi_map_entry, &exclude);
+}
+
+/*
+ * Map the passed in VA in EFI space to a void * using the efi memory table to
+ * find the PA and return it in the DMAP, if it exists. We're used between the
+ * calls to pmap_bootstrap() and physmem_init_kernel_globals() to parse CFG
+ * tables We assume that either the entry you are mapping fits within its page,
+ * or if it spills to the next page, that's contiguous in PA and in the DMAP.
+ * All observed tables obey the first part of this precondition.
+ */
+struct early_map_data
+{
+ vm_offset_t va;
+ vm_offset_t pa;
+};
+
+static void
+efi_early_map_entry(struct efi_md *p, void *argp)
+{
+ struct early_map_data *emdp = argp;
+ vm_offset_t s, e;
+
+ if (emdp->pa != 0)
+ return;
+ if ((p->md_attr & EFI_MD_ATTR_RT) == 0)
+ return;
+ s = p->md_virt;
+ e = p->md_virt + p->md_pages * EFI_PAGE_SIZE;
+ if (emdp->va < s || emdp->va >= e)
+ return;
+ emdp->pa = p->md_phys + (emdp->va - p->md_virt);
+}
+
+static void *
+efi_early_map(vm_offset_t va)
+{
+ struct early_map_data emd = { .va = va };
+
+ foreach_efi_map_entry(efihdr, efi_early_map_entry, &emd);
+ if (emd.pa == 0)
+ return NULL;
+ return (void *)PHYS_TO_DMAP(emd.pa);
+}
+
+
+/*
+ * When booted via kboot, the prior kernel will pass in reserved memory areas in
+ * a EFI config table. We need to find that table and walk through it excluding
+ * the memory ranges in it. btw, this is called too early for the printf to do
+ * anything since msgbufp isn't initialized, let alone a console...
+ */
+static void
+exclude_efi_memreserve(vm_offset_t efi_systbl_phys)
+{
+ struct efi_systbl *systbl;
+ efi_guid_t efi_memreserve = LINUX_EFI_MEMRESERVE_TABLE;
+
+ systbl = (struct efi_systbl *)PHYS_TO_DMAP(efi_systbl_phys);
+ if (systbl == NULL) {
+ printf("can't map systbl\n");
+ return;
+ }
+ if (systbl->st_hdr.th_sig != EFI_SYSTBL_SIG) {
+ printf("Bad signature for systbl %#lx\n",
+ systbl->st_hdr.th_sig);
+ return;
+ }
+
+ /*
+ * We don't yet have the pmap system booted enough to create a pmap for
+ * the efi firmware's preferred address space from the GetMemoryMap()
+ * table. The st_cfgtbl is a VA in this space, so we need to do the
+ * mapping ourselves to a kernel VA with efi_early_map. We assume that
+ * the cfgtbl entries don't span a page. Other pointers are PAs, as
+ * noted below.
+ */
+ /* Failsafe: st_entries should == 0 in this case. */
+ if (systbl->st_cfgtbl == 0)
+ return;
+ for (int i = 0; i < systbl->st_entries; i++) {
+ struct efi_cfgtbl *cfgtbl;
+ struct linux_efi_memreserve *mr;
+
+ cfgtbl = efi_early_map(systbl->st_cfgtbl + i * sizeof(*cfgtbl));
+ if (cfgtbl == NULL)
+ panic("Can't map the config table entry %d\n", i);
+ if (memcmp(&cfgtbl->ct_guid, &efi_memreserve,
+ sizeof(efi_guid_t)) != 0)
+ continue;
+
+ /*
+ * cfgtbl points are either VA or PA, depending on the GUID of
+ * the table. memreserve GUID pointers are PA and not converted
+ * after a SetVirtualAddressMap(). The list's mr_next pointer
+ * is also a PA.
+ */
+ mr = (struct linux_efi_memreserve *)PHYS_TO_DMAP(
+ (vm_offset_t)cfgtbl->ct_data);
+ while (true) {
+ for (int j = 0; j < mr->mr_count; j++) {
+ struct linux_efi_memreserve_entry *mre;
+
+ mre = &mr->mr_entry[j];
+ physmem_exclude_region(mre->mre_base,
+ mre->mre_size,
+ EXFLAG_NODUMP | EXFLAG_NOALLOC);
+ }
+ if (mr->mr_next == 0)
+ break;
+ mr = (struct linux_efi_memreserve *)
+ PHYS_TO_DMAP(mr->mr_next);
+ };
+ }
+
+}
+
+
+/*
+ * Find the ACPI RSDP physical address from the EFI system table's
+ * configuration table (ACPI 2.0 GUID). Mirrors exclude_efi_memreserve():
+ * the system table and the table pointers are reached via PHYS_TO_DMAP;
+ * st_cfgtbl itself is a VA in the firmware's address space, so each entry
+ * is mapped with efi_early_map(). Used by AcpiOsGetRootPointer() before
+ * ACPICA is fully initialized, so the uart console can find the SPCR.
+ */
+/*
+ * Set by try_load_dtb() once OF_init() succeeds, i.e. the loader actually
+ * passed a device tree. loongarch_efi_acpi_rsdp() uses it to prefer FDT
+ * (QEMU) over ACPI at runtime, independent of the DEV_ACPI compile-time
+ * switch -- loongarch reads ACPI tables directly.
+ */
+static int loongarch_have_fdt;
+
+vm_paddr_t
+loongarch_efi_acpi_rsdp(void)
+{
+ struct efi_systbl *systbl;
+ /* ACPI 2.0 Table GUID (8868e871-e4f1-11d3-bc22-0080c73c8881). */
+ efi_guid_t acpi_guid =
+ {0x8868e871,0xe4f1,0x11d3,
+ {0xbc,0x22,0x00,0x80,0xc7,0x3c,0x88,0x81}};
+ int i;
+
+ /*
+ * Prefer FDT when a device tree was passed (QEMU): report "no ACPI" so
+ * nexus (ACPI interrupt-controller children), CPU enumeration, the IC
+ * probes and ACPICA all take the FDT path. Only without an FDT (real
+ * 3A6000, UEFI/ACPI) do we look up the RSDP in the EFI config tables.
+ */
+ if (loongarch_have_fdt)
+ return (0);
+
+ if (efi_systbl_phys == 0)
+ return (0);
+ systbl = (struct efi_systbl *)PHYS_TO_DMAP(efi_systbl_phys);
+ if (systbl == NULL ||
+ systbl->st_hdr.th_sig != EFI_SYSTBL_SIG ||
+ systbl->st_cfgtbl == 0)
+ return (0);
+
+ for (i = 0; i < systbl->st_entries; i++) {
+ struct efi_cfgtbl *cfgtbl;
+
+ cfgtbl = efi_early_map(systbl->st_cfgtbl +
+ (vm_offset_t)i * sizeof(*cfgtbl));
+ if (cfgtbl == NULL)
+ continue;
+ if (memcmp(&cfgtbl->ct_guid, &acpi_guid,
+ sizeof(efi_guid_t)) == 0)
+ return ((vm_paddr_t)cfgtbl->ct_data);
+ }
+
+ return (0);
+}
+
+#ifdef FDT
+static void
+try_load_dtb(caddr_t kmdp)
+{
+ vm_offset_t dtbp;
+
+ dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t);
+
+#if defined(FDT_DTB_STATIC)
+ /*
+ * In case the device tree blob was not retrieved (from metadata) try
+ * to use the statically embedded one.
+ */
+ if (dtbp == (vm_offset_t)NULL)
+ dtbp = (vm_offset_t)&fdt_static_dtb;
+#endif
+
+ if (dtbp == (vm_offset_t)NULL) {
+ printf("ERROR loading DTB\n");
+ return;
+ }
+
+ if (OF_install(OFW_FDT, 0) == FALSE)
+ panic("Cannot install FDT");
+
+ if (OF_init((void *)dtbp) != 0)
+ panic("OF_init failed with the found device tree");
+
+ /* A device tree was passed: prefer FDT over ACPI in this boot. */
+ loongarch_have_fdt = 1;
+
+ ged_init();
+}
+#endif
+
+static void
+cache_setup(void)
+{
+
+ /* TODO */
+
+ dcache_line_size = 0;
+ icache_line_size = 0;
+ idcache_line_size = 0;
+}
+
+/*
+ * Fake up a boot descriptor table.
+ */
+static void
+fake_preload_metadata(struct loongarch_bootparams *bp)
+{
+ static uint32_t fake_preload[48];
+ vm_offset_t lastaddr;
+ size_t fake_size, dtb_size;
+
+#define PRELOAD_PUSH_VALUE(type, value) do { \
+ *(type *)((char *)fake_preload + fake_size) = (value); \
+ fake_size += sizeof(type); \
+} while (0)
+
+#define PRELOAD_PUSH_STRING(str) do { \
+ uint32_t ssize; \
+ ssize = strlen(str) + 1; \
+ PRELOAD_PUSH_VALUE(uint32_t, ssize); \
+ strcpy(((char *)fake_preload + fake_size), str); \
+ fake_size += ssize; \
+ fake_size = roundup(fake_size, sizeof(u_long)); \
+} while (0)
+
+ fake_size = 0;
+ lastaddr = (vm_offset_t)&end;
+
+ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_NAME);
+ PRELOAD_PUSH_STRING("kernel");
+ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_TYPE);
+ PRELOAD_PUSH_STRING("elf kernel");
+
+ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_ADDR);
+ PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t));
+ PRELOAD_PUSH_VALUE(uint64_t, KERNBASE);
+
+ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_SIZE);
+ PRELOAD_PUSH_VALUE(uint32_t, sizeof(size_t));
+ PRELOAD_PUSH_VALUE(uint64_t, (size_t)((vm_offset_t)&end - KERNBASE));
+
+ /* Copy the DTB to KVA space. */
+ lastaddr = roundup(lastaddr, sizeof(int));
+ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_DTBP);
+ PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t));
+ PRELOAD_PUSH_VALUE(vm_offset_t, lastaddr);
+ dtb_size = fdt_totalsize(bp->dtbp_virt);
+ memmove((void *)lastaddr, (const void *)bp->dtbp_virt, dtb_size);
+ lastaddr = roundup(lastaddr + dtb_size, sizeof(int));
+
+ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_KERNEND);
+ PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t));
+ PRELOAD_PUSH_VALUE(vm_offset_t, lastaddr);
+
+ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_HOWTO);
+ PRELOAD_PUSH_VALUE(uint32_t, sizeof(int));
+ PRELOAD_PUSH_VALUE(int, RB_VERBOSE);
+
+ /* End marker */
+ PRELOAD_PUSH_VALUE(uint32_t, 0);
+ PRELOAD_PUSH_VALUE(uint32_t, 0);
+ preload_metadata = (caddr_t)fake_preload;
+
+ /* Check if bootloader clobbered part of the kernel with the DTB. */
+ KASSERT(bp->dtbp_phys + dtb_size <= bp->kern_phys ||
+ bp->dtbp_phys >= bp->kern_phys + (lastaddr - KERNBASE),
+ ("FDT (%lx-%lx) and kernel (%lx-%lx) overlap", bp->dtbp_phys,
+ bp->dtbp_phys + dtb_size, bp->kern_phys,
+ bp->kern_phys + (lastaddr - KERNBASE)));
+ KASSERT(fake_size < sizeof(fake_preload),
+ ("Too many fake_preload items"));
+
+ if (boothowto & RB_VERBOSE)
+ printf("FDT phys (%lx-%lx), kernel phys (%lx-%lx)\n",
+ bp->dtbp_phys, bp->dtbp_phys + dtb_size,
+ bp->kern_phys, bp->kern_phys + (lastaddr - KERNBASE));
+}
+
+/* Support for FDT configurations only. */
+CTASSERT(FDT);
+
+#ifdef FDT
+static void
+parse_fdt_bootargs(void)
+{
+ char bootargs[512];
+
+ bootargs[sizeof(bootargs) - 1] = '\0';
+ if (fdt_get_chosen_bootargs(bootargs, sizeof(bootargs) - 1) == 0) {
+ boothowto |= boot_parse_cmdline(bootargs);
+ }
+}
+#endif
+
+static vm_offset_t
+parse_metadata(void)
+{
+ caddr_t kmdp;
+ vm_offset_t lastaddr;
+#ifdef DDB
+ vm_offset_t ksym_start, ksym_end;
+#endif
+ char *kern_envp;
+
+ /* Find the kernel address */
+ kmdp = preload_search_by_type("elf kernel");
+ if (kmdp == NULL)
+ kmdp = preload_search_by_type("elf64 kernel");
+ KASSERT(kmdp != NULL, ("No preload metadata found!"));
+
+ /* Read the boot metadata */
+ boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
+ lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t);
+ kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *);
+ if (kern_envp != NULL)
+ init_static_kenv(kern_envp, 0);
+ else
+ init_static_kenv(static_kenv, sizeof(static_kenv));
+#ifdef DDB
+ ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
+ ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
+ db_fetch_ksymtab(ksym_start, ksym_end, 0);
+#endif
+#ifdef FDT
+ try_load_dtb(kmdp);
+ if (kern_envp == NULL)
+ parse_fdt_bootargs();
+#endif
+ return (lastaddr);
+}
+
+#ifdef FDT
+static void
+fdt_physmem_hardware_region_cb(const struct mem_region *mr, void *arg __unused)
+{
+
+ physmem_hardware_region(mr->mr_start, mr->mr_size);
+}
+
+static void
+fdt_physmem_exclude_region_cb(const struct mem_region *mr, void *arg __unused)
+{
+
+ physmem_exclude_region(mr->mr_start, mr->mr_size,
+ EXFLAG_NODUMP | EXFLAG_NOALLOC);
+}
+#endif
+
+/*
+ * Minimal polled UART output for debugging before cninit(). The console
+ * UART (Loongson LPC ns8250 at 0x1fe001e0) is reached through the DMW0
+ * direct-map window (set up in locore right after the jump to KERNBASE), so it
+ * stays reachable independent of the page-table devmap -- including after
+ * pmap_bootstrap tears the devmap down, before cninit() brings the console up.
+ */
+#define LOONGARCH_EARLY_UART_PHYS 0x1fe001e0
+#define LOONGARCH_EARLY_UART \
+ ((volatile uint8_t *)(CSR_DMW0_BASE | LOONGARCH_EARLY_UART_PHYS))
+
+static void
+loongarch_early_putc(int c)
+{
+ volatile uint8_t *uart = LOONGARCH_EARLY_UART;
+
+ if (c == '\n')
+ loongarch_early_putc('\r');
+ while ((uart[5] & 0x20) == 0) /* LSR: TX holding register empty */
+ ;
+ uart[0] = c; /* THR */
+}
+
+void
+loongarch_early_puts(const char *s)
+{
+ while (*s != '\0')
+ loongarch_early_putc(*s++);
+}
+
+void
+loongarch_early_put_hex(uint64_t v)
+{
+ static const char hex[] = "0123456789abcdef";
+ int i;
+
+ loongarch_early_putc('0');
+ loongarch_early_putc('x');
+ for (i = 60; i >= 0; i -= 4)
+ loongarch_early_putc(hex[(v >> i) & 0xf]);
+}
+
+void
+initloongarch(struct loongarch_bootparams *bp)
+{
+ struct pcpu *pcpup;
+ vm_offset_t lastaddr;
+ vm_size_t kernlen;
+ char *env;
+ struct efi_fb *efifb;
+ caddr_t kmdp;
+
+ printf("in %s\n", __func__);
+
+ TSRAW(&thread0, TS_ENTER, __func__, NULL);
+
+ /* Set the pcpu data, this is needed by pmap_bootstrap */
+ pcpup = &__pcpu[0];
+ pcpu_init(pcpup, 0, sizeof(struct pcpu));
+
+ /* Set the pcpu pointer */
+ __asm __volatile("move $r21, %0" :: "r"(pcpup));
+
+ csr_write64((uint64_t)pcpup, PERCPU_BASE_KS);
+
+ PCPU_SET(curthread, &thread0);
+
+ /* Parse the boot metadata. */
+ if (bp->modulep != 0) {
+ preload_metadata = (caddr_t)bp->modulep;
+ } else {
+ fake_preload_metadata(bp);
+ }
+ lastaddr = parse_metadata();
+
+ /* set boot cpu id to 0 */
+ boot_cpu = 0;
+
+ kmdp = preload_search_by_type("elf kernel");
+ if (kmdp == NULL)
+ kmdp = preload_search_by_type("elf64 kernel");
+ /* Initialize the global preload_kmdp for consumers (e.g. vt_efifb). */
+ preload_kmdp = kmdp;
+
+ efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t);
+
+ /* Load the physical memory ranges */
+ efihdr = (struct efi_map_header *)preload_search_info(kmdp,
+ MODINFO_METADATA | MODINFOMD_EFI_MAP);
+ if (efihdr != NULL) {
+ add_efi_map_entries(efihdr);
+ }
+#ifdef FDT
+ else {
+ /* Grab physical memory regions information from device tree. */
+ if (fdt_foreach_mem_region(
+ fdt_physmem_hardware_region_cb, NULL) != 0)
+ panic("Cannot get physical memory regions");
+ }
+
+ /*
+ * Exclude reserved memory specified by the device tree.
+ */
+ fdt_foreach_reserved_mem(fdt_physmem_exclude_region_cb, NULL);
+#endif
+
+ /* Exclude the EFI framebuffer from our view of physical memory. */
+ efifb = (struct efi_fb *)preload_search_info(kmdp,
+ MODINFO_METADATA | MODINFOMD_EFI_FB);
+ if (efifb != NULL)
+ physmem_exclude_region(efifb->fb_addr, efifb->fb_size,
+ EXFLAG_NOALLOC);
+ /*
+ * Identify CPU/ISA features.
+ */
+ identify_cpu(0);
+
+ /* setup EENTRY */
+ csr_write64((uint64_t)cpu_exception_handler, LOONGARCH_CSR_EENTRY);
+
+ /* init asid */
+ asid_init(0);
+
+ /* pt walker init */
+ pt_walker_init(0);
+
+ sched_instance_select();
+
+ link_elf_ireloc();
+
+ /* Do basic tuning, hz etc */
+ init_param1();
+
+ cache_setup();
+
+ /* Bootstrap enough of pmap to enter the kernel proper */
+ kernlen = (lastaddr - KERNBASE);
+ pmap_bootstrap(bp->kern_pgd, bp->kern_phys, kernlen);
+
+ printf("serial init ok!\n");
+ /* Exclude entries needed in the DMAP region, but not phys_avail */
+ if (efihdr != NULL)
+ exclude_efi_map_entries(efihdr);
+ /* Do the same for reserve entries in the EFI MEMRESERVE table */
+ if (efi_systbl_phys != 0)
+ exclude_efi_memreserve(efi_systbl_phys);
+
+ physmem_init_kernel_globals();
+
+ /* Establish static device mappings */
+ devmap_bootstrap();
+ printf("devmap init ok!\n");
+ cninit();
+
+ /*
+ * Dump the boot metadata. We have to wait for cninit() since console
+ * output is required. If it's grossly incorrect the kernel will never
+ * make it this far.
+ */
+ if (getenv_is_true("debug.dump_modinfo_at_boot"))
+ preload_dump();
+
+ init_proc0(bp->kern_stack);
+
+ msgbufinit(msgbufp, msgbufsize);
+ mutex_init();
+ init_param2(physmem);
+ la_watch_init();
+ kdb_init();
+#ifdef KDB
+ if ((boothowto & RB_KDB) != 0)
+ kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger");
+#endif
+
+ env = kern_getenv("kernelname");
+ if (env != NULL)
+ strlcpy(kernelname, env, sizeof(kernelname));
+
+ if (boothowto & RB_VERBOSE)
+ physmem_print_tables();
+
+ early_boot = 0;
+
+ printf("early boot ok!\n");
+
+ TSEXIT();
+}
diff --git a/sys/loongarch/loongarch/mem.c b/sys/loongarch/loongarch/mem.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/mem.c
@@ -0,0 +1,143 @@
+/*-
+ * Copyright (c) 2014 Andrew Turner
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/malloc.h>
+#include <sys/memrange.h>
+#include <sys/uio.h>
+
+#include <machine/memdev.h>
+#include <machine/vmparam.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+
+struct mem_range_softc mem_range_softc;
+
+int
+memrw(struct cdev *dev, struct uio *uio, int flags)
+{
+ ssize_t orig_resid;
+ vm_offset_t off, v;
+ struct iovec *iov;
+ struct vm_page m;
+ vm_page_t marr;
+ u_int cnt;
+ int error;
+
+ error = 0;
+ orig_resid = uio->uio_resid;
+ while (uio->uio_resid > 0 && error == 0) {
+ iov = uio->uio_iov;
+ if (iov->iov_len == 0) {
+ uio->uio_iov++;
+ uio->uio_iovcnt--;
+ if (uio->uio_iovcnt < 0)
+ panic("memrw");
+ continue;
+ }
+
+ v = uio->uio_offset;
+ off = v & PAGE_MASK;
+ cnt = ulmin(iov->iov_len, PAGE_SIZE - (u_int)off);
+ if (cnt == 0)
+ continue;
+
+ switch(dev2unit(dev)) {
+ case CDEV_MINOR_KMEM:
+ /* If the address is in the DMAP just copy it */
+ if (VIRT_IN_DMAP(v)) {
+ error = uiomove((void *)v, cnt, uio);
+ break;
+ }
+
+ if (!kernacc((void *)v, cnt, uio->uio_rw == UIO_READ ?
+ VM_PROT_READ : VM_PROT_WRITE)) {
+ error = EFAULT;
+ break;
+ }
+
+ /* Get the physical address to read */
+ v = pmap_extract(kernel_pmap, v);
+ if (v == 0) {
+ error = EFAULT;
+ break;
+ }
+
+ /* FALLTHROUGH */
+ case CDEV_MINOR_MEM:
+ /* If within the DMAP use this to copy from */
+ if (PHYS_IN_DMAP(v)) {
+ v = (vm_offset_t)PHYS_TO_DMAP(v);
+ error = uiomove((void *)v, cnt, uio);
+ break;
+ }
+
+ /* Have uiomove_fromphys handle the data */
+ m.phys_addr = trunc_page(v);
+ marr = &m;
+ uiomove_fromphys(&marr, off, cnt, uio);
+ break;
+ }
+ }
+
+ /*
+ * Don't return error if any byte was written. Read and write
+ * can return error only if no i/o was performed.
+ */
+ if (uio->uio_resid != orig_resid)
+ error = 0;
+
+ return (error);
+}
+
+/*
+ * Allow user processes to MMAP some memory sections
+ * instead of going through read/write.
+ */
+int
+memmmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr,
+ int prot __unused, vm_memattr_t *memattr __unused)
+{
+ if (dev2unit(dev) == CDEV_MINOR_MEM) {
+ *paddr = offset;
+ return (0);
+ }
+ return (-1);
+}
+
+int
+memioctl_md(struct cdev *dev __unused, u_long cmd __unused,
+ caddr_t data __unused, int flags __unused, struct thread *td __unused)
+{
+ return (ENOTTY);
+}
diff --git a/sys/loongarch/loongarch/minidump_machdep.c b/sys/loongarch/loongarch/minidump_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/minidump_machdep.c
@@ -0,0 +1,370 @@
+/*-
+ * Copyright (c) 2006 Peter Wemm
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ * Copyright (c) 2019 Mitchell Horne
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include "opt_watchdog.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/cons.h>
+#include <sys/kernel.h>
+#include <sys/kerneldump.h>
+#include <sys/msgbuf.h>
+#include <sys/watchdog.h>
+#include <sys/vmmeter.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_page.h>
+#include <vm/vm_phys.h>
+#include <vm/vm_dumpset.h>
+#include <vm/pmap.h>
+
+#include <machine/atomic.h>
+#include <machine/elf.h>
+#include <machine/md_var.h>
+#include <machine/minidump.h>
+
+CTASSERT(sizeof(struct kerneldumpheader) == 512);
+
+static struct kerneldumpheader kdh;
+
+/* Handle chunked writes. */
+static size_t fragsz;
+static void *dump_va;
+static size_t dumpsize;
+
+static uint64_t tmpbuffer[PAGE_SIZE / sizeof(uint64_t)];
+
+static int
+blk_flush(struct dumperinfo *di)
+{
+ int error;
+
+ if (fragsz == 0)
+ return (0);
+
+ error = dump_append(di, dump_va, fragsz);
+ fragsz = 0;
+ return (error);
+}
+
+/*
+ * Write a block of data to the dump file.
+ *
+ * Caller can provide data through a pointer or by specifying its
+ * physical address.
+ *
+ * XXX writes using pa should be no larger than PAGE_SIZE.
+ */
+static int
+blk_write(struct dumperinfo *di, char *ptr, vm_paddr_t pa, size_t sz)
+{
+ size_t len;
+ int error, c;
+ u_int maxdumpsz;
+
+ maxdumpsz = min(di->maxiosize, MAXDUMPPGS * PAGE_SIZE);
+ if (maxdumpsz == 0) /* seatbelt */
+ maxdumpsz = PAGE_SIZE;
+ error = 0;
+ if ((sz % PAGE_SIZE) != 0) {
+ printf("size not page aligned\n");
+ return (EINVAL);
+ }
+ if (ptr != NULL && pa != 0) {
+ printf("cant have both va and pa!\n");
+ return (EINVAL);
+ }
+ if ((((uintptr_t)pa) % PAGE_SIZE) != 0) {
+ printf("address not page aligned %#lx\n", (uintptr_t)pa);
+ return (EINVAL);
+ }
+ if (ptr != NULL) {
+ /*
+ * If we're doing a virtual dump, flush any
+ * pre-existing pa pages.
+ */
+ error = blk_flush(di);
+ if (error != 0)
+ return (error);
+ }
+ while (sz) {
+ len = maxdumpsz - fragsz;
+ if (len > sz)
+ len = sz;
+
+ dumpsys_pb_progress(len);
+ wdog_kern_pat(WD_LASTVAL);
+
+ if (ptr) {
+ error = dump_append(di, ptr, len);
+ if (error != 0)
+ return (error);
+ ptr += len;
+ sz -= len;
+ } else {
+ dump_va = (void *)PHYS_TO_DMAP(pa);
+ fragsz += len;
+ pa += len;
+ sz -= len;
+ error = blk_flush(di);
+ if (error != 0)
+ return (error);
+ }
+
+ /* Check for user abort */
+ c = cncheckc();
+ if (c == 0x03)
+ return (ECANCELED);
+ if (c != -1)
+ printf(" (CTRL-C to abort) ");
+ }
+
+ return (0);
+}
+
+int
+cpu_minidumpsys(struct dumperinfo *di, const struct minidumpstate *state)
+{
+ pd_entry_t *l1, *l2, l2e;
+ pt_entry_t *l3, l3e;
+ struct minidumphdr mdhdr;
+ struct msgbuf *mbp;
+ uint32_t pmapsize;
+ vm_offset_t va, kva_max;
+ vm_paddr_t pa;
+ int error;
+ int i;
+ int retry_count;
+
+ retry_count = 0;
+retry:
+ retry_count++;
+ error = 0;
+ pmapsize = 0;
+
+ /* Snapshot the KVA upper bound in case it grows. */
+ kva_max = kernel_vm_end;
+
+ /*
+ * Walk the kernel page table pages, setting the active entries in the
+ * dump bitmap.
+ *
+ * NB: for a live dump, we may be racing with updates to the page
+ * tables, so care must be taken to read each entry only once.
+ */
+ for (va = VM_MIN_KERNEL_ADDRESS; va < kva_max; va += L2_SIZE) {
+ pmapsize += PAGE_SIZE;
+ if (!pmap_get_tables(pmap_kernel(), va, &l1, &l2, &l3))
+ continue;
+
+ /* We should always be using the l2 table for kvm */
+ if (l2 == NULL)
+ continue;
+
+ /* l2 may be a superpage */
+ l2e = atomic_load_64(l2);
+ if (PTE_RWX(l2e)) {
+ pa = (l2e >> L2_SHIFT) << L2_SHIFT;
+ for (i = 0; i < Ln_ENTRIES; i++, pa += PAGE_SIZE) {
+ if (vm_phys_is_dumpable(pa))
+ vm_page_dump_add(state->dump_bitset,
+ pa);
+ }
+ } else {
+ for (i = 0; i < Ln_ENTRIES; i++) {
+ l3e = atomic_load_64(&l3[i]);
+ if ((l3e & PTE_V) == 0)
+ continue;
+ pa = (l3e >> L3_SHIFT) * PAGE_SIZE;
+ if (PHYS_IN_DMAP(pa) && vm_phys_is_dumpable(pa))
+ vm_page_dump_add(state->dump_bitset,
+ pa);
+ }
+ }
+ }
+
+ /* Calculate dump size */
+ mbp = state->msgbufp;
+ dumpsize = pmapsize;
+ dumpsize += round_page(mbp->msg_size);
+ dumpsize += round_page(sizeof(dump_avail));
+ dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages));
+ VM_PAGE_DUMP_FOREACH(state->dump_bitset, pa) {
+ /* Clear out undumpable pages now if needed */
+ if (PHYS_IN_DMAP(pa) && vm_phys_is_dumpable(pa))
+ dumpsize += PAGE_SIZE;
+ else
+ vm_page_dump_drop(state->dump_bitset, pa);
+ }
+ dumpsize += PAGE_SIZE;
+
+ dumpsys_pb_init(dumpsize);
+
+ /* Initialize mdhdr */
+ bzero(&mdhdr, sizeof(mdhdr));
+ strcpy(mdhdr.magic, MINIDUMP_MAGIC);
+ mdhdr.version = MINIDUMP_VERSION;
+ mdhdr.msgbufsize = mbp->msg_size;
+ mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages));
+ mdhdr.pmapsize = pmapsize;
+ mdhdr.kernbase = KERNBASE;
+ mdhdr.dmapphys = DMAP_MIN_PHYSADDR;
+ mdhdr.dmapbase = DMAP_MIN_ADDRESS;
+ mdhdr.dmapend = DMAP_MAX_ADDRESS;
+ mdhdr.dumpavailsize = round_page(sizeof(dump_avail));
+
+ dump_init_header(di, &kdh, KERNELDUMPMAGIC,
+ KERNELDUMP_LOONGARCH_VERSION, dumpsize);
+
+ error = dump_start(di, &kdh);
+ if (error != 0)
+ goto fail;
+
+ printf("Dumping %llu out of %ju MB:", (long long)dumpsize >> 20,
+ ptoa((uintmax_t)physmem) / 1048576);
+
+ /* Dump minidump header */
+ bzero(&tmpbuffer, sizeof(tmpbuffer));
+ bcopy(&mdhdr, &tmpbuffer, sizeof(mdhdr));
+ error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE);
+ if (error)
+ goto fail;
+
+ /* Dump msgbuf up front */
+ error = blk_write(di, mbp->msg_ptr, 0, round_page(mbp->msg_size));
+ if (error)
+ goto fail;
+
+ /* Dump dump_avail */
+ _Static_assert(sizeof(dump_avail) <= sizeof(tmpbuffer),
+ "Large dump_avail not handled");
+ bzero(tmpbuffer, sizeof(tmpbuffer));
+ memcpy(tmpbuffer, dump_avail, sizeof(dump_avail));
+ error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE);
+ if (error)
+ goto fail;
+
+ /* Dump bitmap */
+ error = blk_write(di, (char *)vm_page_dump, 0,
+ round_page(BITSET_SIZE(vm_page_dump_pages)));
+ if (error)
+ goto fail;
+
+ /* Dump kernel page directory pages */
+ bzero(&tmpbuffer, sizeof(tmpbuffer));
+ for (va = VM_MIN_KERNEL_ADDRESS; va < kva_max; va += L2_SIZE) {
+ if (!pmap_get_tables(pmap_kernel(), va, &l1, &l2, &l3)) {
+ /* We always write a page, even if it is zero */
+ error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE);
+ if (error)
+ goto fail;
+ /* Flush, in case we reuse tmpbuffer in the
+ * same block */
+ error = blk_flush(di);
+ if (error)
+ goto fail;
+ continue;
+ }
+
+ l2e = atomic_load_64(l2);
+ if (PTE_RWX(l2e)) {
+ /* Generate fake l3 entries based on the l2 superpage */
+ for (i = 0; i < Ln_ENTRIES; i++) {
+ tmpbuffer[i] = (l2e | (i << L2_SHIFT));
+ }
+ /* We always write a page, even if it is zero */
+ error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE);
+ if (error)
+ goto fail;
+ /* Flush, in case we reuse tmpbuffer in the
+ * same block */
+ error = blk_flush(di);
+ if (error)
+ goto fail;
+ bzero(&tmpbuffer, sizeof(tmpbuffer));
+ } else {
+ pa = (l2e >> L2_SHIFT) * PAGE_SIZE;
+
+ /*
+ * We always write a page, even if it is zero. If pa
+ * is malformed, write the zeroed tmpbuffer.
+ */
+ if (PHYS_IN_DMAP(pa) && vm_phys_is_dumpable(pa))
+ error = blk_write(di, NULL, pa, PAGE_SIZE);
+ else
+ error = blk_write(di, (char *)&tmpbuffer, 0,
+ PAGE_SIZE);
+ if (error)
+ goto fail;
+ }
+ }
+
+ /* Dump memory chunks */
+ /* XXX cluster it up and use blk_dump() */
+ VM_PAGE_DUMP_FOREACH(state->dump_bitset, pa) {
+ error = blk_write(di, 0, pa, PAGE_SIZE);
+ if (error)
+ goto fail;
+ }
+
+ error = blk_flush(di);
+ if (error)
+ goto fail;
+
+ error = dump_finish(di, &kdh);
+ if (error != 0)
+ goto fail;
+
+ printf("\nDump complete\n");
+ return (0);
+
+fail:
+ if (error < 0)
+ error = -error;
+
+ printf("\n");
+ if (error == ENOSPC) {
+ printf("Dump map grown while dumping. ");
+ if (retry_count < 5) {
+ printf("Retrying...\n");
+ goto retry;
+ }
+ printf("Dump failed.\n");
+ }
+ else if (error == ECANCELED)
+ printf("Dump aborted\n");
+ else if (error == E2BIG) {
+ printf("Dump failed. Partition too small (about %lluMB were "
+ "needed this time).\n", (long long)dumpsize >> 20);
+ } else
+ printf("** DUMP FAILED (ERROR %d) **\n", error);
+ return (error);
+}
diff --git a/sys/loongarch/loongarch/mp_machdep.c b/sys/loongarch/loongarch/mp_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/mp_machdep.c
@@ -0,0 +1,1052 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Portions of this software were developed by Andrew Turner under
+ * sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_kstack_pages.h"
+#include "opt_platform.h"
+#include "opt_acpi.h"
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/cpuset.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/kdb.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+
+#include <machine/intr.h>
+#include <machine/cpufunc.h>
+#include <machine/smp.h>
+#include <machine/tlb.h>
+#include <machine/vmparam.h>
+
+#ifdef FDT
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_cpu.h>
+#endif
+
+#define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE)
+
+uint32_t __loongarch_boot_ap[MAXCPU];
+
+static enum {
+ CPUS_UNKNOWN,
+#ifdef FDT
+ CPUS_FDT,
+#endif
+#ifdef DEV_ACPI
+ CPUS_ACPI,
+#endif
+} cpu_enum_method;
+
+static device_identify_t loongarch64_cpu_identify;
+static device_probe_t loongarch64_cpu_probe;
+static device_attach_t loongarch64_cpu_attach;
+
+static int ipi_handler(void *);
+
+extern uint32_t boot_cpu;
+extern cpuset_t all_cpus;
+extern void cpu_exception_handler(void);
+
+#ifdef INVARIANTS
+static uint32_t cpu_reg[MAXCPU][2];
+#endif
+static device_t cpu_list[MAXCPU];
+
+void mpentry(u_long cpuid);
+void init_secondary(uint32_t);
+
+static struct mtx ap_boot_mtx;
+
+/* Stacks for AP initialization, discarded once idle threads are started. */
+void *bootstack;
+static void *bootstacks[MAXCPU];
+
+/* Count of started APs, used to synchronize access to bootstack. */
+static volatile int aps_started;
+
+/* Set to 1 once we're ready to let the APs out of the pen. */
+static volatile int aps_ready;
+
+/*
+ * SMP TLB shootdown state. Each CPU owns a pool of shootdown descriptors
+ * (la_tlb_pool[cpu]) and two linked lists built from them: an SLIST free list
+ * (la_tlb_free[cpu]) and a pending list (la_tlb_q[cpu]). An initiator pops a
+ * descriptor, fills it, appends it to its pending list, and IPIs the targets;
+ * each target drains every other initiator's pending list. Because every
+ * in-flight shootdown from a CPU has its own descriptor, rapid fire-and-forget
+ * shootdowns (IE-off callers such as pmap_qenter/UMA) no longer overwrite one
+ * another before the targets process them -- the race that silently dropped
+ * invalidations and left stale TLBs on remote CPUs under multi-AP load.
+ *
+ * Per-descriptor cpusets record which targets must flush (targets) and which
+ * already have (acked); a descriptor returns to the free list once every
+ * target has acked it. On pool exhaustion the initiator escalates to a full
+ * flush (la_tlb_flushall_gen) rather than appending, so no invalidation lost.
+ */
+#define LA_TLB_POOL 128 /* shootdown descriptors per CPU */
+
+struct la_tlb_sd {
+ pmap_t pmap;
+ uint64_t addr1;
+ uint64_t addr2;
+ cpuset_t targets; /* CPUs that must flush this */
+ cpuset_t acked; /* CPUs that have flushed this */
+ volatile u_int remaining; /* atomic: targets not yet acked */
+ uint8_t op; /* LA_TLB_OP_* */
+ SLIST_ENTRY(la_tlb_sd) free_link; /* on la_tlb_free[cpu] */
+ TAILQ_ENTRY(la_tlb_sd) pend_link; /* on la_tlb_q[cpu] */
+};
+
+SLIST_HEAD(la_tlb_freelist, la_tlb_sd);
+TAILQ_HEAD(la_tlb_pendingq, la_tlb_sd);
+
+static struct la_tlb_sd la_tlb_pool[MAXCPU][LA_TLB_POOL];
+static struct la_tlb_freelist la_tlb_free[MAXCPU];
+static struct la_tlb_pendingq la_tlb_q[MAXCPU];
+static struct mtx la_tlb_mtx[MAXCPU]; /* guards free+pending lists */
+static uint32_t la_tlb_flushall_gen[MAXCPU]; /* bumped on pool exhaustion */
+static uint32_t la_tlb_flushall_done[MAXCPU][MAXCPU]; /* [target][initiator] */
+static cpuset_t la_tlb_pending; /* initiators with a non-empty la_tlb_q[] */
+
+/*
+ * Per-CPU stale-ASID tracking. When pmap_activate_sw drops an old pmap's
+ * ASID, the old ASID's TLB entries linger. Record it here; la_tlb_local_flush
+ * batch-flushes them (via invtlb op 4) before the target ASID's flush.
+ */
+#define LA_STALE_MAX 128
+struct la_stale {
+ uint32_t asids[LA_STALE_MAX];
+ int count;
+ bool overflow;
+};
+static struct la_stale la_stale[MAXCPU];
+
+void
+la_stale_add(uint32_t asid)
+{
+ int cpuid;
+
+ if (asid == 0)
+ return;
+ cpuid = PCPU_GET(cpuid);
+ if (la_stale[cpuid].count < LA_STALE_MAX)
+ la_stale[cpuid].asids[la_stale[cpuid].count++] = asid;
+ else
+ la_stale[cpuid].overflow = true;
+}
+
+/* Temporary variables for init_secondary() */
+void *dpcpu[MAXCPU - 1];
+
+static device_method_t loongarch64_cpu_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_identify, loongarch64_cpu_identify),
+ DEVMETHOD(device_probe, loongarch64_cpu_probe),
+ DEVMETHOD(device_attach, loongarch64_cpu_attach),
+
+ DEVMETHOD_END
+};
+
+static driver_t loongarch64_cpu_driver = {
+ "loongarch64_cpu",
+ loongarch64_cpu_methods,
+ 0
+};
+
+DRIVER_MODULE(loongarch64_cpu, cpu, loongarch64_cpu_driver, 0, 0);
+
+static void
+loongarch64_cpu_identify(driver_t *driver, device_t parent)
+{
+
+ if (device_find_child(parent, "loongarch64_cpu", -1) != NULL)
+ return;
+ if (BUS_ADD_CHILD(parent, 0, "loongarch64_cpu", -1) == NULL)
+ device_printf(parent, "add child failed\n");
+}
+
+static int
+loongarch64_cpu_probe(device_t dev)
+{
+ u_int cpuid;
+
+ cpuid = device_get_unit(dev);
+ if (cpuid >= MAXCPU || cpuid > mp_maxid)
+ return (EINVAL);
+
+ device_quiet(dev);
+ return (0);
+}
+
+static int
+loongarch64_cpu_attach(device_t dev)
+{
+ const uint32_t *reg;
+ size_t reg_size;
+ u_int cpuid;
+ int i;
+
+ cpuid = device_get_unit(dev);
+
+ if (cpuid >= MAXCPU || cpuid > mp_maxid)
+ return (EINVAL);
+ KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid));
+
+ reg = cpu_get_cpuid(dev, &reg_size);
+ if (reg == NULL)
+ return (EINVAL);
+
+ if (bootverbose) {
+ device_printf(dev, "register <");
+ for (i = 0; i < reg_size; i++)
+ printf("%s%x", (i == 0) ? "" : " ", reg[i]);
+ printf(">\n");
+ }
+
+ /* Set the device to start it later */
+ cpu_list[cpuid] = dev;
+
+ return (0);
+}
+
+static void
+release_aps(void *dummy __unused)
+{
+ int i, tcpu, tdesc;
+
+ if (mp_ncpus == 1)
+ return;
+
+ /* Initialize the per-CPU TLB shootdown pools before APs come online. */
+ for (tcpu = 0; tcpu < MAXCPU; tcpu++) {
+ SLIST_INIT(&la_tlb_free[tcpu]);
+ TAILQ_INIT(&la_tlb_q[tcpu]);
+ for (tdesc = 0; tdesc < LA_TLB_POOL; tdesc++)
+ SLIST_INSERT_HEAD(&la_tlb_free[tcpu],
+ &la_tlb_pool[tcpu][tdesc], free_link);
+ mtx_init(&la_tlb_mtx[tcpu], "la tlb shootdown",
+ NULL, MTX_SPIN);
+ }
+
+ /* Setup the IPI handler */
+ cpuic_setup_ipi(ipi_handler);
+
+ /* Enable all IPI vectors in the IOCSR IPI controller. */
+ iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
+
+ atomic_store_rel_int(&aps_ready, 1);
+
+ /*
+ * APs busy-wait on aps_ready in init_secondary(), so no IPI is needed
+ * to release them. (They can't take an IPI yet anyway: mpentry leaves
+ * CRMD.IE=0 and their IPI isn't armed until later in init_secondary.)
+ */
+ printf("Release APs\n");
+
+ for (i = 0; i < 2000; i++) {
+ if (atomic_load_acq_int(&smp_started))
+ return;
+ DELAY(1000);
+ }
+
+ printf("APs not started\n");
+}
+SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
+
+void
+init_secondary(uint32_t cpuid)
+{
+ struct pcpu *pcpup;
+
+ if (cpuid < boot_cpu)
+ cpuid += mp_maxid + 1;
+ cpuid -= boot_cpu;
+
+ /* Setup the pcpu pointer */
+ pcpup = &__pcpu[cpuid];
+ __asm __volatile("move $r21, %0" :: "r"(pcpup));
+
+ csr_write64((uint64_t)pcpup, PERCPU_BASE_KS);
+
+ /* cpu identify */
+ identify_cpu(cpuid);
+ /* Configure per-CPU CSRs: ASID and page table walker. */
+ asid_init(cpuid);
+ pt_walker_init(cpuid);
+
+ /* eentry */
+ csr_write64((uint64_t)cpu_exception_handler, LOONGARCH_CSR_EENTRY);
+
+ /* No printf can be called before while loop exit
+ pc_curthread is NULL !!!
+ */
+
+ /* Signal the BSP and spin until it has released all APs. Busy-wait */
+ atomic_add_int(&aps_started, 1);
+ while (!atomic_load_int(&aps_ready))
+ cpu_spinwait();
+
+ /* Clear the IPI. */
+ ipi_read_clear(cpuid);
+
+ /* Initialize curthread */
+ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
+ pcpup->pc_curthread = pcpup->pc_idlethread;
+ schedinit_ap();
+
+ /*
+ * Enable IOCSR IPI delivery; ECFG unmasking of the IRQ_IPI line is
+ * done by cpuic_init_secondary() below (it enables any IRQ with a
+ * registered handler, and cpuic_setup_ipi registered the filter).
+ */
+ iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
+
+ /* Initialize per-CPU EIOINTC routing */
+ eiointc_init_cpu();
+
+ /*
+ * Initialize per-CPU interrupt state for the root PIC.
+ * This re-enables any interrupts that were set up on the BSP.
+ */
+ intr_pic_init_secondary();
+
+#ifndef EARLY_AP_STARTUP
+ /* Start per-CPU event timers. */
+ cpu_initclocks_ap();
+#endif
+
+ /* Activate this cpu in the kernel pmap. */
+ CPU_SET_ATOMIC(cpuid, &kernel_pmap->pm_active);
+
+ /* Activate process 0's pmap. */
+ pmap_activate_boot(vmspace_pmap(proc0.p_vmspace));
+
+ mtx_lock_spin(&ap_boot_mtx);
+
+ atomic_add_rel_32(&smp_cpus, 1);
+
+ if (smp_cpus == mp_ncpus) {
+ /* enable IPI's, tlb shootdown, freezes etc */
+ atomic_store_rel_int(&smp_started, 1);
+ }
+
+ mtx_unlock_spin(&ap_boot_mtx);
+
+ if (bootverbose)
+ printf("Secondary CPU %u fully online\n", cpuid);
+
+ sched_ap_entry();
+
+ panic("scheduler returned us to init_secondary");
+ /* NOTREACHED */
+}
+
+static void
+smp_after_idle_runnable(void *arg __unused)
+{
+ int cpu;
+
+ if (mp_ncpus == 1)
+ return;
+
+ KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__));
+
+ /*
+ * Wait for all APs to handle an interrupt. After that, we know that
+ * the APs have entered the scheduler at least once, so the boot stacks
+ * are safe to free.
+ */
+ smp_rendezvous(smp_no_rendezvous_barrier, NULL,
+ smp_no_rendezvous_barrier, NULL);
+
+ for (cpu = 1; cpu <= mp_maxid; cpu++) {
+ if (bootstacks[cpu] != NULL)
+ kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE);
+ }
+}
+SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
+ smp_after_idle_runnable, NULL);
+
+void
+la_stale_reset(void)
+{
+ int cpuid = PCPU_GET(cpuid);
+
+ la_stale[cpuid].count = 0;
+ la_stale[cpuid].overflow = false;
+}
+
+/*
+ * Perform the local TLB invalidation described by (pmap, op, addr1, addr2).
+ */
+static void
+la_tlb_local_flush(pmap_t pmap, uint8_t op, uint64_t addr1, uint64_t addr2)
+{
+ uint32_t asid;
+ uint64_t va;
+
+ if (pmap == kernel_pmap) {
+ asid = csr_read32(LOONGARCH_CSR_ASID) & CSR_ASID_ASID;
+ switch (op) {
+ case LA_TLB_OP_PAGE:
+ invtlb(INVTLB_ADDR_GTRUE_OR_ASID, asid,
+ addr1 & ~((PAGE_SIZE << 1) - 1));
+ break;
+ case LA_TLB_OP_RANGE:
+ for (va = addr1 & ~((PAGE_SIZE << 1) - 1);
+ va < addr2; va += (PAGE_SIZE << 1))
+ invtlb(INVTLB_ADDR_GTRUE_OR_ASID, asid, va);
+ break;
+ default:
+ invtlb_all(INVTLB_CURRENT_ALL);
+ /*
+ * CURRENT_ALL flushed every entry on this CPU,
+ * including all stale user-ASID entries. Reset
+ * the stale tracker so the next user-pmap flush
+ * doesn't repeat a full G=0 invalidation for
+ * entries that no longer exist.
+ */
+ la_stale_reset();
+ break;
+ }
+ } else {
+ int cpuid = PCPU_GET(cpuid);
+ int i;
+
+ critical_enter();
+
+ /*
+ * Batch-flush stale ASIDs from prior context switches.
+ * If the stale array overflowed (many switches without
+ * an invalidation), do a full G=0 flush to catch all
+ * stale ASIDs including the dropped ones -- the target
+ * ASID's entries are also flushed, so skip it.
+ */
+ if (la_stale[cpuid].overflow) {
+ invtlb_all(INVTLB_CURRENT_GFALSE);
+ } else {
+ for (i = 0; i < la_stale[cpuid].count; i++)
+ invtlb(INVTLB_GFALSE_AND_ASID,
+ la_stale[cpuid].asids[i], 0);
+ /* Flush the target pmap's ASID entries. */
+ asid = pmap_get_asid(pmap, cpuid);
+ if (asid != 0)
+ invtlb(INVTLB_GFALSE_AND_ASID, asid, 0);
+ else
+ invtlb_all(INVTLB_CURRENT_GFALSE);
+ }
+ la_stale[cpuid].count = 0;
+ la_stale[cpuid].overflow = false;
+
+ critical_exit();
+ }
+}
+
+/*
+ * Synchronous TLB shootdown via per-CPU lists of descriptors.
+ *
+ * The initiator pops a descriptor from its per-CPU pool, fills it, appends it
+ * to its pending list, and IPIs the targets; the handler drains every other
+ * initiator's pending list. Because every in-flight shootdown from a CPU owns
+ * its own descriptor, fire-and-forget shootdowns from the same CPU (IE-off
+ * callers such as pmap_qenter/UMA) no longer overwrite an earlier one before
+ * the targets have flushed it -- the race that silently dropped invalidations
+ * and left stale TLBs on remote CPUs under multi-AP load.
+ *
+ * A descriptor returns to the pool once every target has acked it
+ * (sd->remaining == 0); the synchronous (IE-on) path spin-waits on that. An
+ * IE-off caller skips the wait (fire-and-forget) -- its descriptor stays on the
+ * list until the targets drain it. If the pool is exhausted the initiator
+ * escalates to a full flush (la_tlb_flushall_gen), so no invalidation is lost.
+ */
+void
+smp_targeted_tlb_shootdown(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2,
+ uint8_t op)
+{
+ struct la_tlb_sd *sd, *old;
+ cpuset_t mask;
+ u_int count;
+ uint32_t fagen;
+ int cpu, self;
+
+ if (kdb_active || KERNEL_PANICKED() || !smp_started) {
+ la_tlb_local_flush(pmap, op, addr1, addr2);
+ return;
+ }
+
+ mask = pmap->pm_active;
+ self = PCPU_GET(cpuid);
+ CPU_CLR(self, &mask);
+ if (CPU_EMPTY(&mask)) {
+ la_tlb_local_flush(pmap, op, addr1, addr2);
+ return;
+ }
+ count = CPU_COUNT(&mask);
+
+ /*
+ * Critical section + per-CPU spinlock across the list manipulation,
+ * IPI, and (optional) wait. IE stays on so incoming shootdown IPIs are
+ * still serviced; la_invlop_handler skips this CPU's own list, so it
+ * never blocks on the lock we hold here (no self-deadlock).
+ */
+ critical_enter();
+ mtx_lock_spin(&la_tlb_mtx[self]);
+
+ /* Reclaim fully-acked descriptors (any position) back to the
+ * free list. */
+ TAILQ_FOREACH_SAFE(sd, &la_tlb_q[self], pend_link, old) {
+ if (atomic_load_int(&sd->remaining) == 0) {
+ TAILQ_REMOVE(&la_tlb_q[self], sd, pend_link);
+ SLIST_INSERT_HEAD(&la_tlb_free[self], sd, free_link);
+ }
+ }
+ if (TAILQ_EMPTY(&la_tlb_q[self]))
+ CPU_CLR_ATOMIC(self, &la_tlb_pending);
+
+ sd = SLIST_FIRST(&la_tlb_free[self]);
+ if (sd != NULL) {
+ SLIST_REMOVE_HEAD(&la_tlb_free[self], free_link);
+ sd->pmap = pmap;
+ sd->addr1 = addr1;
+ sd->addr2 = addr2;
+ sd->op = op;
+ sd->targets = mask;
+ CPU_ZERO(&sd->acked);
+ atomic_store_int(&sd->remaining, count);
+ TAILQ_INSERT_TAIL(&la_tlb_q[self], sd, pend_link);
+ CPU_SET_ATOMIC(self, &la_tlb_pending);
+ mtx_unlock_spin(&la_tlb_mtx[self]);
+
+ ipi_selected(mask, IPI_TLB);
+ la_tlb_local_flush(pmap, op, addr1, addr2);
+
+ if ((csr_read32(LOONGARCH_CSR_CRMD) & CSR_CRMD_IE) != 0) {
+ TLB_DBG("TLB shootdown: cpu%d sync wait mask=%#x "
+ "op=%d va=%#lx\n",
+ self, *(uint32_t *)&mask, op, (u_long)addr1);
+ while (atomic_load_int(&sd->remaining) != 0)
+ cpu_spinwait();
+ } else {
+ /*
+ * Interrupts disabled: we cannot spin-wait for the
+ * targets to drain this descriptor. It stays on the
+ * pending list, still referencing sd->pmap, until
+ * la_invlop_handler drains it (and calls
+ * pmap_get_asid(sd->pmap)). Only kernel_pmap
+ * (immortal) is safe to reference that way -- a user
+ * pmap could be freed by pmap_release before the
+ * targets drain, a use-after-free. Every current
+ * IE-off caller targets kernel_pmap; keep it that way.
+ */
+ KASSERT(pmap == kernel_pmap,
+ ("IE-off TLB shootdown posted for non-kernel "
+ "pmap %p",
+ pmap));
+ }
+ } else {
+ /*
+ * Pool exhausted (>= LA_TLB_POOL in-flight from this CPU):
+ * escalate to a full flush so no invalidation is lost; the
+ * targets pick this up via la_tlb_flushall_gen.
+ */
+ atomic_add_int(&la_tlb_flushall_gen[self], 1);
+ fagen = atomic_load_int(&la_tlb_flushall_gen[self]);
+ CPU_SET_ATOMIC(self, &la_tlb_pending);
+ mtx_unlock_spin(&la_tlb_mtx[self]);
+
+ ipi_selected(mask, IPI_TLB);
+ la_tlb_local_flush(pmap, LA_TLB_OP_ALL, 0, 0);
+
+ if ((csr_read32(LOONGARCH_CSR_CRMD) & CSR_CRMD_IE) != 0) {
+ CPU_FOREACH(cpu) {
+ if (!CPU_ISSET(cpu, &mask))
+ continue;
+ while (atomic_load_int(
+ &la_tlb_flushall_done[cpu][self]) < fagen)
+ cpu_spinwait();
+ }
+ }
+ }
+ critical_exit();
+}
+
+/*
+ * IPI_TLB handler: drain every other initiator's pending list, flushing each
+ * descriptor this CPU hasn't acked yet, and service any pending full-flush
+ * fallback. This CPU's own list (i == self) is skipped -- the initiator
+ * flushes itself locally, and taking our own lock here would self-deadlock if
+ * we interrupted the initiator while it holds it. The flush happens before
+ * the ack so the initiator's wait (on sd->remaining) only completes once this
+ * CPU's stale entry is actually gone.
+ */
+static void
+la_invlop_handler(void)
+{
+ struct la_tlb_sd *sd;
+ cpuset_t pend;
+ uint32_t fagen;
+ int self, i;
+
+ self = PCPU_GET(cpuid);
+ /*
+ * Scan only initiators that currently have descriptors queued, instead
+ * of every CPU. An initiator sets its bit under la_tlb_mtx[] before
+ * sending the IPI, and the IPI doorbell fence publishes that store
+ * before this handler runs, so a just-posted descriptor is never
+ * missed. A stale bit (the initiator drained its queue after this
+ * snapshot) is harmless: we lock, scan an empty queue, and exit.
+ */
+ pend = la_tlb_pending;
+ CPU_CLR(self, &pend);
+ CPU_FOREACH(i) {
+ if (!CPU_ISSET(i, &pend))
+ continue;
+ mtx_lock_spin(&la_tlb_mtx[i]);
+ TAILQ_FOREACH(sd, &la_tlb_q[i], pend_link) {
+ if (CPU_ISSET(self, &sd->targets) &&
+ !CPU_ISSET(self, &sd->acked)) {
+ la_tlb_local_flush(sd->pmap, sd->op,
+ sd->addr1, sd->addr2);
+ CPU_SET(self, &sd->acked);
+ atomic_subtract_int(&sd->remaining, 1);
+ }
+ }
+ fagen = atomic_load_int(&la_tlb_flushall_gen[i]);
+ if (fagen > atomic_load_int(&la_tlb_flushall_done[self][i])) {
+ la_tlb_local_flush(kernel_pmap, LA_TLB_OP_ALL, 0, 0);
+ atomic_store_int(&la_tlb_flushall_done[self][i], fagen);
+ }
+ mtx_unlock_spin(&la_tlb_mtx[i]);
+ }
+}
+
+static int
+ipi_handler(void *arg)
+{
+ u_int ipi_bitmap;
+ u_int cpu, ipi;
+ int bit;
+
+ cpu = PCPU_GET(cpuid);
+
+ for (;;) {
+ mb();
+
+ ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis));
+
+ if (ipi_bitmap == 0) {
+ /*
+ * Clear the hardware IPI status. Without this
+ * the interrupt line stays asserted and the CPU
+ * would re-enter the handler immediately on ertn.
+ */
+ ipi_read_clear(cpu);
+ /*
+ * Re-check after clearing: an IPI may have arrived
+ * between the readandclear and the ipi_read_clear.
+ * Its hardware status was just cleared so the
+ * interrupt will not re-fire — we must process it
+ * here to avoid losing it.
+ */
+ if (atomic_load_int(PCPU_PTR(pending_ipis)) == 0)
+ return (FILTER_HANDLED);
+ continue;
+ }
+
+ while ((bit = ffs(ipi_bitmap))) {
+ bit = (bit - 1);
+ ipi = (1 << bit);
+ ipi_bitmap &= ~ipi;
+
+ mb();
+
+ switch (ipi) {
+ case IPI_AST:
+ CTR0(KTR_SMP, "IPI_AST");
+ break;
+ case IPI_PREEMPT:
+ CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
+ sched_preempt(curthread);
+ break;
+ case IPI_RENDEZVOUS:
+ CTR0(KTR_SMP, "IPI_RENDEZVOUS");
+ smp_rendezvous_action();
+ break;
+ case IPI_STOP:
+ case IPI_STOP_HARD:
+ CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" :
+ "IPI_STOP_HARD");
+ savectx(&stoppcbs[cpu]);
+
+ /* Indicate we are stopped */
+ CPU_SET_ATOMIC(cpu, &stopped_cpus);
+
+ /* Wait for restart */
+ while (!CPU_ISSET(cpu, &started_cpus))
+ cpu_spinwait();
+
+ CPU_CLR_ATOMIC(cpu, &started_cpus);
+ CPU_CLR_ATOMIC(cpu, &stopped_cpus);
+ CTR0(KTR_SMP, "IPI_STOP (restart)");
+
+ /*
+ * The kernel debugger might have set a
+ * breakpoint, so flush the instruction cache.
+ */
+ flush_icache();
+ break;
+ case IPI_HARDCLOCK:
+ CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
+ hardclockintr();
+ break;
+ case IPI_TLB:
+ CTR1(KTR_SMP, "%s: IPI_TLB", __func__);
+ la_invlop_handler();
+ break;
+ default:
+ panic("Unknown IPI %#0x on cpu %d",
+ ipi, curcpu);
+ }
+ }
+
+ /*
+ * Clear hardware IPI status. An IPI that arrived
+ * during processing set pc_pending_ipis but its
+ * hardware status is cleared here. Re-check to avoid
+ * losing it.
+ */
+ ipi_read_clear(cpu);
+
+ if (atomic_load_int(PCPU_PTR(pending_ipis)) == 0)
+ return (FILTER_HANDLED);
+ }
+}
+
+struct cpu_group *
+cpu_topo(void)
+{
+
+ return (smp_topo_none());
+}
+
+/* Determine if we running MP machine */
+int
+cpu_mp_probe(void)
+{
+
+ return (mp_ncpus > 1);
+}
+
+#ifdef FDT
+static bool
+cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
+{
+ struct pcpu *pcpup;
+ u_int cpuid;
+ uint64_t reg_data;
+ int naps;
+ uint64_t addr;
+
+ KASSERT(id < MAXCPU, ("Too many CPUs"));
+
+ KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size"));
+#ifdef INVARIANTS
+ cpu_reg[id][0] = reg[0];
+ if (addr_size == 2)
+ cpu_reg[id][1] = reg[1];
+#endif
+
+ reg_data = reg[0];
+ if (addr_size == 2) {
+ reg_data <<= 32;
+ reg_data |= reg[1];
+ }
+
+ cpuid = (u_int)reg_data;
+
+ KASSERT(cpuid < MAXCPU, ("Too many cpus."));
+
+ /* We are already running on this cpu */
+ if (cpuid == boot_cpu)
+ return (true);
+
+ /*
+ * Rotate the CPU IDs to put the boot CPU as CPU 0.
+ * We keep the other CPUs ordered.
+ */
+ if (cpuid < boot_cpu)
+ cpuid += mp_maxid + 1;
+ cpuid -= boot_cpu;
+
+ /* Check if we are able to start this cpu */
+ if (cpuid > mp_maxid)
+ return (false);
+
+ pcpup = &__pcpu[cpuid];
+ pcpu_init(pcpup, cpuid, sizeof(struct pcpu));
+
+ dpcpu[cpuid - 1] = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
+ dpcpu_init(dpcpu[cpuid - 1], cpuid);
+
+ bootstacks[cpuid] = kmem_malloc(MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO);
+
+ naps = atomic_load_int(&aps_started);
+ bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE;
+
+ if (bootverbose)
+ printf("Starting CPU %u\n", cpuid);
+
+ addr = pmap_extract(pmap_kernel(), (vm_offset_t)mpentry);
+ printf("secondary cpu entry: %p\n", (void*)addr);
+ /*
+ * Address the AP by its hardware id (the raw FDT reg value in
+ * reg_data), not the renumbered 0-based cpuid: csr_mail_send targets a
+ * physical CPU, and the AP side (locore.S mpentry) indexes
+ * __loongarch_boot_ap by CSR_CPUID == reg_data. ipi_cpu() takes the
+ * 0-based cpuid (it indexes cpuid_to_pcpu[]), so it keeps cpuid.
+ */
+ csr_mail_send(addr, (u_int)reg_data, 0);
+ ipi_cpu(cpuid, 2);
+
+ atomic_store_32(&__loongarch_boot_ap[(u_int)reg_data], 1);
+
+ /* Wait for the AP to switch to its boot stack. */
+ while (atomic_load_int(&aps_started) < naps + 1)
+ cpu_spinwait();
+
+ CPU_SET(cpuid, &all_cpus);
+
+ return (true);
+}
+#endif
+
+#ifdef DEV_ACPI
+/*
+ * ACPI twin of cpu_init_fdt: start one AP identified by its hardware core id
+ * (the MADT CORE_PIC CoreId, == CSR_CPUID). The AP is woken the same way as
+ * on FDT (csr_mail_send of the mpentry address + IPI); only the enumeration
+ * source differs.
+ */
+static bool
+cpu_init_acpi(u_int coreid)
+{
+ struct pcpu *pcpup;
+ u_int cpuid;
+ int naps;
+ uint64_t addr;
+
+ if (coreid == boot_cpu)
+ return (true);
+
+ cpuid = coreid;
+ if (cpuid < boot_cpu)
+ cpuid += mp_maxid + 1;
+ cpuid -= boot_cpu;
+
+ if (cpuid > mp_maxid)
+ return (false);
+
+ pcpup = &__pcpu[cpuid];
+ pcpu_init(pcpup, cpuid, sizeof(struct pcpu));
+
+ dpcpu[cpuid - 1] = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
+ dpcpu_init(dpcpu[cpuid - 1], cpuid);
+
+ bootstacks[cpuid] = kmem_malloc(MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO);
+
+ naps = atomic_load_int(&aps_started);
+ bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE;
+
+ if (bootverbose)
+ printf("Starting CPU %u\n", cpuid);
+
+ addr = pmap_extract(pmap_kernel(), (vm_offset_t)mpentry);
+ csr_mail_send(0, coreid, 0);
+ csr_mail_send(addr, coreid, 0);
+ ipi_write_action(coreid, 1);
+
+ atomic_store_32(&__loongarch_boot_ap[coreid], 1);
+
+ /* Wait for the AP to switch to its boot stack. */
+ while (atomic_load_int(&aps_started) < naps + 1)
+ cpu_spinwait();
+
+ /*
+ * Mark the AP present: CPU_ABSENT() tests all_cpus, and without this
+ * cpu_initclocks_ap()->hardclock_sync() panics "Absent CPU N" on every
+ * AP (concurrent panics garble the console). Matches cpu_init_fdt().
+ */
+ CPU_SET(cpuid, &all_cpus);
+
+ return (true);
+}
+#endif
+
+/* Initialize and fire up non-boot processors */
+void
+cpu_mp_start(void)
+{
+ mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
+
+ CPU_SET(boot_cpu, &all_cpus);
+
+#ifdef DEV_ACPI
+ /*
+ * On ACPI/3A6000: enable all IOCSR IPI vectors and the extended I/O
+ * interrupt controller before waking the APs. Firmware on QEMU leaves
+ * these enabled (and its FDT eioic driver attaches), but on 3A6000 they
+ * are masked and the eioic driver never attaches, so do it here.
+ */
+ iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
+ /*
+ * EXTIOI was already enabled + the EIOINTC router programmed once
+ * in eioic_attach() (BUS_PASS_INTERRUPT, before device probe).
+ * Do NOT re-enable here: eiointc_enable_extioi() resets the global
+ * ROUTE to 0, creating a window where all device interrupts are
+ * dropped. eiointc_enable() is called only on the first CPU of
+ * each node.
+ */
+#endif
+
+ switch(cpu_enum_method) {
+#ifdef FDT
+ case CPUS_FDT:
+ ofw_cpu_early_foreach(cpu_init_fdt, true);
+ break;
+#endif
+#ifdef DEV_ACPI
+ case CPUS_ACPI:
+ loongarch_acpi_madt_foreach_cpu(cpu_init_acpi);
+ break;
+#endif
+ case CPUS_UNKNOWN:
+ break;
+ }
+}
+
+/* Introduce rest of cores to the world */
+void
+cpu_mp_announce(void)
+{
+ u_int cpu;
+
+ CPU_FOREACH(cpu) {
+ /* Already announced. */
+ if (cpu == 0)
+ continue;
+
+ printcpuinfo(cpu);
+ }
+}
+
+void
+cpu_mp_setmaxid(void)
+{
+ int cores;
+ int maxcpus = MAXCPU;
+
+ /*
+ * Honor the kern.maxcpus loader tunable to limit the number of CPUs
+ * brought online (for SMP debugging). Without this, all cores found in
+ * the MADT/FDT are started regardless.
+ */
+ TUNABLE_INT_FETCH("kern.maxcpus", &maxcpus);
+ if (maxcpus < 1)
+ maxcpus = 1;
+
+#ifdef DEV_ACPI
+ if (loongarch_efi_acpi_rsdp() != 0) {
+ cores = loongarch_acpi_madt_cpu_count();
+ if (cores > 0) {
+ cores = MIN(cores, MAXCPU);
+ cores = MIN(cores, maxcpus);
+ if (bootverbose)
+ printf("Found %d CPUs in the MADT\n", cores);
+ mp_ncpus = cores;
+ mp_maxid = cores - 1;
+ cpu_enum_method = CPUS_ACPI;
+ } else {
+ if (bootverbose)
+ printf("No CPUs in MADT, limiting to 1 core\n");
+ mp_ncpus = 1;
+ mp_maxid = 0;
+ }
+ } else
+#endif
+#ifdef FDT
+ {
+ cores = ofw_cpu_early_foreach(NULL, true);
+ if (cores > 0) {
+ cores = MIN(cores, MAXCPU);
+ cores = MIN(cores, maxcpus);
+ if (bootverbose)
+ printf("Found %d CPUs in the device "
+ "tree\n", cores);
+ mp_ncpus = cores;
+ mp_maxid = cores - 1;
+ cpu_enum_method = CPUS_FDT;
+ } else {
+ if (bootverbose)
+ printf("No CPU data, limiting to 1 core\n");
+ mp_ncpus = 1;
+ mp_maxid = 0;
+ }
+ }
+#else
+ {
+ if (bootverbose)
+ printf("No CPU data, limiting to 1 core\n");
+ mp_ncpus = 1;
+ mp_maxid = 0;
+ }
+#endif
+
+ if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) {
+ if (cores > 0 && cores < mp_ncpus) {
+ mp_ncpus = cores;
+ mp_maxid = cores - 1;
+ }
+ }
+}
diff --git a/sys/loongarch/loongarch/nexus.c b/sys/loongarch/loongarch/nexus.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/nexus.c
@@ -0,0 +1,522 @@
+/*-
+ * Copyright 1998 Massachusetts Institute of Technology
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that both the above copyright notice and this
+ * permission notice appear in all copies, that both the above
+ * copyright notice and this permission notice appear in all
+ * supporting documentation, and that the name of M.I.T. not be used
+ * in advertising or publicity pertaining to distribution of the
+ * software without specific, written prior permission. M.I.T. makes
+ * no representations about the suitability of this software for any
+ * purpose. It is provided "as is" without express or implied
+ * warranty.
+ *
+ * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS
+ * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
+ * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*
+ * This code implements a `root nexus' for LoongArch machines.
+ * The function of the root nexus is to serve as an attachment point
+ * for both processors and buses, and to manage resources which are
+ * common to all of them.
+ */
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/interrupt.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <machine/intr.h>
+#include <machine/machdep.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#ifdef FDT
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/openfirm.h>
+#include "ofw_bus_if.h"
+#endif
+
+#include "opt_acpi.h"
+#ifdef DEV_ACPI
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+#include "acpi_bus_if.h"
+#endif
+
+extern struct bus_space memmap_bus;
+
+static MALLOC_DEFINE(M_NEXUSDEV, "nexusdev", "Nexus device");
+
+struct nexus_device {
+ struct resource_list nx_resources;
+};
+
+#define DEVTONX(dev) ((struct nexus_device *)device_get_ivars(dev))
+
+static struct rman mem_rman;
+static struct rman irq_rman;
+
+static device_probe_t nexus_fdt_probe;
+static device_attach_t nexus_attach;
+
+static bus_add_child_t nexus_add_child;
+static bus_print_child_t nexus_print_child;
+
+static bus_activate_resource_t nexus_activate_resource;
+static bus_alloc_resource_t nexus_alloc_resource;
+static bus_deactivate_resource_t nexus_deactivate_resource;
+static bus_get_resource_list_t nexus_get_reslist;
+static bus_get_rman_t nexus_get_rman;
+static bus_map_resource_t nexus_map_resource;
+static bus_unmap_resource_t nexus_unmap_resource;
+
+static bus_config_intr_t nexus_config_intr;
+static bus_describe_intr_t nexus_describe_intr;
+static bus_setup_intr_t nexus_setup_intr;
+static bus_teardown_intr_t nexus_teardown_intr;
+
+static bus_get_bus_tag_t nexus_get_bus_tag;
+
+static ofw_bus_map_intr_t nexus_ofw_map_intr;
+#ifdef DEV_ACPI
+static int nexus_acpi_map_intr(device_t, device_t, u_int,
+ int, int);
+#endif
+
+static device_method_t nexus_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_probe, nexus_fdt_probe),
+ DEVMETHOD(device_attach, nexus_attach),
+ DEVMETHOD(device_shutdown, bus_generic_shutdown),
+
+ /* OFW interface */
+ DEVMETHOD(ofw_bus_map_intr, nexus_ofw_map_intr),
+
+ #ifdef DEV_ACPI
+ /* ACPI bus interface */
+ DEVMETHOD(acpi_bus_map_intr, nexus_acpi_map_intr),
+ #endif
+
+ /* Bus interface */
+ DEVMETHOD(bus_add_child, nexus_add_child),
+ DEVMETHOD(bus_print_child, nexus_print_child),
+ DEVMETHOD(bus_activate_resource, nexus_activate_resource),
+ DEVMETHOD(bus_adjust_resource, bus_generic_rman_adjust_resource),
+ DEVMETHOD(bus_alloc_resource, nexus_alloc_resource),
+ DEVMETHOD(bus_deactivate_resource, nexus_deactivate_resource),
+ DEVMETHOD(bus_delete_resource, bus_generic_rl_delete_resource),
+ DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
+ DEVMETHOD(bus_get_resource_list, nexus_get_reslist),
+ DEVMETHOD(bus_get_rman, nexus_get_rman),
+ DEVMETHOD(bus_map_resource, nexus_map_resource),
+ DEVMETHOD(bus_release_resource, bus_generic_rman_release_resource),
+ DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
+ DEVMETHOD(bus_unmap_resource, nexus_unmap_resource),
+ DEVMETHOD(bus_config_intr, nexus_config_intr),
+ DEVMETHOD(bus_describe_intr, nexus_describe_intr),
+ DEVMETHOD(bus_setup_intr, nexus_setup_intr),
+ DEVMETHOD(bus_teardown_intr, nexus_teardown_intr),
+ DEVMETHOD(bus_get_bus_tag, nexus_get_bus_tag),
+
+ DEVMETHOD_END
+};
+
+static driver_t nexus_fdt_driver = {
+ "nexus",
+ nexus_methods,
+ 1 /* no softc */
+};
+
+EARLY_DRIVER_MODULE(nexus_fdt, root, nexus_fdt_driver, 0, 0,
+ BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);
+
+static int
+nexus_fdt_probe(device_t dev)
+{
+
+ device_quiet(dev);
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+nexus_attach(device_t dev)
+{
+
+ mem_rman.rm_start = 0;
+ mem_rman.rm_end = BUS_SPACE_MAXADDR;
+ mem_rman.rm_type = RMAN_ARRAY;
+ mem_rman.rm_descr = "I/O memory addresses";
+ if (rman_init(&mem_rman) ||
+ rman_manage_region(&mem_rman, 0, BUS_SPACE_MAXADDR))
+ panic("nexus_attach mem_rman");
+ irq_rman.rm_start = 0;
+ irq_rman.rm_end = ~0;
+ irq_rman.rm_type = RMAN_ARRAY;
+ irq_rman.rm_descr = "Interrupts";
+ if (rman_init(&irq_rman) || rman_manage_region(&irq_rman, 0, ~0))
+ panic("nexus_attach irq_rman");
+
+ /*
+ * Add direct children of nexus. Devices will be probed and attached
+ * through ofwbus0.
+ *
+ * On ACPI systems (no FDT) the interrupt controllers have no
+ * device-tree nodes, so instantiate them directly here: cpuic
+ * (CPU-local), eioic (Extended I/O), and pch_pic (the platform PIC
+ * that owns the GSI space and routes device interrupts to the
+ * EIOINTC). The timer needs its IRQ_TI ISRC via
+ * cpuic_setup_cpu_intr().
+ */
+ if (loongarch_efi_acpi_rsdp() != 0) {
+ nexus_add_child(dev, 0, "cpuic", 0);
+ nexus_add_child(dev, 1, "eioic", 0);
+ nexus_add_child(dev, 2, "pch_pic", 0);
+ nexus_add_child(dev, 3, "pch_msi", 0);
+ nexus_add_child(dev, 4, "acpi", 0);
+ }
+ nexus_add_child(dev, 5, "timer", 0);
+ nexus_add_child(dev, 6, "rcons", 0);
+ nexus_add_child(dev, 7, "ofwbus", 0);
+
+ bus_identify_children(dev);
+ bus_attach_children(dev);
+
+ return (0);
+}
+
+static int
+nexus_print_child(device_t bus, device_t child)
+{
+ int retval = 0;
+
+ retval += bus_print_child_header(bus, child);
+ retval += printf("\n");
+
+ return (retval);
+}
+
+static device_t
+nexus_add_child(device_t bus, u_int order, const char *name, int unit)
+{
+ device_t child;
+ struct nexus_device *ndev;
+
+ ndev = malloc(sizeof(struct nexus_device), M_NEXUSDEV, M_NOWAIT|M_ZERO);
+ if (!ndev)
+ return (0);
+ resource_list_init(&ndev->nx_resources);
+
+ child = device_add_child_ordered(bus, order, name, unit);
+
+ device_set_ivars(child, ndev);
+
+ return (child);
+}
+
+static struct rman *
+nexus_get_rman(device_t bus, int type, u_int flags)
+{
+
+ switch (type) {
+ case SYS_RES_IRQ:
+ return (&irq_rman);
+ case SYS_RES_MEMORY:
+ return (&mem_rman);
+ default:
+ return (NULL);
+ }
+}
+
+/*
+ * Allocate a resource on behalf of child. NB: child is usually going to be a
+ * child of one of our descendants, not a direct child of nexus0.
+ */
+static struct resource *
+nexus_alloc_resource(device_t bus, device_t child, int type, int rid,
+ rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
+{
+ struct nexus_device *ndev = DEVTONX(child);
+ struct resource_list_entry *rle;
+
+ /*
+ * If this is an allocation of the "default" range for a given
+ * RID, and we know what the resources for this device are
+ * (ie. they aren't maintained by a child bus), then work out
+ * the start/end values.
+ */
+ if (RMAN_IS_DEFAULT_RANGE(start, end) && (count == 1)) {
+ if (device_get_parent(child) != bus || ndev == NULL) {
+ /*
+ * PCI INTx quirk: QEMU virt doesn't program PCI
+ * interrupt-line. Resolve via swizzle and register
+ * through intr_map_irq so irq_map[] is populated.
+ */
+#ifdef FDT
+ if (type == SYS_RES_IRQ) {
+ uint8_t intpin = pci_get_intpin(child);
+ intptr_t xref = pch_pic_get_xref();
+ if (intpin != 0 && xref != 0) {
+ u_int pch_irq = 16 + ((intpin - 1 +
+ pci_get_slot(child)) % 4);
+ struct intr_map_data_fdt *fdt;
+ int irq;
+
+ fdt = (struct intr_map_data_fdt *)
+ intr_alloc_map_data(
+ INTR_MAP_DATA_FDT,
+ sizeof(*fdt) + 2*sizeof(pcell_t),
+ M_WAITOK | M_ZERO);
+ fdt->iparent = (phandle_t)xref;
+ fdt->ncells = 2;
+ fdt->cells[0] = pch_irq;
+ fdt->cells[1] = 4; /* LEVEL_HIGH */
+ irq = intr_map_irq(NULL, xref,
+ (struct intr_map_data *)fdt);
+ if (irq >= 0) {
+ start = irq;
+ end = irq;
+ }
+ }
+ }
+#endif
+ if (RMAN_IS_DEFAULT_RANGE(start, end))
+ return (NULL);
+ } else {
+ rle = resource_list_find(&ndev->nx_resources,
+ type, rid);
+ if (rle == NULL)
+ return (NULL);
+ start = rle->start;
+ end = rle->end;
+ count = rle->count;
+ }
+ }
+
+ return (bus_generic_rman_alloc_resource(bus, child, type, rid,
+ start, end, count, flags));
+}
+
+static int
+nexus_config_intr(device_t dev, int irq, enum intr_trigger trig,
+ enum intr_polarity pol)
+{
+
+ return (EOPNOTSUPP);
+}
+
+static int
+nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags,
+ driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep)
+{
+ int error;
+
+ if ((rman_get_flags(res) & RF_SHAREABLE) == 0)
+ flags |= INTR_EXCL;
+
+ /* We depend here on rman_activate_resource() being idempotent. */
+ error = rman_activate_resource(res);
+ if (error != 0)
+ return (error);
+
+ error = intr_setup_irq(child, res, filt, intr, arg, flags, cookiep);
+
+ return (error);
+}
+
+static int
+nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
+{
+
+ return (intr_teardown_irq(child, r, ih));
+}
+
+static int
+nexus_describe_intr(device_t dev, device_t child, struct resource *irq,
+ void *cookie, const char *descr)
+{
+
+ return (intr_describe_irq(child, irq, cookie, descr));
+}
+
+static bus_space_tag_t
+nexus_get_bus_tag(device_t bus __unused, device_t child __unused)
+{
+
+ return (&memmap_bus);
+}
+
+static int
+nexus_activate_resource(device_t bus, device_t child, struct resource *r)
+{
+ int error;
+
+ switch (rman_get_type(r)) {
+ case SYS_RES_MEMORY:
+ error = bus_generic_rman_activate_resource(bus, child, r);
+ break;
+ case SYS_RES_IRQ:
+ error = rman_activate_resource(r);
+ if (error != 0)
+ return (error);
+ error = intr_activate_irq(child, r);
+ if (error != 0) {
+ rman_deactivate_resource(r);
+ return (error);
+ }
+ break;
+ default:
+ error = EINVAL;
+ break;
+ }
+ return (error);
+}
+
+static struct resource_list *
+nexus_get_reslist(device_t dev, device_t child)
+{
+ struct nexus_device *ndev = DEVTONX(child);
+
+ return (&ndev->nx_resources);
+}
+
+static int
+nexus_deactivate_resource(device_t bus, device_t child, struct resource *r)
+{
+ int error;
+
+ switch (rman_get_type(r)) {
+ case SYS_RES_MEMORY:
+ error = bus_generic_rman_deactivate_resource(bus, child, r);
+ break;
+ case SYS_RES_IRQ:
+ error = rman_deactivate_resource(r);
+ if (error != 0)
+ return (error);
+ intr_deactivate_irq(child, r);
+ break;
+ default:
+ error = EINVAL;
+ break;
+ }
+ return (error);
+}
+
+static int
+nexus_map_resource(device_t bus, device_t child, struct resource *r,
+ struct resource_map_request *argsp, struct resource_map *map)
+{
+ struct resource_map_request args;
+ rman_res_t length, start;
+ int error;
+
+ /* Resources must be active to be mapped. */
+ if ((rman_get_flags(r) & RF_ACTIVE) == 0)
+ return (ENXIO);
+
+ /* Mappings are only supported on memory resources. */
+ switch (rman_get_type(r)) {
+ case SYS_RES_MEMORY:
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ resource_init_map_request(&args);
+ error = resource_validate_map_request(r, argsp, &args, &start, &length);
+ if (error)
+ return (error);
+
+ map->r_vaddr = pmap_mapdev(start, length);
+ map->r_bustag = &memmap_bus;
+ map->r_size = length;
+
+ /*
+ * The handle is the virtual address.
+ */
+ map->r_bushandle = (bus_space_handle_t)map->r_vaddr;
+ return (0);
+}
+
+static int
+nexus_unmap_resource(device_t bus, device_t child, struct resource *r,
+ struct resource_map *map)
+{
+ switch (rman_get_type(r)) {
+ case SYS_RES_MEMORY:
+ pmap_unmapdev(map->r_vaddr, map->r_size);
+ return (0);
+ default:
+ return (EINVAL);
+ }
+}
+
+#ifdef DEV_ACPI
+/*
+ * ACPI interrupt mapping. Each device GSI (global system interrupt) from
+ * ACPI _CRS/_PRT is wrapped in an intr_map_data_acpi and recorded in the
+ * INTRNG irq_map[] table via intr_map_irq(), which returns the resource id.
+ * Device GSIs are routed through the PCH-PIC, the controller that owns the
+ * GSI space (ACPI_INTR_XREF); pch_pic_map_intr later resolves the GSI to an
+ * EIOINTC ISRC when the resource is activated.
+ */
+static int
+nexus_acpi_map_intr(device_t bus, device_t dev, u_int irq, int trig, int pol)
+{
+ struct intr_map_data_acpi *acpi_data;
+
+ acpi_data = (struct intr_map_data_acpi *)intr_alloc_map_data(
+ INTR_MAP_DATA_ACPI, sizeof(*acpi_data), M_WAITOK | M_ZERO);
+ acpi_data->irq = irq;
+ acpi_data->pol = pol;
+ acpi_data->trig = trig;
+
+ return (intr_map_irq(NULL, ACPI_INTR_XREF,
+ (struct intr_map_data *)acpi_data));
+}
+#endif
+
+static int
+nexus_ofw_map_intr(device_t dev, device_t child, phandle_t iparent, int icells,
+ pcell_t *intr)
+{
+ struct intr_map_data_fdt *fdt_data;
+ size_t len;
+ u_int irq;
+
+ len = sizeof(*fdt_data) + icells * sizeof(pcell_t);
+ fdt_data = (struct intr_map_data_fdt *)intr_alloc_map_data(
+ INTR_MAP_DATA_FDT, len, M_WAITOK | M_ZERO);
+ fdt_data->iparent = iparent;
+ fdt_data->ncells = icells;
+ memcpy(fdt_data->cells, intr, icells * sizeof(pcell_t));
+ irq = intr_map_irq(NULL, iparent, (struct intr_map_data *)fdt_data);
+
+ return (irq);
+}
diff --git a/sys/loongarch/loongarch/ofw_machdep.c b/sys/loongarch/loongarch/ofw_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/ofw_machdep.c
@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 2015 Ian Lepore <ian@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/bus.h>
+
+#include <machine/bus.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_subr.h>
+
+extern struct bus_space memmap_bus;
+
+int
+OF_decode_addr(phandle_t dev, int regno, bus_space_tag_t *tag,
+ bus_space_handle_t *handle, bus_size_t *sz)
+{
+ bus_addr_t addr;
+ bus_size_t size;
+ int err;
+
+ err = ofw_reg_to_paddr(dev, regno, &addr, &size, NULL);
+ if (err != 0)
+ return (err);
+
+ *tag = &memmap_bus;
+
+ if (sz != NULL)
+ *sz = size;
+
+ return (bus_space_map(*tag, addr, size, 0, handle));
+}
diff --git a/sys/loongarch/loongarch/pch_pic.c b/sys/loongarch/loongarch/pch_pic.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/pch_pic.c
@@ -0,0 +1,615 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * LoongArch PCH-PIC interrupt controller driver.
+ *
+ * The PCH-PIC is a hardware interrupt router in the LS7A chipset that
+ * maps up to 64 device interrupt inputs (INTx, LPC, etc.) to EIOINTC
+ * vectors. It does NOT have its own interrupt handler — each input is
+ * individually routed to a specific EIOINTC vector via the HTVEC
+ * register. EIOINTC handles all dispatch.
+ *
+ * QEMU virt configures 32 PCH-PIC inputs starting at EIOINTC vector 0.
+ * PCI INTx uses inputs 16-19 (VIRT_DEVICE_IRQS).
+ *
+ * Registers (MMIO, base typically 0x10000000):
+ * PCH_PIC_MASK 0x20 Mask (1 = disabled)
+ * PCH_PIC_HTMSI_EN 0x40 HT MSI enable
+ * PCH_PIC_EDGE 0x60 Edge/Level (1 = edge)
+ * PCH_PIC_CLR 0x80 Clear edge latch
+ * PCH_PIC_AUTO0 0xc0 Auto-bounce control
+ * PCH_PIC_AUTO1 0xe0
+ * PCH_INT_ROUTE(i) 0x100+i Per-IRQ route byte
+ * PCH_INT_HTVEC(i) 0x200+i Per-IRQ HT vector number
+ * PCH_PIC_POL 0x3e0 Polarity (1 = active low)
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/interrupt.h>
+#include <sys/callout.h>
+#include <sys/time.h>
+
+#include <machine/bus.h>
+#include <machine/intr.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <machine/machdep.h>
+
+#include "opt_acpi.h"
+#ifdef DEV_ACPI
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+#endif
+
+#include "pic_if.h"
+#include "eioic.h"
+
+#define PCH_PIC_MASK 0x20
+#define PCH_PIC_HTMSI_EN 0x40
+#define PCH_PIC_EDGE 0x60
+#define PCH_PIC_CLR 0x80
+#define PCH_PIC_AUTO0 0xc0
+#define PCH_PIC_AUTO1 0xe0
+#define PCH_INT_ROUTE(i) (0x100 + (i))
+#define PCH_INT_HTVEC(i) (0x200 + (i))
+#define PCH_PIC_POL 0x3e0
+
+#define PIC_COUNT_PER_REG 32
+#define PIC_COUNT 64
+
+/*
+ * Spurious-storm throttling.
+ *
+ * An input whose physical line is stuck asserted (e.g. the LS7A OHCI INTx
+ * line, which stays low even at intrstatus==0) re-fires on every post_ithread
+ * unmask, producing ~329k interrupts/sec (~10% CPU) with no real work.
+ *
+ * We detect this by measuring the gap between an unmask (post_ithread) and
+ * the next fire (pre_ithread): a gap below PCH_PIC_STORM_THRESH means the line
+ * was still asserted when we re-enabled it. storm_refire counts such gaps;
+ * once it reaches PCH_PIC_STORM_LIMIT we stop unmasking in post_ithread and
+ * instead re-arm the input from a callout every PCH_PIC_STORM_COOLDOWN,
+ * collapsing the rate to ~1/COOLDOWN while still letting real interrupts
+ * (e.g. OHCI WDH) through. A normal gap decrements storm_refire, so a line
+ * that stops being stuck resumes immediate unmasking automatically. Normal
+ * inputs (ahci/xhci/MSI) never see small gaps, so they are never throttled.
+ */
+#define PCH_PIC_STORM_THRESH (SBT_1MS * 2) /* gap < 2ms = fast re-fire */
+#define PCH_PIC_STORM_LIMIT 8 /* consecutive fast re-fires to trip */
+#define PCH_PIC_STORM_SAT (PCH_PIC_STORM_LIMIT + 4) /* refire counter cap */
+#define PCH_PIC_STORM_COOLDOWN SBT_1MS /* re-arm interval while throttled */
+
+#define FDT_IRQ_TYPE_LEVEL_HIGH 4
+
+struct pch_pic_irqsrc {
+ struct intr_irqsrc isrc;
+ u_int irq; /* PCH-PIC input index */
+ int storm_refire; /* fast re-fire counter */
+ bool storm_throttled; /* delayed-unmask mode */
+ sbintime_t storm_last_post; /* last post_ithread time */
+ struct callout storm_unmask_co; /* deferred re-arm while
+ * throttled */
+};
+
+struct pch_pic_softc {
+ device_t dev;
+ struct resource *mem_res;
+ int mem_rid;
+ u_int vec_base; /* EIOINTC vector base */
+ u_int vec_count;
+ u_int gsi_base; /* ACPI GSI base
+ * (LOONGSON_PCH_IRQ_BASE) */
+ bool is_acpi; /* ACPI/direct vs FDT
+ * instantiation */
+ intptr_t xref; /* this PIC's xref (FDT or
+ * ACPI_INTR_XREF) */
+ struct mtx mtx;
+ struct pch_pic_irqsrc isrcs[PIC_COUNT];
+};
+
+static device_t pch_pic_dev; /* single-instance guard */
+
+/*
+ * Look up the PCH-PIC's xref (0 until the pch_pic driver has attached).
+ * Used by nexus for PCI INTx routing.
+ */
+intptr_t
+pch_pic_get_xref(void)
+{
+ device_t d;
+ struct pch_pic_softc *sc;
+
+ d = devclass_get_device(devclass_find("pch_pic"), 0);
+ if (d == NULL)
+ return (0);
+ sc = device_get_softc(d);
+ return (sc->xref);
+}
+
+static inline uint32_t
+pch_pic_read(struct pch_pic_softc *sc, bus_size_t off)
+{
+
+ return (bus_read_4(sc->mem_res, off));
+}
+
+static inline void
+pch_pic_write(struct pch_pic_softc *sc, bus_size_t off, uint32_t val)
+{
+
+ bus_write_4(sc->mem_res, off, val);
+}
+
+static void
+pch_pic_bit_set(struct pch_pic_softc *sc, int offset, int bit)
+{
+ uint32_t val;
+ bus_size_t reg;
+
+ reg = offset + (bit / PIC_COUNT_PER_REG) * 4;
+ val = pch_pic_read(sc, reg);
+ val |= (1U << (bit % PIC_COUNT_PER_REG));
+ pch_pic_write(sc, reg, val);
+}
+
+static void
+pch_pic_bit_clr(struct pch_pic_softc *sc, int offset, int bit)
+{
+ uint32_t val;
+ bus_size_t reg;
+
+ reg = offset + (bit / PIC_COUNT_PER_REG) * 4;
+ val = pch_pic_read(sc, reg);
+ val &= ~(1U << (bit % PIC_COUNT_PER_REG));
+ pch_pic_write(sc, reg, val);
+}
+
+static void
+pch_pic_reset(struct pch_pic_softc *sc)
+{
+ int i;
+
+ for (i = 0; i < (int)sc->vec_count; i++) {
+ /* Write EIOINTC vector number to HTVEC register */
+ bus_write_1(sc->mem_res, PCH_INT_HTVEC(i),
+ sc->vec_base + i);
+ /* Route to HT0 */
+ bus_write_1(sc->mem_res, PCH_INT_ROUTE(i), 1);
+ }
+
+ for (i = 0; i < 2; i++) {
+ /*
+ * Clear all MASK bits (0 = enabled). The MASK register
+ * gates interrupt forwarding; since the ISRC returned by
+ * pic_map_intr is owned by EIOINTC, PIC_ENABLE_INTR goes
+ * to EIOINTC and never reaches PCH-PIC to clear MASK.
+ * We unmask everything at init and rely on EIOINTC's
+ * per-vector enable/disable for interrupt control.
+ */
+ pch_pic_write(sc, PCH_PIC_MASK + 4 * i, 0);
+ pch_pic_write(sc, PCH_PIC_CLR + 4 * i, 0xFFFFFFFF);
+ /*
+ * Distribution mode = fixed (7A1000 manual §5.4: AUTO_CTRL
+ * selects fixed/rotating/idle/busy distribution among the two
+ * interrupt outputs -- NOT "auto-bounce"; fixed is
+ * recommended and must not be changed mid-run). Routing
+ * follows ROUTE_ENTRY.
+ */
+ pch_pic_write(sc, PCH_PIC_AUTO0 + 4 * i, 0);
+ pch_pic_write(sc, PCH_PIC_AUTO1 + 4 * i, 0);
+ /* Enable HTMSI transformer for all */
+ pch_pic_write(sc, PCH_PIC_HTMSI_EN + 4 * i, 0xFFFFFFFF);
+ /*
+ * PCI legacy INTx is active-low, level-triggered (PCI spec).
+ * Default all inputs to level/active-low here.
+ */
+ pch_pic_write(sc, PCH_PIC_EDGE + 4 * i, 0);
+ pch_pic_write(sc, PCH_PIC_POL + 4 * i, 0xFFFFFFFF);
+ }
+}
+
+static int
+pch_pic_probe(device_t dev)
+{
+
+ /*
+ * Two paths. FDT node: used only when ACPI is absent (FDT fallback),
+ * otherwise ofwbus duplicates the nexus (ACPI) instance. nexus/ACPI
+ * direct child (no OFW node): match by name "pch_pic".
+ */
+ if (ofw_bus_get_node(dev) != -1) {
+ if (loongarch_efi_acpi_rsdp() != 0)
+ return (ENXIO);
+ if (!ofw_bus_is_compatible(dev, "loongson,pch-pic-1.0"))
+ return (ENXIO);
+ } else {
+ if (strcmp(device_get_name(dev), "pch_pic") != 0)
+ return (ENXIO);
+ }
+
+ device_set_desc(dev, "LoongArch PCH PIC");
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+pch_pic_attach(device_t dev)
+{
+ struct pch_pic_softc *sc;
+ phandle_t node;
+ pcell_t vec_base;
+ intptr_t xref;
+ int i, error;
+
+ /*
+ * Single instance: the FDT and direct/nexus paths must not both
+ * attach.
+ */
+ if (pch_pic_dev != NULL)
+ return (ENXIO);
+ pch_pic_dev = dev;
+
+ sc = device_get_softc(dev);
+ sc->dev = dev;
+ sc->mem_rid = 0;
+ mtx_init(&sc->mtx, "pch_pic", NULL, MTX_SPIN);
+
+#ifdef DEV_ACPI
+ sc->is_acpi = (loongarch_efi_acpi_rsdp() != 0);
+#else
+ sc->is_acpi = false;
+#endif
+
+ if (sc->is_acpi) {
+#ifdef DEV_ACPI
+ vm_paddr_t addr;
+ u_int size, gsi_base;
+
+ /*
+ * Get the PCH-PIC parameters from the MADT BIO_PIC subtable:
+ * MMIO register base, window size and GSI base. The root
+ * PCH-PIC starts at EIOINTC vector 0.
+ */
+ if (!loongarch_acpi_pch_pic_info(&addr, &size, &gsi_base)) {
+ device_printf(dev, "no MADT BIO_PIC entry\n");
+ return (ENXIO);
+ }
+ sc->gsi_base = gsi_base;
+ sc->vec_base = 0;
+ if (bus_set_resource(dev, SYS_RES_MEMORY, 0, addr, size) != 0) {
+ device_printf(dev, "cannot set MMIO resource\n");
+ return (ENXIO);
+ }
+#endif
+ } else {
+ node = ofw_bus_get_node(dev);
+
+ /* Read pic-base-vec from FDT */
+ if (OF_getencprop(node, "loongson,pic-base-vec", &vec_base,
+ sizeof(vec_base)) < 0) {
+ device_printf(dev, "missing loongson,pic-base-vec\n");
+ return (ENXIO);
+ }
+ sc->vec_base = (u_int)vec_base;
+ sc->gsi_base = 0;
+ }
+
+ sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+ &sc->mem_rid, RF_ACTIVE);
+ if (sc->mem_res == NULL) {
+ device_printf(dev, "cannot allocate memory resource\n");
+ return (ENXIO);
+ }
+
+ /* Determine number of interrupt inputs from hardware version */
+ sc->vec_count = ((pch_pic_read(sc, 4) >> 16) & 0xff) + 1;
+ if (sc->vec_count == 0 || sc->vec_count > PIC_COUNT)
+ sc->vec_count = PIC_COUNT;
+
+ /* Initialize routing and mask all interrupts */
+ pch_pic_reset(sc);
+
+ /*
+ * Register our own ISRCs (one per input). pic_map_intr returns these
+ * to INTRNG and associates each with its EIOINTC vector (eioic child
+ * pointer), so eioic_intr dispatches the pch_pic ISRC -- and thus our
+ * pre/post_ithread actually run to ack/re-arm (CLR) per interrupt.
+ */
+ for (i = 0; i < sc->vec_count; i++) {
+ sc->isrcs[i].irq = i;
+ sc->isrcs[i].storm_refire = 0;
+ sc->isrcs[i].storm_throttled = false;
+ sc->isrcs[i].storm_last_post = 0;
+ callout_init(&sc->isrcs[i].storm_unmask_co, 1);
+ error = intr_isrc_register(&sc->isrcs[i].isrc, dev, 0,
+ "%s,%u", device_get_nameunit(dev), i);
+ if (error != 0) {
+ device_printf(dev, "cannot register isrc %d\n", i);
+ goto fail;
+ }
+ }
+
+ /*
+ * Register as a PIC. On ACPI, own the GSI space via ACPI_INTR_XREF;
+ * on FDT, use the device-tree xref.
+ */
+ xref = sc->is_acpi ? ACPI_INTR_XREF :
+ OF_xref_from_node(ofw_bus_get_node(dev));
+ sc->xref = xref;
+ if (intr_pic_register(dev, xref) == NULL) {
+ device_printf(dev, "cannot register PIC\n");
+ error = ENXIO;
+ goto fail;
+ }
+
+ device_printf(dev, "vec_base=%u vec_count=%u gsi_base=%u%s\n",
+ sc->vec_base, sc->vec_count, sc->gsi_base,
+ sc->is_acpi ? " (ACPI)" : "");
+
+ return (0);
+
+fail:
+ bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res);
+ return (error);
+}
+
+/* Map an interrupt (FDT or ACPI) to the EIOINTC ISRC for its vector. */
+static int
+pch_pic_map_intr(device_t dev, struct intr_map_data *data,
+ struct intr_irqsrc **isrcp)
+{
+ struct pch_pic_softc *sc;
+ struct eioic_softc *eioic;
+ u_int irq, eioic_vec;
+
+ sc = device_get_softc(dev);
+
+ switch (data->type) {
+#ifdef DEV_ACPI
+ case INTR_MAP_DATA_ACPI: {
+ struct intr_map_data_acpi *daa;
+
+ daa = (struct intr_map_data_acpi *)data;
+ /* GSI space: gsi_base .. gsi_base + vec_count - 1 */
+ if (daa->irq < sc->gsi_base ||
+ daa->irq >= sc->gsi_base + sc->vec_count)
+ return (EINVAL);
+ irq = daa->irq - sc->gsi_base;
+ break;
+ }
+#endif
+ case INTR_MAP_DATA_FDT: {
+ struct intr_map_data_fdt *daf;
+
+ daf = (struct intr_map_data_fdt *)data;
+ if (daf->ncells != 2)
+ return (EINVAL);
+ irq = daf->cells[0];
+ if (irq >= sc->vec_count)
+ return (EINVAL);
+ break;
+ }
+ default:
+ return (ENOTSUP);
+ }
+
+ eioic_vec = sc->vec_base + irq;
+
+ /*
+ * Return our own ISRC for this input, and associate it with the
+ * EIOINTC vector so eioic_intr dispatches us (the child) instead of
+ * the EIOINTC's own ISRC. This makes our pre/post_ithread run.
+ */
+ eioic = eioic_get_softc();
+ if (eioic == NULL)
+ return (ENXIO);
+ if (eioic_vec >= eioic->vec_count)
+ return (EINVAL);
+ eioic->isrcs[eioic_vec].child = &sc->isrcs[irq].isrc;
+
+ *isrcp = &sc->isrcs[irq].isrc;
+ return (0);
+}
+
+static void
+pch_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct pch_pic_softc *sc = device_get_softc(dev);
+ struct pch_pic_irqsrc *pisrc = (struct pch_pic_irqsrc *)isrc;
+ u_int irq = pisrc->irq;
+
+ if (irq >= sc->vec_count)
+ return;
+
+ mtx_lock_spin(&sc->mtx);
+ /* Ack/re-arm + unmask so the source can fire. */
+ pch_pic_bit_set(sc, PCH_PIC_CLR, irq);
+ pch_pic_bit_clr(sc, PCH_PIC_MASK, irq);
+ mtx_unlock_spin(&sc->mtx);
+}
+
+static void
+pch_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct pch_pic_softc *sc = device_get_softc(dev);
+ struct pch_pic_irqsrc *pisrc = (struct pch_pic_irqsrc *)isrc;
+ u_int irq = pisrc->irq;
+
+ if (irq >= sc->vec_count)
+ return;
+
+ mtx_lock_spin(&sc->mtx);
+ pch_pic_bit_set(sc, PCH_PIC_MASK, irq);
+ mtx_unlock_spin(&sc->mtx);
+}
+
+static int
+pch_pic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
+ struct resource *res, struct intr_map_data *data)
+{
+ struct pch_pic_softc *sc = device_get_softc(dev);
+ struct pch_pic_irqsrc *pisrc = (struct pch_pic_irqsrc *)isrc;
+ bool edge = false, pol_low = false;
+ u_int irq;
+
+ if (data == NULL)
+ return (0);
+
+ irq = pisrc->irq;
+ if (irq >= sc->vec_count)
+ return (EINVAL);
+
+ switch (data->type) {
+#ifdef DEV_ACPI
+ case INTR_MAP_DATA_ACPI:
+ /*
+ * LS7A PCH-PIC INTx: level-triggered, active-low.
+ * Per-source only -- the global reset leaves unused inputs
+ * at the default level/high so floating lines don't fire.
+ */
+ edge = true;
+ pol_low = true;
+ break;
+#endif
+ case INTR_MAP_DATA_FDT: {
+ struct intr_map_data_fdt *daf;
+
+ daf = (struct intr_map_data_fdt *)data;
+ if (daf->ncells < 2)
+ return (0);
+ switch (daf->cells[1] & 0xf) {
+ case 1: /* EDGE_RISING */
+ edge = true; pol_low = false; break;
+ case 2: /* EDGE_FALLING */
+ edge = true; pol_low = true; break;
+ case 4: /* LEVEL_HIGH */
+ edge = false; pol_low = false; break;
+ case 8: /* LEVEL_LOW */
+ edge = false; pol_low = true; break;
+ default:
+ return (0);
+ }
+ break;
+ }
+ default:
+ return (0);
+ }
+
+ mtx_lock_spin(&sc->mtx);
+
+ /* Configure edge/level and polarity for this input. */
+ if (edge)
+ pch_pic_bit_set(sc, PCH_PIC_EDGE, irq);
+ else
+ pch_pic_bit_clr(sc, PCH_PIC_EDGE, irq);
+ if (pol_low)
+ pch_pic_bit_set(sc, PCH_PIC_POL, irq);
+ else
+ pch_pic_bit_clr(sc, PCH_PIC_POL, irq);
+
+ mtx_unlock_spin(&sc->mtx);
+ return (0);
+}
+
+static void
+pch_pic_deferred_unmask(void *arg)
+{
+ struct pch_pic_irqsrc *pisrc = arg;
+ struct pch_pic_softc *sc = device_get_softc(pch_pic_dev);
+
+ mtx_lock_spin(&sc->mtx);
+ pch_pic_bit_clr(sc, PCH_PIC_MASK, pisrc->irq);
+ mtx_unlock_spin(&sc->mtx);
+}
+
+static void
+pch_pic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct pch_pic_softc *sc = device_get_softc(dev);
+ struct pch_pic_irqsrc *pisrc = (struct pch_pic_irqsrc *)isrc;
+ sbintime_t now;
+
+ now = getsbinuptime();
+ mtx_lock_spin(&sc->mtx);
+ if (now - pisrc->storm_last_post < PCH_PIC_STORM_THRESH) {
+ /* Re-fired within THRESH of the last unmask: the line is still
+ * asserted. Charge it toward throttling. */
+ if (pisrc->storm_refire < PCH_PIC_STORM_SAT)
+ pisrc->storm_refire++;
+ } else if (pisrc->storm_refire > 0) {
+ /* Normal gap: a previously-stuck line may be recovering. */
+ pisrc->storm_refire--;
+ }
+ pisrc->storm_throttled = (pisrc->storm_refire >= PCH_PIC_STORM_LIMIT);
+ /* Ack/re-arm (CLR) while masked. */
+ pch_pic_bit_set(sc, PCH_PIC_MASK, pisrc->irq);
+ pch_pic_bit_set(sc, PCH_PIC_CLR, pisrc->irq);
+ mtx_unlock_spin(&sc->mtx);
+}
+
+static void
+pch_pic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct pch_pic_softc *sc = device_get_softc(dev);
+ struct pch_pic_irqsrc *pisrc = (struct pch_pic_irqsrc *)isrc;
+
+ mtx_lock_spin(&sc->mtx);
+ pisrc->storm_last_post = getsbinuptime();
+ if (pisrc->storm_throttled) {
+ /* Leave masked; re-arm from a callout to collapse the
+ * unmask->refire storm while still letting real interrupts
+ * (e.g. OHCI WDH) through every PCH_PIC_STORM_COOLDOWN. */
+ callout_reset_sbt(&pisrc->storm_unmask_co,
+ PCH_PIC_STORM_COOLDOWN, 0, pch_pic_deferred_unmask, pisrc,
+ C_DIRECT_EXEC);
+ } else {
+ /* Unmask immediately: normal input. */
+ pch_pic_bit_clr(sc, PCH_PIC_MASK, pisrc->irq);
+ }
+ mtx_unlock_spin(&sc->mtx);
+}
+
+static void
+pch_pic_post_filter(device_t dev, struct intr_irqsrc *isrc)
+{
+
+ pch_pic_post_ithread(dev, isrc);
+}
+
+static device_method_t pch_pic_methods[] = {
+ DEVMETHOD(device_probe, pch_pic_probe),
+ DEVMETHOD(device_attach, pch_pic_attach),
+
+ DEVMETHOD(pic_disable_intr, pch_pic_disable_intr),
+ DEVMETHOD(pic_enable_intr, pch_pic_enable_intr),
+ DEVMETHOD(pic_map_intr, pch_pic_map_intr),
+ DEVMETHOD(pic_setup_intr, pch_pic_setup_intr),
+ DEVMETHOD(pic_pre_ithread, pch_pic_pre_ithread),
+ DEVMETHOD(pic_post_ithread, pch_pic_post_ithread),
+ DEVMETHOD(pic_post_filter, pch_pic_post_filter),
+
+ DEVMETHOD_END
+};
+
+static driver_t pch_pic_driver = {
+ "pch_pic",
+ pch_pic_methods,
+ sizeof(struct pch_pic_softc),
+};
+
+EARLY_DRIVER_MODULE(pch_pic, simplebus, pch_pic_driver, 0, 0,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
+/* Also register on nexus for direct instantiation on ACPI systems. */
+EARLY_DRIVER_MODULE(pch_pic, nexus, pch_pic_driver, 0, 0,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
diff --git a/sys/loongarch/loongarch/pmap.c b/sys/loongarch/loongarch/pmap.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/pmap.c
@@ -0,0 +1,6045 @@
+/*-
+ * SPDX-License-Identifier: BSD-4-Clause
+ *
+ * Copyright (c) 1991 Regents of the University of California.
+ * All rights reserved.
+ * Copyright (c) 1994 John S. Dyson
+ * All rights reserved.
+ * Copyright (c) 1994 David Greenman
+ * All rights reserved.
+ * Copyright (c) 2003 Peter Wemm
+ * All rights reserved.
+ * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
+ * All rights reserved.
+ * Copyright (c) 2014 Andrew Turner
+ * All rights reserved.
+ * Copyright (c) 2014 The FreeBSD Foundation
+ * All rights reserved.
+ * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * the Systems Programming Group of the University of Utah Computer
+ * Science Department and William Jolitz of UUNET Technologies Inc.
+ *
+ * Portions of this software were developed by Andrew Turner under
+ * sponsorship from The FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by the University of
+ * California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 2003 Networks Associates Technology, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project by Jake Burkholder,
+ * Safeport Network Services, and Network Associates Laboratories, the
+ * Security Research Division of Network Associates, Inc. under
+ * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
+ * CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+/*
+ * Manages physical address maps.
+ *
+ * Since the information managed by this module is
+ * also stored by the logical address mapping module,
+ * this module may throw away valid virtual-to-physical
+ * mappings at almost any time. However, invalidations
+ * of virtual-to-physical mappings must be done as
+ * requested.
+ *
+ * In order to cope with hardware architectures which
+ * make virtual-to-physical map invalidates expensive,
+ * this module may delay invalidate or reduced protection
+ * operations until such time as they are actually
+ * necessary. This module is given full information as
+ * to which processors are currently using which maps,
+ * and to when physical maps must be made correct.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/cpuset.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mman.h>
+#include <sys/msgbuf.h>
+#include <sys/mutex.h>
+#include <sys/physmem.h>
+#include <sys/proc.h>
+#include <sys/rwlock.h>
+#include <sys/sbuf.h>
+#include <sys/sx.h>
+#include <sys/vmem.h>
+#include <sys/vmmeter.h>
+#include <sys/sched.h>
+#include <sys/sysctl.h>
+#include <sys/smp.h>
+#include <machine/smp.h>
+#include <machine/tlb.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_pager.h>
+#include <vm/vm_phys.h>
+#include <vm/vm_radix.h>
+#include <vm/vm_reserv.h>
+#include <vm/vm_dumpset.h>
+#include <vm/uma.h>
+
+#include <machine/machdep.h>
+#include <machine/md_var.h>
+#include <machine/pcb.h>
+#include <machine/pte.h>
+#include <machine/watch.h>
+#include <machine/tlb.h>
+
+/*
+ * Boundary values for the page table page index space:
+ *
+ * L3 pages: [0, NUL2E)
+ * L2 pages: [NUL2E, NUL2E + NUL1E)
+ * L1 pages: [NUL2E + NUL1E, NUL2E + NUL1E + NUL0E)
+ * L0 pages: [NUL2E + NUL1E + NUL0E, ...)
+ */
+#define NUL0E L0_ENTRIES
+#define NUL1E (Ln_ENTRIES * NUL0E)
+#define NUL2E (Ln_ENTRIES * NUL1E)
+
+#ifdef PV_STATS
+#define PV_STAT(x) do { x ; } while (0)
+#define __pv_stat_used
+#else
+#define PV_STAT(x) do { } while (0)
+#define __pv_stat_used __unused
+#endif
+
+#define pmap_l1_pindex(v) (NUL2E + ((v) >> L1_SHIFT))
+#define pmap_l2_pindex(v) ((v) >> L2_SHIFT)
+#define pa_index(pa) ((pa) >> L2_SHIFT)
+#define pa_to_pvh(pa) (&pv_table[pa_index(pa)])
+
+#define NPV_LIST_LOCKS MAXCPU
+
+#define PHYS_TO_PV_LIST_LOCK(pa) \
+ (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS])
+
+#define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \
+ struct rwlock **_lockp = (lockp); \
+ struct rwlock *_new_lock; \
+ \
+ _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \
+ if (_new_lock != *_lockp) { \
+ if (*_lockp != NULL) \
+ rw_wunlock(*_lockp); \
+ *_lockp = _new_lock; \
+ rw_wlock(*_lockp); \
+ } \
+} while (0)
+
+#define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
+
+#define RELEASE_PV_LIST_LOCK(lockp) do { \
+ struct rwlock **_lockp = (lockp); \
+ \
+ if (*_lockp != NULL) { \
+ rw_wunlock(*_lockp); \
+ *_lockp = NULL; \
+ } \
+} while (0)
+
+#define VM_PAGE_TO_PV_LIST_LOCK(m) \
+ PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
+
+static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "VM/pmap parameters");
+
+enum pmap_mode __read_frequently pmap_mode = PMAP_MODE_LA48;
+SYSCTL_INT(_vm_pmap, OID_AUTO, mode, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+ &pmap_mode, 0,
+ "translation mode = LA48");
+
+struct pmap kernel_pmap_store;
+vm_offset_t kernel_load_phys;
+
+vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
+vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
+vm_offset_t kernel_vm_end = 0;
+
+vm_paddr_t dmap_phys_base; /* The start of the dmap region */
+vm_paddr_t dmap_phys_max; /* The limit of the dmap region */
+vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */
+
+static int pmap_growkernel_panic = 0;
+SYSCTL_INT(_vm_pmap, OID_AUTO, growkernel_panic, CTLFLAG_RDTUN,
+ &pmap_growkernel_panic, 0,
+ "panic on failure to allocate kernel page table page");
+
+/* FIXME */
+#if 0
+/* This code assumes all L1 DMAP entries will be used */
+CTASSERT((DMAP_MIN_ADDRESS & ~L1_OFFSET) == DMAP_MIN_ADDRESS);
+CTASSERT((DMAP_MAX_ADDRESS & ~L1_OFFSET) == DMAP_MAX_ADDRESS);
+
+/*
+ * This code assumes that the early DEVMAP is L2_SIZE aligned and is fully
+ * contained within a single L2 entry. The early DTB is mapped immediately
+ * before the devmap L2 entry.
+ */
+CTASSERT((PMAP_MAPDEV_EARLY_SIZE & L2_OFFSET) == 0);
+CTASSERT((VM_EARLY_DTB_ADDRESS & L2_OFFSET) == 0);
+CTASSERT(VM_EARLY_DTB_ADDRESS <
+ (VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE));
+#endif
+
+static struct rwlock_padalign pvh_global_lock;
+
+static int __read_frequently superpages_enabled = 1;
+SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled,
+ CTLFLAG_RDTUN, &superpages_enabled, 0,
+ "Enable support for transparent superpages");
+
+static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "2MB page mapping counters");
+
+static u_long pmap_l2_demotions;
+SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, demotions, CTLFLAG_RD,
+ &pmap_l2_demotions, 0,
+ "2MB page demotions");
+
+static u_long pmap_l2_mappings;
+SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, mappings, CTLFLAG_RD,
+ &pmap_l2_mappings, 0,
+ "2MB page mappings");
+
+/*
+ * Threshold (in pages) above which pmap_remove stops issuing one
+ * pmap_invalidate_range shootdown per contiguous run and instead defers
+ * them all to a single pmap_invalidate_all at the end -- trading one
+ * full-pmap flush for N range IPIs on a large munmap. 0 disables
+ * coalescing (always per-range).
+ */
+int pmap_tlb_ipi_threshold = 32;
+SYSCTL_INT(_vm_pmap, OID_AUTO, tlb_ipi_threshold, CTLFLAG_RWTUN,
+ &pmap_tlb_ipi_threshold, 0,
+ "Page count above which pmap_remove coalesces TLB invalidations");
+
+static u_long pmap_l2_p_failures;
+SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, p_failures, CTLFLAG_RD,
+ &pmap_l2_p_failures, 0,
+ "2MB page promotion failures");
+
+static u_long pmap_l2_promotions;
+SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, promotions, CTLFLAG_RD,
+ &pmap_l2_promotions, 0,
+ "2MB page promotions");
+
+static SYSCTL_NODE(_vm_pmap, OID_AUTO, l1, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "L1 (1GB) page mapping counters");
+
+static COUNTER_U64_DEFINE_EARLY(pmap_l1_demotions);
+SYSCTL_COUNTER_U64(_vm_pmap_l1, OID_AUTO, demotions, CTLFLAG_RD,
+ &pmap_l1_demotions, "L1 (1GB) page demotions");
+
+/*
+ * Data for the pv entry allocation mechanism
+ */
+static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
+static struct mtx pv_chunks_mutex;
+static struct rwlock pv_list_locks[NPV_LIST_LOCKS];
+static struct md_page *pv_table;
+static struct md_page pv_dummy;
+
+/*
+ * Internal flags for pmap_enter()'s helper functions.
+ */
+#define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
+#define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
+
+static void free_pv_chunk(struct pv_chunk *pc);
+static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
+static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);
+static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
+static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap,
+ vm_offset_t va);
+static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
+ vm_offset_t va);
+static bool pmap_demote_l1(pmap_t pmap, pd_entry_t *l1, vm_offset_t va);
+static bool pmap_demote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va);
+static bool pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2,
+ vm_offset_t va, struct rwlock **lockp);
+static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2,
+ u_int flags, vm_page_t m, struct rwlock **lockp);
+static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
+ vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
+static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva,
+ pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp,
+ bool invalidate);
+static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
+ vm_page_t m, struct rwlock **lockp);
+
+static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,
+ struct rwlock **lockp);
+
+static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ struct spglist *free);
+static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);
+
+static int pmap_change_attr_locked(void *va, vm_size_t size, int mode);
+
+static pd_entry_t invalid_pgd_table; /* fwd decl: defined and set below */
+
+#ifdef INVARIANTS
+/*
+ * pmap_inv_active gates the bare-phys wired-child write-watch. It is set at
+ * the end of pmap_bootstrap -- before that, the kernel page tables are built
+ * by locore using pages that are not tracked as wired vm_pages, so the check
+ * would false-fire.
+ */
+static bool pmap_inv_active;
+
+/*
+ * Two invariants, checked at every PTE store:
+ *
+ * 1. A 0 store must target a leaf entry, never a directory entry. Directory
+ * entries are emptied to the sentinel, not 0. Detect a directory page by
+ * the sentinel in its empty slots.
+ *
+ * 2. A bare-phys value (page-aligned, non-zero, non-sentinel, no V or HUGE
+ * bits) stored as a directory child pointer must point at a wired PT page.
+ * A stray store that writes a data page's phys into a directory slot --
+ * the corrupt-directory-entry bug -- is caught here at the WRITE, with the
+ * writer's backtrace.
+ */
+static void
+pmap_pte_sync_invariant(volatile pt_entry_t *pte)
+{
+ pd_entry_t val, *pg;
+ int j, nsent;
+
+ val = *(volatile pt_entry_t *)pte;
+
+ if (val == 0) {
+ /* Invariant 1: 0 must not be stored into a directory page. */
+ pg = (pd_entry_t *)((uintptr_t)pte & ~PAGE_MASK);
+ nsent = 0;
+ for (j = 0; j < 16; j++)
+ if (*(volatile pd_entry_t *)&pg[j] == invalid_pgd_table)
+ nsent++;
+ KASSERT(nsent <= 8,
+ ("pmap: 0 stored into directory page %#lx slot %lu",
+ (unsigned long)(uintptr_t)pg,
+ (unsigned long)(((uintptr_t)pte & PAGE_MASK) / PTE_SIZE)));
+ return;
+ }
+
+ /*
+ * Invariant 2: a bare-phys directory child must point at a wired PT
+ * page. Filter: page-aligned (bits [11:0] = 0), no NX/NR high bits,
+ * non-zero, non-sentinel. This uniquely identifies a directory-child
+ * bare-phys pointer -- leaf PTEs always have at least one low flag bit
+ * (V, D, PLV, G, MAT) or a high bit (NX, NR) set.
+ */
+ if (pmap_inv_active && invalid_pgd_table != 0 &&
+ val != invalid_pgd_table &&
+ val != 0 && (val & (0xfffUL | PTE_NX | PTE_NR)) == 0) {
+ vm_paddr_t cphys = ((val & ~PTE_HI_MASK) >> PAGE_PFN_SHIFT) <<
+ PAGE_SHIFT;
+ vm_page_t m;
+
+ if (cphys == 0)
+ return;
+ m = PHYS_TO_VM_PAGE(cphys);
+ KASSERT(vm_page_wired(m),
+ ("pmap: bare phys %#lx stored as directory child -- "
+ "not wired (pindex=%#lx ref_count=%d) -- corrupt dir "
+ "entry?",
+ (unsigned long)val, (unsigned long)m->pindex,
+ m->ref_count));
+ }
+}
+#endif
+
+static __inline void
+pmap_pte_sync(volatile pt_entry_t *pte)
+{
+
+ c_sync();
+#ifdef INVARIANTS
+ pmap_pte_sync_invariant(pte);
+#endif
+}
+
+#define pmap_clear(pte) pmap_store(pte, 0)
+#define pmap_clear_bits(pte, bits) do { \
+ atomic_clear_64(pte, bits); \
+ pmap_pte_sync(pte); \
+ } while (0)
+#define pmap_load_store(pte, entry) ({ \
+ pt_entry_t _o; \
+ _o = atomic_swap_64(pte, entry); \
+ pmap_pte_sync(pte); \
+ _o; \
+ })
+#define pmap_load_clear(pte) pmap_load_store(pte, 0)
+#define pmap_load(pte) atomic_load_64(pte)
+#define pmap_store(pte, entry) do { \
+ atomic_store_64(pte, entry); \
+ pmap_pte_sync(pte); \
+ } while (0)
+#define pmap_store_bits(pte, bits) do { \
+ atomic_set_64(pte, bits); \
+ pmap_pte_sync(pte); \
+ } while (0)
+#define pmap_cmpset(pte, old, new) ({ \
+ bool _r; \
+ _r = atomic_fcmpset_long(pte, old, new); \
+ pmap_pte_sync(pte); \
+ _r; \
+ })
+
+/* will be initialized in pmap_bootstrap */
+static pd_entry_t invalid_pgd_table = 0;
+static pd_entry_t invalid_pud_table = 0;
+static pd_entry_t invalid_pmd_table = 0;
+
+static inline bool
+pmap_pgd_valid(pd_entry_t *pgd)
+{
+ return (pmap_load(pgd) != (pd_entry_t)invalid_pgd_table);
+}
+
+static inline bool
+pmap_pud_valid(pd_entry_t *pud)
+{
+ return (pmap_load(pud) != (pd_entry_t)invalid_pud_table);
+}
+
+static inline bool
+pmap_pmd_valid(pd_entry_t *pmd)
+{
+ return (pmap_load(pmd) != (pd_entry_t)invalid_pmd_table);
+}
+
+static inline bool
+pmap_pte_valid(pd_entry_t *pte)
+{
+ return ((pmap_load(pte) & PTE_P) != 0);
+}
+
+static inline void
+pmap_init_pd(pd_entry_t *pd)
+{
+ int i;
+
+ for (i = 0; i < Ln_ENTRIES; i++)
+ pmap_store(&pd[i], invalid_pmd_table);
+}
+
+static inline uint32_t
+pmap_current_asid(void)
+{
+ return (csr_read32(LOONGARCH_CSR_ASID) & CSR_ASID_ASID);
+}
+
+#define pmap_pgd_entry(x) (pmap_load(x))
+#define pmap_pud_entry(x) (pmap_load(x))
+#define pmap_pmd_entry(x) (pmap_load(x))
+#define pmap_pte_entry(x) (pmap_load(x))
+
+/********************/
+/* Inline functions */
+/********************/
+
+static inline void
+set_pgd_asid(uint32_t asid, vm_offset_t pgdl)
+{
+ __asm__ __volatile__(
+ "csrwr %[pgdl_val], %[pgdl_reg] \n\t"
+ "csrwr %[asid_val], %[asid_reg] \n\t"
+ : [asid_val] "+r" (asid), [pgdl_val] "+r" (pgdl)
+ : [asid_reg] "i" (LOONGARCH_CSR_ASID), [pgdl_reg] "i" (LOONGARCH_CSR_PGDL)
+ : "memory"
+ );
+}
+
+static __inline void
+pagecopy(void *s, void *d)
+{
+
+ memcpy(d, s, PAGE_SIZE);
+}
+
+static __inline void
+pagezero(void *p)
+{
+
+ bzero(p, PAGE_SIZE);
+}
+
+#define pmap_l0_index(va) (((va) >> L0_SHIFT) & L0_ADDR_MASK)
+#define pmap_l1_index(va) (((va) >> L1_SHIFT) & Ln_ADDR_MASK)
+#define pmap_l2_index(va) (((va) >> L2_SHIFT) & Ln_ADDR_MASK)
+#define pmap_l3_index(va) (((va) >> L3_SHIFT) & Ln_ADDR_MASK)
+
+#define PTE_TO_PHYS(pte) \
+ ((((pte) & ~PTE_HI_MASK) >> PAGE_PFN_SHIFT) << PAGE_SHIFT)
+
+#define L2PTE_TO_PHYS(l2) \
+ ((((l2) & ~PTE_HI_MASK) >> L2_SHIFT) << L2_SHIFT)
+
+#define PTE_TO_VM_PAGE(pte) PHYS_TO_VM_PAGE(PTE_TO_PHYS(pte))
+
+#ifdef INVARIANTS
+/*
+ * Every page-table page is allocated VM_ALLOC_WIRED. A directory entry that
+ * points at a recycled *data* page (corruption) leads the HW walker to read
+ * that page's contents as page-table entries -- the "V=1, impossible PFN"
+ * garbage TLB entries. Verify, on the SW walk, that a directory entry's child
+ * is actually a wired page-table page.
+ */
+static __inline void
+pmap_ptchild_assert_wired(vm_paddr_t phys, vm_offset_t va, const char *fn)
+{
+ vm_page_t m;
+
+ if (phys == 0)
+ return;
+ m = PHYS_TO_VM_PAGE(phys);
+ KASSERT(vm_page_wired(m),
+ ("%s: page-table page: %p child phys %#lx (va %#lx) not wired -- "
+ "pindex=%#lx ref_count=%d -- corrupt dir entry or wire-count bug?",
+ fn, m, (unsigned long)phys, va,
+ (unsigned long)m->pindex, m->ref_count));
+}
+#endif
+
+static __inline pd_entry_t *
+pmap_l0(pmap_t pmap, vm_offset_t va)
+{
+ KASSERT(VIRT_IS_VALID(va),
+ ("%s: malformed virtual address %#lx", __func__, va));
+ return (&pmap->pm_top[pmap_l0_index(va)]);
+}
+
+static __inline pd_entry_t *
+pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va)
+{
+ vm_paddr_t phys;
+ pd_entry_t *l1;
+
+ phys = PTE_TO_PHYS(pmap_pgd_entry(l0));
+#ifdef INVARIANTS
+ if (va < VM_MAXUSER_ADDRESS && pmap_pgd_entry(l0) != invalid_pgd_table)
+ pmap_ptchild_assert_wired(phys, va, __func__);
+#endif
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ return (&l1[pmap_l1_index(va)]);
+}
+
+static __inline pd_entry_t *
+pmap_l1(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *l0;
+
+ KASSERT(VIRT_IS_VALID(va),
+ ("%s: malformed virtual address %#lx", __func__, va));
+
+ l0 = pmap_l0(pmap, va);
+ if (!pmap_pgd_valid(l0))
+ return NULL;
+
+ if (pmap_pgd_entry(l0) & PTE_HUGE)
+ return NULL;
+
+ return (pmap_l0_to_l1(l0, va));
+}
+
+static __inline pd_entry_t *
+pmap_l1_to_l2(pd_entry_t *l1, vm_offset_t va)
+{
+ vm_paddr_t phys;
+ pd_entry_t *l2;
+
+ phys = PTE_TO_PHYS(pmap_pud_entry(l1));
+#ifdef INVARIANTS
+ if (va < VM_MAXUSER_ADDRESS && pmap_pud_entry(l1) != invalid_pgd_table)
+ pmap_ptchild_assert_wired(phys, va, __func__);
+#endif
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+
+ return (&l2[pmap_l2_index(va)]);
+}
+
+static __inline pd_entry_t *
+pmap_l2(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *l1;
+
+ l1 = pmap_l1(pmap, va);
+ if (l1 == NULL || !pmap_pud_valid(l1))
+ return (NULL);
+ if ((pmap_pud_entry(l1) & PTE_HUGE) != 0)
+ return (NULL);
+ return (pmap_l1_to_l2(l1, va));
+}
+
+static __inline pt_entry_t *
+pmap_l2_to_l3(pd_entry_t *l2, vm_offset_t va)
+{
+ vm_paddr_t phys;
+ pt_entry_t *l3;
+
+ phys = PTE_TO_PHYS(pmap_pmd_entry(l2));
+#ifdef INVARIANTS
+ if (va < VM_MAXUSER_ADDRESS && pmap_pmd_entry(l2) != invalid_pgd_table)
+ pmap_ptchild_assert_wired(phys, va, __func__);
+#endif
+ l3 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+
+ return (&l3[pmap_l3_index(va)]);
+}
+
+static __inline pt_entry_t *
+pmap_l3(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *l2;
+ pd_entry_t l2e;
+
+ l2 = pmap_l2(pmap, va);
+ if (l2 == NULL)
+ return (NULL);
+ l2e = pmap_pmd_entry(l2);
+ if (!pmap_pmd_valid(l2) || (l2e & PTE_HUGE) != 0)
+ return (NULL);
+
+ return (pmap_l2_to_l3(l2, va));
+}
+
+static __inline void
+pmap_resident_count_inc(pmap_t pmap, int count)
+{
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ pmap->pm_stats.resident_count += count;
+}
+
+static __inline void
+pmap_resident_count_dec(pmap_t pmap, int count)
+{
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ KASSERT(pmap->pm_stats.resident_count >= count,
+ ("pmap %p resident count underflow %ld %d", pmap,
+ pmap->pm_stats.resident_count, count));
+ pmap->pm_stats.resident_count -= count;
+}
+
+static pt_entry_t *
+pmap_early_page_idx(vm_offset_t l1pt, vm_offset_t va, u_int *l1_slot,
+ u_int *l2_slot)
+{
+ pt_entry_t *l2;
+ pd_entry_t *l1 __diagused;
+
+ l1 = (pd_entry_t *)l1pt;
+ *l1_slot = (va >> L1_SHIFT) & Ln_ADDR_MASK;
+
+ /* Check locore has used a table L1 map */
+ KASSERT(!(l1[*l1_slot] & PTE_HUGE),
+ ("Invalid bootstrap L1 table"));
+
+ /* Find the address of the L2 table */
+ l2 = (pt_entry_t *)l2_pt_va;
+ *l2_slot = pmap_l2_index(va);
+
+ return (l2);
+}
+
+static vm_paddr_t
+pmap_early_vtophys(vm_offset_t l1pt, vm_offset_t va)
+{
+ u_int l1_slot, l2_slot;
+ pt_entry_t *l2;
+ vm_paddr_t ret;
+
+ l2 = pmap_early_page_idx(l1pt, va, &l1_slot, &l2_slot);
+
+ /* Check locore has used L2 superpages */
+ KASSERT((l2[l2_slot] & PTE_HUGE),
+ ("Invalid bootstrap L2 table"));
+
+ /* L2 is superpages */
+ ret = L2PTE_TO_PHYS(l2[l2_slot]);
+ ret += (va & L2_OFFSET);
+
+ return (ret);
+}
+
+static void
+pmap_bootstrap_dmap(vm_paddr_t min_pa, vm_paddr_t max_pa)
+{
+ vm_offset_t va;
+ vm_paddr_t pa;
+ pd_entry_t *l1;
+ u_int l0_slot, l1_slot;
+ pt_entry_t entry;
+ pn_t pn;
+ vm_offset_t dmap_pt_phys;
+
+ pa = dmap_phys_base = min_pa & ~L1_OFFSET;
+ va = DMAP_MIN_ADDRESS;
+ l0_slot = pmap_l0_index(DMAP_MIN_ADDRESS);
+
+ /* Fixup the L0 entry. */
+ dmap_pt_phys = dmap_pt_va - KERNBASE + kernel_load_phys;
+ pmap_store(&kernel_pmap->pm_top[l0_slot], dmap_pt_phys);
+
+ /* DMAP page-table page. */
+ l1 = (pd_entry_t *)dmap_pt_va;
+ l1_slot = pmap_l1_index(DMAP_MIN_ADDRESS);
+
+ for (; va < DMAP_MAX_ADDRESS && pa < max_pa;
+ pa += L1_SIZE, va += L1_SIZE, l1_slot++) {
+ KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index"));
+
+ /* 1G page */
+ pn = pa >> PAGE_PFN_SHIFT;
+ entry = PMD_KERN_HUGE;
+ entry |= (pn << L3_SHIFT);
+ pmap_store(&l1[l1_slot], entry);
+ }
+
+ /* Set the upper limit of the DMAP region */
+ dmap_phys_max = pa;
+ dmap_max_addr = va;
+
+ invtlb_all(INVTLB_CURRENT_ALL);
+}
+
+static vm_offset_t
+pmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l3_start)
+{
+ vm_offset_t l3pt;
+ pt_entry_t entry;
+ pd_entry_t *l2;
+ vm_paddr_t pa;
+ u_int l2_slot;
+ pn_t pn;
+
+ KASSERT((va & L2_OFFSET) == 0, ("Invalid virtual address"));
+
+ l2 = pmap_l2(kernel_pmap, va);
+ l2 = (pd_entry_t *)((uintptr_t)l2 & ~(PAGE_SIZE - 1));
+ l2_slot = pmap_l2_index(va);
+ l3pt = l3_start;
+
+ printf("l2: %p, l2_slot: 0x%x\n", l2, l2_slot);
+
+ for (; va < VM_MAX_KERNEL_ADDRESS; l2_slot++, va += L2_SIZE) {
+ KASSERT(l2_slot < Ln_ENTRIES, ("Invalid L2 index"));
+
+ pa = pmap_early_vtophys(l1pt, l3pt);
+ pn = pa >> PAGE_PFN_SHIFT;
+ entry = pn << L3_SHIFT;
+ pmap_store(&l2[l2_slot], entry);
+ l3pt += PAGE_SIZE;
+ }
+
+ /* Clean the L3 page table */
+ memset((void *)l3_start, 0, l3pt - l3_start);
+
+ return (l3pt);
+}
+
+/*
+ * Map a range of physical addresses into kernel virtual address space,
+ * assigning the given attributes. Used by subr_devmap for static device
+ * mappings. Currently unused on loongarch (no static devmap table is
+ * registered), but must exist to satisfy the link.
+ */
+void
+pmap_preboot_map_attr(vm_paddr_t pa, vm_offset_t va, vm_size_t size,
+ vm_prot_t prot, vm_memattr_t attr)
+{
+
+ panic("%s: not implemented", __func__);
+}
+
+/*
+ * Bootstrap the system enough to run with virtual memory.
+ */
+void
+pmap_bootstrap(vm_offset_t pgd, vm_paddr_t kernstart, vm_size_t kernlen)
+{
+ vm_paddr_t physmap[PHYS_AVAIL_ENTRIES];
+ vm_offset_t dpcpu, freemempos, msgbufpv;
+ vm_paddr_t max_pa, min_pa;
+ u_int l1_slot, l2_slot;
+ u_int physmap_idx;
+ int i;
+ vm_offset_t l1, va;
+ pd_entry_t *l0_table, *l1_table, *l2_table, *sentinel_page;
+ int l2_end;
+
+ printf("pmap_bootstrap %lx %lx %lx\n", pgd, kernstart, kernlen);
+
+ kernel_load_phys = kernstart;
+
+ /* Set this early so we can use the pagetable walking functions */
+ /* pgd is physical address here !! */
+ kernel_pmap_store.pm_top = (pd_entry_t *)(KERNBASE + pgd - kernstart);
+
+ /* point to invalid L0 */
+ invalid_pgd_table = pgd - PAGE_SIZE;
+ invalid_pud_table = invalid_pgd_table;
+ invalid_pmd_table = invalid_pgd_table;
+
+ /*
+ * Fill the sentinel page (at invalid_pgd_table, i.e. pgd - PAGE_SIZE)
+ * with the sentinel value itself. The LoongArch HW walker follows
+ * bare-phys directory entries unconditionally -- there is no V bit on
+ * directory entries -- so if the sentinel page is all-zero, the walker
+ * follows 0x00 entries to phys 0 and reads its contents as PTEs,
+ * producing garbage TLB entries (V=1, bogus PFN). By filling the
+ * sentinel page with invalid_pgd_table (page-aligned, bit 0 = 0), the
+ * walker descends through this page at each remaining level and, when
+ * it reaches the leaf PTE level, reads the sentinel value as a PTE with
+ * V=0 -> TLB miss -> clean page fault.
+ */
+ sentinel_page = (pd_entry_t *)
+ (KERNBASE + invalid_pgd_table - kernstart);
+ for (i = 0; i < Ln_ENTRIES; i++)
+ pmap_store(&sentinel_page[i], invalid_pgd_table);
+
+ kernel_pmap_store.pm_pgdl = invalid_pgd_table;
+ kernel_pmap_store.pm_pgdh = pgd;
+
+ /* reinitialize kernel page table */
+ l0_table = (pd_entry_t*)kernel_pmap_store.pm_top;
+ for (i = 0; i < L0_ENTRIES; i++) {
+ if (l0_table[i] == 0)
+ pmap_store(&l0_table[i], invalid_pgd_table);
+ }
+
+ l1 = (vm_offset_t)l1_pt_va;
+ l1_table = (pd_entry_t*)l1;
+ l2_table = pmap_early_page_idx(l1, KERNBASE, &l1_slot, &l2_slot);
+ for (i = l1_slot + 1; i < Ln_ENTRIES; i++) {
+ if (l1_table[i] == 0)
+ pmap_store(&l1_table[i], invalid_pud_table);
+ }
+
+ l2_end = l2_slot + ((kernlen + L2_SIZE - 1) / L2_SIZE);
+
+ for (i = l2_end; i < Ln_ENTRIES; i++) {
+ if (l2_table[i] == 0)
+ pmap_store(&l2_table[i], invalid_pmd_table);
+ }
+
+ mtx_init(&kernel_pmap->pm_mtx, "kernel pmap", NULL, MTX_DEF);
+ TAILQ_INIT(&kernel_pmap->pm_pvchunk);
+ vm_radix_init(&kernel_pmap->pm_root);
+
+ rw_init(&pvh_global_lock, "pmap pv global");
+
+ /*
+ * Set the current CPU as active in the kernel pmap. Secondary cores
+ * will add themselves later in init_secondary().
+ */
+ CPU_SET(PCPU_GET(cpuid), &kernel_pmap->pm_active);
+
+ /* Assume the address we were loaded to is a valid physical address. */
+ min_pa = max_pa = kernstart;
+
+ physmap_idx = physmem_avail(physmap, nitems(physmap));
+ physmap_idx /= 2;
+
+ /*
+ * Find the minimum physical address. physmap is sorted,
+ * but may contain empty ranges.
+ */
+ for (i = 0; i < physmap_idx * 2; i += 2) {
+ if (physmap[i] == physmap[i + 1])
+ continue;
+ if (physmap[i] <= min_pa)
+ min_pa = physmap[i];
+ if (physmap[i + 1] > max_pa)
+ max_pa = physmap[i + 1];
+ }
+ printf("physmap_idx %u\n", physmap_idx);
+ printf("min_pa %lx\n", min_pa);
+ printf("max_pa %lx\n", max_pa);
+
+ /* Create a direct map region early so we can use it for pa -> va */
+ pmap_bootstrap_dmap(min_pa, max_pa);
+
+ /* Sanity check the index, KERNBASE should be the first VA */
+ KASSERT(l2_slot == 0, ("The L2 index is non-zero"));
+
+ freemempos = roundup2(KERNBASE + kernlen, PAGE_SIZE);
+
+ /* Create the l3 tables for the early devmap */
+ freemempos = pmap_bootstrap_l3(l1,
+ VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE, freemempos);
+
+#if 0
+ /*
+ * Invalidate the mapping we created for the DTB. At this point a copy
+ * has been created, and we no longer need it. We want to avoid the
+ * possibility of an aliased mapping in the future.
+ */
+ l2p = pmap_l2(kernel_pmap, VM_EARLY_DTB_ADDRESS);
+ if ((pmap_pmd_entry(l2p) & PTE_V) != 0)
+ pmap_clear(l2p);
+
+#endif
+
+ /* invalid kernel pgdl */
+ csr_write64(kernel_pmap->pm_pgdl, LOONGARCH_CSR_PGDL);
+
+#define alloc_pages(var, np) \
+ (var) = freemempos; \
+ freemempos += (np * PAGE_SIZE); \
+ memset((char *)(var), 0, ((np) * PAGE_SIZE));
+
+ /* Allocate dynamic per-cpu area. */
+ alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
+ dpcpu_init((void *)dpcpu, 0);
+
+ /* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */
+ alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE);
+ msgbufp = (void *)msgbufpv;
+
+ virtual_avail = KERNBASE + LA_KMOD_VA_SKIP;
+
+ /* clear 2M mappings after kernel image setup by locore */
+ for (va = roundup2(KERNBASE + kernlen, L2_SIZE);
+ va < virtual_avail; va += L2_SIZE) {
+ pd_entry_t *l2 = pmap_l2(kernel_pmap, va);
+ if (l2 != NULL && (pmap_pmd_entry(l2) & PTE_HUGE) != 0)
+ pmap_store(l2, invalid_pmd_table);
+ }
+
+ virtual_end = VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE;
+ kernel_vm_end = virtual_avail;
+
+ physmem_exclude_region(kernstart,
+ pmap_early_vtophys(l1, freemempos) - kernstart, EXFLAG_NOALLOC);
+
+ /* Initialize pv chunk mutex early -- UMA (uma_startup1) runs before
+ * pmap_init() and may trigger pmap_enter/pmap_kenter which need
+ * pv_chunks_mutex. */
+ mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
+
+ invtlb_all(INVTLB_CURRENT_ALL);
+
+#ifdef INVARIANTS
+ pmap_inv_active = true;
+#endif
+}
+
+/*
+ * Initialize a vm_page's machine-dependent fields.
+ */
+void
+pmap_page_init(vm_page_t m)
+{
+
+ TAILQ_INIT(&m->md.pv_list);
+ m->md.pv_memattr = VM_MEMATTR_WRITE_BACK;
+}
+
+/*
+ * Initialize the pmap module.
+ *
+ * Called by vm_mem_init(), to initialize any structures that the pmap
+ * system needs to map virtual memory.
+ */
+void
+pmap_init(void)
+{
+ vm_size_t s;
+ int i, pv_npg;
+
+ /*
+ * Initialize the pool of pv list locks.
+ */
+ for (i = 0; i < NPV_LIST_LOCKS; i++)
+ rw_init(&pv_list_locks[i], "pmap pv list");
+
+ /*
+ * Calculate the size of the pv head table for superpages.
+ */
+ pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L2_SIZE);
+
+ /*
+ * Allocate memory for the pv head table for superpages.
+ */
+ s = (vm_size_t)(pv_npg * sizeof(struct md_page));
+ s = round_page(s);
+ pv_table = kmem_malloc(s, M_WAITOK | M_ZERO);
+ for (i = 0; i < pv_npg; i++)
+ TAILQ_INIT(&pv_table[i].pv_list);
+ TAILQ_INIT(&pv_dummy.pv_list);
+
+ if (superpages_enabled)
+ pagesizes[1] = L2_SIZE;
+}
+
+#ifdef SMP
+/*
+ * For SMP, each invalidation builds a single shootdown request handled by
+ * smp_targeted_tlb_shootdown(): it computes the remote mask, sends one IPI,
+ * flushes locally, and (when interrupts are enabled) waits for every target
+ * to flush. pmap_invalidate_range is now a single shootdown instead of a
+ * per-page IPI loop.
+ */
+static void
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
+{
+
+ sched_pin();
+ smp_targeted_tlb_shootdown(pmap, va, 0, LA_TLB_OP_PAGE);
+ sched_unpin();
+}
+
+static void
+pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+
+ sched_pin();
+ smp_targeted_tlb_shootdown(pmap, sva, eva, LA_TLB_OP_RANGE);
+ sched_unpin();
+}
+
+static void
+pmap_invalidate_all(pmap_t pmap)
+{
+
+ sched_pin();
+ smp_targeted_tlb_shootdown(pmap, 0, 0, LA_TLB_OP_ALL);
+ sched_unpin();
+}
+
+#else
+/*
+ * Normal, non-SMP, invalidation functions.
+ * We inline these within pmap.c for speed.
+ */
+static __inline void
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
+{
+ if (pmap == kernel_pmap)
+ invtlb(INVTLB_ADDR_GTRUE_OR_ASID, pmap_current_asid(), va);
+ else
+ invtlb(INVTLB_ADDR_GFALSE_AND_ASID,
+ pmap_get_asid(pmap, PCPU_GET(cpuid)), va);
+}
+
+static __inline void
+pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+ if (pmap == kernel_pmap) {
+ uint32_t curasid = pmap_current_asid();
+ while (sva < eva) {
+ invtlb(INVTLB_ADDR_GTRUE_OR_ASID, curasid, sva);
+ sva += PAGE_SIZE;
+ }
+ } else {
+ uint32_t asid = pmap_get_asid(pmap, PCPU_GET(cpuid));
+ while (sva < eva) {
+ invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, sva);
+ sva += PAGE_SIZE;
+ }
+ }
+}
+
+static __inline void
+pmap_invalidate_all(pmap_t pmap)
+{
+ TLB_DBG("TLB inv: cpu%d %s ALL\n",
+ PCPU_GET(cpuid),
+ pmap == kernel_pmap ? "kernel" : "user");
+ if (pmap == kernel_pmap)
+ invtlb_all(INVTLB_CURRENT_ALL);
+ else
+ flush_tlb_local(pmap);
+}
+#endif
+
+/*
+ * Routine: pmap_extract
+ * Function:
+ * Extract the physical page address associated
+ * with the given map/virtual_address pair.
+ */
+vm_paddr_t
+pmap_extract(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *l2p, l2;
+ pt_entry_t *l3p;
+ vm_paddr_t pa;
+
+ /*
+ * DMW0 is a hardware direct-map window with no page-table entry; the
+ * physical address is just the low 48 bits of the VA. Handle it
+ * before touching the page table (pmap_l1() would KASSERT on these
+ * VAs, which are outside the page-table-managed kernel range).
+ */
+ if (VA_IN_DMW0(va))
+ return (DMW0_TO_PHYS(va));
+
+ pa = 0;
+
+ /*
+ * Start with an L2 lookup, L1 superpages are currently not implemented.
+ */
+ PMAP_LOCK(pmap);
+ l2p = pmap_l2(pmap, va);
+ if (l2p != NULL && pmap_pmd_valid(l2p)) {
+ l2 = pmap_pmd_entry(l2p);
+ if (!(l2 & PTE_HUGE)) {
+ l3p = pmap_l2_to_l3(l2p, va);
+ pa = PTE_TO_PHYS(pmap_pte_entry(l3p));
+ pa |= (va & L3_OFFSET);
+ } else {
+ /* L2 is a superpage mapping. */
+ pa = L2PTE_TO_PHYS(l2);
+ pa |= (va & L2_OFFSET);
+ }
+ }
+ PMAP_UNLOCK(pmap);
+ return (pa);
+}
+
+/*
+ * Routine: pmap_extract_and_hold
+ * Function:
+ * Atomically extract and hold the physical page
+ * with the given pmap and virtual address pair
+ * if that mapping permits the given protection.
+ */
+vm_page_t
+pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
+{
+ pd_entry_t *l2p, l2;
+ pt_entry_t *l3p, l3;
+ vm_page_t m;
+
+ m = NULL;
+ PMAP_LOCK(pmap);
+ l2p = pmap_l2(pmap, va);
+ if (l2p == NULL || !pmap_pmd_valid(l2p)) {
+ PMAP_UNLOCK(pmap);
+ return (NULL);
+ }
+
+ l2 = pmap_pmd_entry(l2p);
+ if ((l2 & PTE_HUGE)) {
+ if ((l2 & PTE_W) != 0 || (prot & VM_PROT_WRITE) == 0) {
+ m = PHYS_TO_VM_PAGE(L2PTE_TO_PHYS(l2) +
+ (va & L2_OFFSET));
+ }
+ } else {
+ l3p = pmap_l2_to_l3(l2p, va);
+ if (pmap_pte_valid(l3p)) {
+ l3 = pmap_pte_entry(l3p);
+ if ((l3 & PTE_W) != 0 || (prot & VM_PROT_WRITE) == 0)
+ m = PTE_TO_VM_PAGE(l3);
+ }
+ }
+
+ if (m != NULL && !vm_page_wire_mapped(m))
+ m = NULL;
+
+ PMAP_UNLOCK(pmap);
+ return (m);
+}
+
+vm_paddr_t
+pmap_kextract(vm_offset_t va)
+{
+ pd_entry_t *l2, l2e;
+ pt_entry_t *l3;
+ vm_paddr_t pa;
+
+ if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
+ pa = DMAP_TO_PHYS(va);
+ } else if (VA_IN_DMW0(va)) {
+ /* DMW0 hardware direct-map window: no page-table entry. */
+ pa = DMW0_TO_PHYS(va);
+ } else {
+ l2 = pmap_l2(kernel_pmap, va);
+ if (l2 == NULL)
+ panic("pmap_kextract: No l2");
+ l2e = pmap_pmd_entry(l2);
+ /*
+ * Beware of concurrent promotion and demotion! We must
+ * use l2e rather than loading from l2 multiple times to
+ * ensure we see a consistent state, including the
+ * implicit load in pmap_l2_to_l3. It is, however, safe
+ * to use an old l2e because the L3 page is preserved by
+ * promotion.
+ */
+ if (pmap_pmd_valid(l2) && (l2e & PTE_HUGE)) {
+ /* superpages */
+ pa = L2PTE_TO_PHYS(l2e);
+ pa |= (va & L2_OFFSET);
+ return (pa);
+ }
+
+ l3 = pmap_l2_to_l3(&l2e, va);
+ if (l3 == NULL)
+ panic("pmap_kextract: No l3...");
+ pa = PTE_TO_PHYS(pmap_pte_entry(l3));
+ pa |= (va & PAGE_MASK);
+ }
+ return (pa);
+}
+
+/***************************************************
+ * Low level mapping routines.....
+ ***************************************************/
+
+void
+pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode)
+{
+ pt_entry_t entry;
+ pt_entry_t *l3;
+ vm_offset_t va, eva;
+ pn_t pn;
+
+ KASSERT((pa & L3_OFFSET) == 0,
+ ("pmap_kenter_device: Invalid physical address"));
+ KASSERT((sva & L3_OFFSET) == 0,
+ ("pmap_kenter_device: Invalid virtual address"));
+ KASSERT((size & PAGE_MASK) == 0,
+ ("pmap_kenter_device: Mapping is not page-sized"));
+
+ va = sva;
+ while (size != 0) {
+ pd_entry_t *l2;
+
+ l2 = pmap_l2(kernel_pmap, va);
+ if (l2 != NULL && (pmap_pmd_entry(l2) & PTE_HUGE) != 0) {
+ PMAP_LOCK(kernel_pmap);
+ (void)pmap_demote_l2(kernel_pmap, l2,
+ va & ~L2_OFFSET);
+ PMAP_UNLOCK(kernel_pmap);
+ }
+
+ l3 = pmap_l3(kernel_pmap, va);
+ KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va));
+
+ pn = pa >> PAGE_PFN_SHIFT;
+ if (mode == VM_MEMATTR_DEVICE ||
+ mode == VM_MEMATTR_UNCACHEABLE ||
+ mode == VM_MEMATTR_WRITE_COMBINING)
+ entry = PTE_KERN_SUC;
+ else
+ entry = PTE_KERN;
+ entry |= pn << L3_SHIFT;
+ pmap_store(l3, entry);
+
+ va += PAGE_SIZE;
+ pa += PAGE_SIZE;
+ size -= PAGE_SIZE;
+ }
+ pmap_invalidate_range(kernel_pmap, sva, va);
+
+ eva = va;
+ for (va = sva; va < eva; va += PAGE_SIZE)
+ pmap_update_tlb(kernel_pmap, va);
+}
+
+void
+pmap_kenter_device(vm_offset_t sva, vm_size_t size, vm_paddr_t pa)
+{
+ pmap_kenter(sva, size, pa, VM_MEMATTR_DEVICE);
+}
+
+/*
+ * Remove a page from the kernel pagetables.
+ * Note: not SMP coherent.
+ */
+void
+pmap_kremove(vm_offset_t va)
+{
+ pt_entry_t *l3;
+
+ l3 = pmap_l3(kernel_pmap, va);
+ KASSERT(l3 != NULL, ("pmap_kremove: Invalid address"));
+
+ pmap_clear(l3);
+
+ pmap_invalidate_page(kernel_pmap, va);
+}
+
+void
+pmap_kremove_device(vm_offset_t sva, vm_size_t size)
+{
+ pt_entry_t *l3;
+ vm_offset_t va;
+
+ KASSERT((sva & L3_OFFSET) == 0,
+ ("pmap_kremove_device: Invalid virtual address"));
+ KASSERT((size & PAGE_MASK) == 0,
+ ("pmap_kremove_device: Mapping is not page-sized"));
+
+ va = sva;
+ while (size != 0) {
+ pd_entry_t *l2;
+
+ l2 = pmap_l2(kernel_pmap, va);
+ if (l2 != NULL && (pmap_pmd_entry(l2) & PTE_HUGE) != 0) {
+ PMAP_LOCK(kernel_pmap);
+ (void)pmap_demote_l2(kernel_pmap, l2,
+ va & ~L2_OFFSET);
+ PMAP_UNLOCK(kernel_pmap);
+ }
+
+ l3 = pmap_l3(kernel_pmap, va);
+ KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va));
+ pmap_clear(l3);
+
+ va += PAGE_SIZE;
+ size -= PAGE_SIZE;
+ }
+
+ pmap_invalidate_range(kernel_pmap, sva, va);
+}
+
+/*
+ * Used to map a range of physical addresses into kernel
+ * virtual address space.
+ *
+ * The value passed in '*virt' is a suggested virtual address for
+ * the mapping. Architectures which can support a direct-mapped
+ * physical to virtual region can return the appropriate address
+ * within that region, leaving '*virt' unchanged. Other
+ * architectures should map the pages starting at '*virt' and
+ * update '*virt' with the first usable address after the mapped
+ * region.
+ */
+void *
+pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
+{
+
+ return (PHYS_TO_DMAP(start));
+}
+
+/*
+ * Add a list of wired pages to the kva
+ * this routine is only used for temporary
+ * kernel mappings that do not need to have
+ * page modification or references recorded.
+ * Note that old mappings are simply written
+ * over. The page *must* be wired.
+ * Note: SMP coherent. Uses a ranged shootdown IPI.
+ */
+void
+pmap_qenter(void *sva, vm_page_t *ma, int count)
+{
+ pt_entry_t *l3, pa;
+ vm_offset_t va, eva;
+ vm_page_t m;
+ pt_entry_t entry;
+ pn_t pn;
+ int i;
+
+ va = (vm_offset_t)sva;
+ for (i = 0; i < count; i++) {
+ pt_entry_t *l2;
+ m = ma[i];
+ pa = VM_PAGE_TO_PHYS(m);
+ pn = pa >> PAGE_PFN_SHIFT;
+
+ /* 2M page must be demoted to 4K */
+ l2 = pmap_l2(kernel_pmap, va);
+ if (l2 != NULL && (pmap_pmd_entry(l2) & PTE_HUGE) != 0) {
+ PMAP_LOCK(kernel_pmap);
+ (void)pmap_demote_l2(kernel_pmap, l2,
+ va & ~L2_OFFSET);
+ PMAP_UNLOCK(kernel_pmap);
+ }
+
+ l3 = pmap_l3(kernel_pmap, va);
+
+ entry = PTE_KERN;
+ entry |= (pn << L3_SHIFT);
+ pmap_store(l3, entry);
+
+ va += L3_SIZE;
+ }
+
+ pmap_invalidate_range(kernel_pmap, (vm_offset_t)sva, va);
+
+ eva = va;
+ for (va = (vm_offset_t)sva; va < eva; va += L3_SIZE)
+ pmap_update_tlb(kernel_pmap, va);
+}
+
+/*
+ * This routine tears out page mappings from the
+ * kernel -- it is meant only for temporary mappings.
+ * Note: SMP coherent. Uses a ranged shootdown IPI.
+ */
+void
+pmap_qremove(void *sva, int count)
+{
+ pt_entry_t *l3;
+ vm_offset_t va;
+
+ KASSERT((vm_offset_t)sva >= VM_MIN_KERNEL_ADDRESS,
+ ("usermode va %lx", (vm_offset_t)sva));
+
+ for (va = (vm_offset_t)sva; count-- > 0; va += PAGE_SIZE) {
+ l3 = pmap_l3(kernel_pmap, va);
+ KASSERT(l3 != NULL, ("pmap_kremove: Invalid address"));
+ pmap_clear(l3);
+ }
+ pmap_invalidate_range(kernel_pmap, (vm_offset_t)sva, va);
+}
+
+bool
+pmap_ps_enabled(pmap_t pmap __unused)
+{
+
+ return (superpages_enabled);
+}
+
+/***************************************************
+ * Page table page management routines.....
+ ***************************************************/
+/*
+ * Schedule the specified unused page table page to be freed. Specifically,
+ * add the page to the specified list of pages that will be released to the
+ * physical memory manager after the TLB has been updated.
+ */
+static __inline void
+pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO)
+{
+
+ if (set_PG_ZERO)
+ m->flags |= PG_ZERO;
+ else
+ m->flags &= ~PG_ZERO;
+ SLIST_INSERT_HEAD(free, m, plinks.s.ss);
+}
+
+/*
+ * Inserts the specified page table page into the specified pmap's collection
+ * of idle page table pages. Each of a pmap's page table pages is responsible
+ * for mapping a distinct range of virtual addresses. The pmap's collection is
+ * ordered by this virtual address range.
+ *
+ * If "promoted" is false, then the page table page "ml3" must be zero filled.
+ * mpte's valid filed will be set to 1.
+ *
+ * If promoted and l3_all_PTE_A are both true, then mpte must contain valid
+ * mappings with identical attributes include PTE_A, mpte's valid field will
+ * be set to VM_PAGE_BITS_ALL.
+ */
+static __inline int
+pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,
+ bool l3_all_PTE_A)
+{
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ KASSERT(promoted || !l3_all_PTE_A,
+ ("a zero-filled PTP can't have PTE_A set in every PTE"));
+ mpte->valid = promoted ? (l3_all_PTE_A ? VM_PAGE_BITS_ALL : 1) : 0;
+ return (vm_radix_insert(&pmap->pm_root, mpte));
+}
+
+/*
+ * Removes the page table page mapping the specified virtual address from the
+ * specified pmap's collection of idle page table pages, and returns it.
+ * Otherwise, returns NULL if there is no page table page corresponding to the
+ * specified virtual address.
+ */
+static __inline vm_page_t
+pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
+{
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va)));
+}
+
+/*
+ * Decrements a page table page's reference count, which is used to record the
+ * number of valid page table entries within the page. If the reference count
+ * drops to zero, then the page table page is unmapped. Returns true if the
+ * page table page was unmapped and false otherwise.
+ */
+static inline bool
+pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
+{
+ KASSERT(m != NULL, ("%s: m is NULL va %#lx", __func__, va));
+ KASSERT(m->ref_count > 0,
+ ("%s: page %p ref count underflow", __func__, m));
+
+ --m->ref_count;
+ if (m->ref_count == 0) {
+ _pmap_unwire_ptp(pmap, va, m, free);
+ return (true);
+ } else {
+ return (false);
+ }
+}
+
+static void
+_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
+{
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ if (m->pindex >= NUL2E + NUL1E) {
+ pd_entry_t *l0;
+ l0 = pmap_l0(pmap, va);
+ pmap_store(l0, invalid_pgd_table);
+ } else if (m->pindex >= NUL2E) {
+ pd_entry_t *l1;
+ l1 = pmap_l1(pmap, va);
+ pmap_store(l1, invalid_pud_table);
+ } else {
+ pd_entry_t *l2;
+ l2 = pmap_l2(pmap, va);
+#ifdef INVARIANTS
+ /*
+ * Verify the L2 entry we're about to clear actually points at
+ * this page. If it points at a different page (or is already
+ * the sentinel), we'd be clearing the WRONG L2 entry — the
+ * real one would survive, pointing at the freed page → the
+ * freed-while-linked corruption.
+ */
+ KASSERT(l2 != NULL && pmap_pmd_valid(l2) &&
+ PTE_TO_PHYS(pmap_pmd_entry(l2)) == VM_PAGE_TO_PHYS(m),
+ ("_pmap_unwire_ptp: L2 entry for va %#lx doesn't point at "
+ "this page (l2e=%#lx page phys %#lx) — wrong L2 cleared",
+ va, l2 ? pmap_pmd_entry(l2) : 0, VM_PAGE_TO_PHYS(m)));
+#endif
+ pmap_store(l2, invalid_pmd_table);
+ }
+ pmap_resident_count_dec(pmap, 1);
+ if (m->pindex < NUL2E) {
+ pd_entry_t *l1;
+ vm_page_t pdpg;
+
+ l1 = pmap_l1(pmap, va);
+ pdpg = PTE_TO_VM_PAGE(pmap_pud_entry(l1));
+ pmap_unwire_ptp(pmap, va, pdpg, free);
+ } else if (m->pindex < NUL2E + NUL1E) {
+ pd_entry_t *l0;
+ vm_page_t pdpg;
+
+ l0 = pmap_l0(pmap, va);
+ pdpg = PTE_TO_VM_PAGE(pmap_pgd_entry(l0));
+ pmap_unwire_ptp(pmap, va, pdpg, free);
+ }
+ pmap_invalidate_page(pmap, va);
+
+ vm_wire_sub(1);
+
+ /*
+ * Put page on a list so that it is released after *ALL* TLB shootdown
+ * is done. Only a leaf (L3) page-table page is actually zeroed at this
+ * point: non-leaf (L2/L1) pages still hold the shared
+ * invalid-page-table sentinels (invalid_pmd_table / invalid_pud_table)
+ * in their slots, so they are NOT zeroed and must not be marked
+ * PG_ZERO. Claiming PG_ZERO on them made vm_page_alloc(VM_ALLOC_ZERO)
+ * hand them out unzeroed, panicking UMA ctors under INVARIANTS ("lock
+ * already initialized").
+ */
+ pmap_add_delayed_free_list(m, free, m->pindex < NUL2E);
+}
+
+/*
+ * After removing a page table entry, this routine is used to
+ * conditionally free the page, and manage the reference count.
+ */
+static int
+pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,
+ struct spglist *free)
+{
+ vm_page_t mpte;
+
+ if (va >= VM_MAXUSER_ADDRESS)
+ return (0);
+ KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
+ mpte = PTE_TO_VM_PAGE(ptepde);
+ KASSERT(mpte >= vm_page_array &&
+ mpte < &vm_page_array[vm_page_array_size],
+ ("pmap_unuse_pt: ptepde %#lx pte_to_phys %#llx mpte %p va %#lx",
+ ptepde, PTE_TO_PHYS(ptepde), mpte, va));
+ return (pmap_unwire_ptp(pmap, va, mpte, free));
+}
+
+void
+pmap_pinit0(pmap_t pmap)
+{
+ PMAP_LOCK_INIT(pmap);
+ bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
+ bzero(&pmap->asid, sizeof(pmap->asid));
+ pmap->pm_top = kernel_pmap->pm_top;
+ pmap->pm_pgdl = kernel_pmap->pm_pgdl;
+ pmap->pm_pgdh = kernel_pmap->pm_pgdh;
+ CPU_ZERO(&pmap->pm_active);
+ TAILQ_INIT(&pmap->pm_pvchunk);
+ vm_radix_init(&pmap->pm_root);
+ pmap_activate_boot(pmap);
+}
+
+int
+pmap_pinit(pmap_t pmap)
+{
+ vm_paddr_t topphys;
+ vm_page_t mtop;
+
+ mtop = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO |
+ VM_ALLOC_WAITOK);
+
+ topphys = VM_PAGE_TO_PHYS(mtop);
+ pmap->pm_top = (pd_entry_t *)PHYS_TO_DMAP(topphys);
+ pmap->pm_pgdl = topphys;
+ pmap->pm_pgdh = kernel_pmap->pm_pgdh;
+
+ pmap_init_pd(pmap->pm_top);
+
+ bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
+ bzero(&pmap->asid, sizeof(pmap->asid));
+
+ CPU_ZERO(&pmap->pm_active);
+
+ TAILQ_INIT(&pmap->pm_pvchunk);
+ vm_radix_init(&pmap->pm_root);
+
+ return (1);
+}
+
+/*
+ * This routine is called if the desired page table page does not exist.
+ *
+ * If page table page allocation fails, this routine may sleep before
+ * returning NULL. It sleeps only if a lock pointer was given.
+ *
+ * Note: If a page allocation fails at page table level two or three,
+ * one or two pages may be held during the wait, only to be released
+ * afterwards. This conservative approach is easily argued to avoid
+ * race conditions.
+ */
+static vm_page_t
+_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
+{
+ vm_page_t m, pdpg;
+ pt_entry_t entry;
+ vm_paddr_t phys;
+ pn_t pn;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+ /*
+ * Allocate a page table page.
+ */
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+ if (m == NULL) {
+ if (lockp != NULL) {
+ RELEASE_PV_LIST_LOCK(lockp);
+ PMAP_UNLOCK(pmap);
+ rw_runlock(&pvh_global_lock);
+ vm_wait(NULL);
+ rw_rlock(&pvh_global_lock);
+ PMAP_LOCK(pmap);
+ }
+
+ /*
+ * Indicate the need to retry. While waiting, the page table
+ * page may have been allocated.
+ */
+ return (NULL);
+ }
+ m->pindex = ptepindex;
+
+ /*
+ * Map the pagetable page into the process address space, if
+ * it isn't already there.
+ */
+ pn = VM_PAGE_TO_PHYS(m) >> PAGE_PFN_SHIFT;
+ if (ptepindex >= NUL2E + NUL1E) {
+ pd_entry_t *l0;
+ vm_pindex_t l0index;
+
+ pmap_init_pd(VM_PAGE_TO_DMAP(m));
+
+ KASSERT(pmap_mode == PMAP_MODE_LA48,
+ ("%s: pindex %#lx not in LA48 mode", __func__, ptepindex));
+ KASSERT(ptepindex < NUL2E + NUL1E + NUL0E,
+ ("%s: pindex %#lx out of range", __func__, ptepindex));
+
+ l0index = ptepindex - (NUL2E + NUL1E);
+ l0 = &pmap->pm_top[l0index];
+ KASSERT(!(pmap_pgd_valid(l0)),
+ ("%s: L0 entry %#lx is valid",
+ __func__, pmap_pgd_entry(l0)));
+
+ entry = pn << L3_SHIFT;
+ pmap_store(l0, entry);
+ } else if (ptepindex >= NUL2E) {
+ pd_entry_t *l0, *l1;
+ vm_pindex_t l0index, l1index;
+
+ l1index = ptepindex - NUL2E;
+ pmap_init_pd(VM_PAGE_TO_DMAP(m));
+
+ l0index = l1index >> Ln_ENTRIES_SHIFT;
+ l0 = &pmap->pm_top[l0index];
+ if (!pmap_pgd_valid(l0)) {
+ /* Recurse to allocate the L1 page. */
+ if (_pmap_alloc_l3(pmap,
+ NUL2E + NUL1E + l0index, lockp) == NULL)
+ goto fail;
+ phys = PTE_TO_PHYS(pmap_pgd_entry(l0));
+ } else {
+ phys = PTE_TO_PHYS(pmap_pgd_entry(l0));
+ pdpg = PHYS_TO_VM_PAGE(phys);
+ pdpg->ref_count++;
+ }
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ l1 = &l1[ptepindex & Ln_ADDR_MASK];
+
+ KASSERT(!pmap_pud_valid(l1),
+ ("%s: L1 entry %#lx is valid",
+ __func__, pmap_pud_entry(l1)));
+
+ entry = pn << L3_SHIFT;
+ pmap_store(l1, entry);
+ } else {
+ vm_pindex_t l0index, l1index;
+ pd_entry_t *l0, *l1, *l2;
+
+ l1index = ptepindex >> (L1_SHIFT - L2_SHIFT);
+ l0index = l1index >> Ln_ENTRIES_SHIFT;
+ l0 = &pmap->pm_top[l0index];
+ if (!pmap_pgd_valid(l0)) {
+ /* Recurse to allocate the L1 entry. */
+ if (_pmap_alloc_l3(pmap, NUL2E + l1index,
+ lockp) == NULL)
+ goto fail;
+ phys = PTE_TO_PHYS(pmap_pgd_entry(l0));
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ l1 = &l1[l1index & Ln_ADDR_MASK];
+ } else {
+ phys = PTE_TO_PHYS(pmap_pgd_entry(l0));
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ l1 = &l1[l1index & Ln_ADDR_MASK];
+ if (!pmap_pud_valid(l1)) {
+ /* Recurse to allocate the L2 page. */
+ if (_pmap_alloc_l3(pmap,
+ NUL2E + l1index, lockp) == NULL)
+ goto fail;
+ } else {
+ pdpg = PTE_TO_VM_PAGE(pmap_pud_entry(l1));
+ pdpg->ref_count++;
+ }
+ }
+
+ phys = PTE_TO_PHYS(pmap_pud_entry(l1));
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ l2 = &l2[ptepindex & Ln_ADDR_MASK];
+ KASSERT(!pmap_pmd_valid(l2),
+ ("%s: L2 entry %#lx is valid",
+ __func__, pmap_pmd_entry(l2)));
+
+ entry = (pn << L3_SHIFT);
+ pmap_store(l2, entry);
+ }
+
+ pmap_resident_count_inc(pmap, 1);
+
+ return (m);
+
+fail:
+ vm_page_unwire_noq(m);
+ /*
+ * m may be a non-leaf page-table page whose slots were filled with the
+ * shared invalid-page-table sentinels by pmap_init_pd() above (the L1
+ * branch reaches here after that call), so it is not necessarily
+ * zeroed. Use vm_page_free(), not vm_page_free_zero(), to avoid
+ * falsely marking it PG_ZERO.
+ */
+ vm_page_free(m);
+ return (NULL);
+}
+
+static vm_page_t
+pmap_alloc_l2(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
+{
+ pd_entry_t *l1;
+ vm_page_t l2pg;
+ vm_pindex_t pindex;
+
+retry:
+ l1 = pmap_l1(pmap, va);
+ if (l1 != NULL && pmap_pud_valid(l1)) {
+ KASSERT(!(pmap_pud_entry(l1) & PTE_HUGE),
+ ("%s: L1 entry %#lx for VA %#lx is a leaf", __func__,
+ pmap_pud_entry(l1), va));
+ /* Add a reference to the L2 page. */
+ l2pg = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_pud_entry(l1)));
+ l2pg->ref_count++;
+ } else {
+ /* Allocate a L2 page. */
+ pindex = pmap_l1_pindex(va);
+ l2pg = _pmap_alloc_l3(pmap, pindex, lockp);
+ if (l2pg == NULL && lockp != NULL)
+ goto retry;
+ }
+ return (l2pg);
+}
+
+static vm_page_t
+pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
+{
+ vm_pindex_t ptepindex;
+ pd_entry_t *l2;
+ vm_paddr_t phys;
+ vm_page_t m;
+
+ /*
+ * Calculate pagetable page index
+ */
+ ptepindex = pmap_l2_pindex(va);
+retry:
+ /*
+ * Get the page directory entry
+ */
+ l2 = pmap_l2(pmap, va);
+
+ /*
+ * If the page table page is mapped, we just increment the
+ * hold count, and activate it.
+ */
+ if (l2 != NULL && pmap_pmd_valid(l2)) {
+ phys = PTE_TO_PHYS(pmap_pmd_entry(l2));
+ m = PHYS_TO_VM_PAGE(phys);
+ m->ref_count++;
+ } else {
+ /*
+ * Here if the pte page isn't mapped, or if it has been
+ * deallocated.
+ */
+ m = _pmap_alloc_l3(pmap, ptepindex, lockp);
+ if (m == NULL && lockp != NULL)
+ goto retry;
+ }
+ return (m);
+}
+
+/***************************************************
+ * Pmap allocation/deallocation routines.
+ ***************************************************/
+
+/*
+ * Reconcile pm_stats.resident_count against an actual walk of the user page
+ * table, run from pmap_release when the count is non-zero. By now
+ * pmap_remove_pages() (PV-tracked) and pmap_remove() (via vm_map_delete) have
+ * torn the tables down, so both the count and the walk should be 0.
+ *
+ * The LoongArch resident_count model sums: each valid L3 leaf (+1), each L3
+ * page-table page (+1), and each 2MB superpage (+L2_SIZE/PAGE_SIZE). A wired
+ * superpage's shadow PT page is also +1 but lives only in the PT-page trie,
+ * not in the page table, so it is not counted here. Comparing the walked
+ * total to resident_count localizes the leak:
+ * expected=0, count>0 -> phantom inc (an inc with no matching dec)
+ * ptpages>0 -> L3 pages whose L2 slot was never reverted to the
+ * sentinel (the _pmap_unwire_ptp path was skipped)
+ * leaves>0 -> real mappings that survived teardown (e.g. an
+ * unmanaged special page outside every vm_map range)
+ * Every level is read through the DMAP window so the probe never depends on
+ * the (already-torn-down) user mappings.
+ */
+static void
+pmap_release_audit(pmap_t pmap)
+{
+ pd_entry_t *l0, *l1, *l2, *l3t, e;
+ vm_paddr_t phys, phys_max;
+ int i0, i1, i2, i3;
+ uint64_t leaves = 0, ptpages = 0, superpages = 0, expected;
+
+ phys_max = dmap_phys_max;
+ l0 = pmap->pm_top;
+ for (i0 = 0; i0 < Ln_ENTRIES; i0++) {
+ e = pmap_load(&l0[i0]);
+ if (e == invalid_pgd_table || e == 0)
+ continue;
+ phys = PTE_TO_PHYS(e);
+ if (phys == 0 || phys >= phys_max)
+ continue;
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ for (i1 = 0; i1 < Ln_ENTRIES; i1++) {
+ e = pmap_load(&l1[i1]);
+ if (e == invalid_pud_table || e == 0)
+ continue;
+ if ((e & PTE_HUGE) != 0) {
+ superpages += L1_SIZE / PAGE_SIZE;
+ continue;
+ }
+ phys = PTE_TO_PHYS(e);
+ if (phys == 0 || phys >= phys_max)
+ continue;
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ for (i2 = 0; i2 < Ln_ENTRIES; i2++) {
+ e = pmap_load(&l2[i2]);
+ if (e == invalid_pmd_table || e == 0)
+ continue;
+ if ((e & PTE_HUGE) != 0) {
+ superpages++;
+ continue;
+ }
+ ptpages++; /* L2 points at an L3 table */
+ phys = PTE_TO_PHYS(e);
+ if (phys == 0 || phys >= phys_max)
+ continue;
+ l3t = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ for (i3 = 0; i3 < Ln_ENTRIES; i3++) {
+ e = pmap_load(&l3t[i3]);
+ if ((e & PTE_V) != 0)
+ leaves++;
+ }
+ }
+ }
+ }
+ expected = leaves + ptpages + superpages * (L2_SIZE / PAGE_SIZE);
+ printf("pmap_release_audit: resident_count=%ld expected=%lu "
+ "(leaves=%lu ptpages=%lu superpages=%lu)\n",
+ pmap->pm_stats.resident_count, expected, leaves, ptpages,
+ superpages);
+}
+
+/*
+ * Release any resources held by the given physical map.
+ * Called when a pmap initialized by pmap_pinit is being released.
+ * Should only be called if the map contains no valid mappings.
+ */
+void
+pmap_release(pmap_t pmap)
+{
+ vm_page_t m;
+
+ if (pmap != kernel_pmap && pmap->pm_stats.resident_count != 0)
+ pmap_release_audit(pmap);
+
+ KASSERT(pmap->pm_stats.resident_count == 0,
+ ("pmap_release: pmap resident count %ld != 0",
+ pmap->pm_stats.resident_count));
+ KASSERT(CPU_EMPTY(&pmap->pm_active),
+ ("releasing active pmap %p", pmap));
+
+ m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_top));
+ vm_page_unwire_noq(m);
+ vm_page_free(m);
+}
+
+static int
+kvm_size(SYSCTL_HANDLER_ARGS)
+{
+ unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
+
+ return sysctl_handle_long(oidp, &ksize, 0, req);
+}
+SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE,
+ 0, 0, kvm_size, "LU",
+ "Size of KVM");
+
+static int
+kvm_free(SYSCTL_HANDLER_ARGS)
+{
+ unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
+
+ return sysctl_handle_long(oidp, &kfree, 0, req);
+}
+SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE,
+ 0, 0, kvm_free, "LU",
+ "Amount of KVM free");
+
+/*
+ * grow the number of kernel page table entries, if needed
+ */
+static int
+pmap_growkernel_nopanic(vm_offset_t addr)
+{
+ vm_paddr_t paddr;
+ vm_page_t nkpg;
+ pd_entry_t *l1, *l2;
+ pt_entry_t entry;
+ pn_t pn;
+
+ mtx_assert(&kernel_map->system_mtx, MA_OWNED);
+
+ addr = roundup2(addr, L2_SIZE);
+ if (addr - 1 >= vm_map_max(kernel_map))
+ addr = vm_map_max(kernel_map);
+ while (kernel_vm_end < addr) {
+ l1 = pmap_l1(kernel_pmap, kernel_vm_end);
+ if (!pmap_pud_valid(l1)) {
+ /* We need a new PDP entry */
+ nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
+ VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+ if (nkpg == NULL)
+ return (KERN_RESOURCE_SHORTAGE);
+
+ nkpg->pindex = pmap_l1_pindex(kernel_vm_end);
+ pmap_init_pd(VM_PAGE_TO_DMAP(nkpg));
+ paddr = VM_PAGE_TO_PHYS(nkpg);
+
+ pn = paddr >> PAGE_PFN_SHIFT;
+ entry = 0;
+ entry |= (pn << L3_SHIFT);
+ pmap_store(l1, entry);
+ continue; /* try again */
+ }
+ l2 = pmap_l1_to_l2(l1, kernel_vm_end);
+ if (pmap_pmd_valid(l2)) {
+ kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
+ if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
+ kernel_vm_end = vm_map_max(kernel_map);
+ break;
+ }
+ continue;
+ }
+
+ nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
+ VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+ if (nkpg == NULL)
+ return (KERN_RESOURCE_SHORTAGE);
+ nkpg->pindex = pmap_l2_pindex(kernel_vm_end);
+ /*
+ * This is an L3 (leaf PTE) page, not a directory page: it must
+ * read as zero/empty, never the directory sentinel.
+ * VM_ALLOC_ZERO already zeroed it, so do NOT pmap_init_pd()
+ * here -- doing so filled every empty PTE slot with
+ * invalid_p*d_table, which the HW walker read as a (V=0) PTE
+ * and which also tripped the "0 stored into a directory page"
+ * watch on pmap_qremove().
+ */
+ paddr = VM_PAGE_TO_PHYS(nkpg);
+
+ /*
+ * Store the L3 page table physical address in the L2
+ * directory entry. LoongArch hardware does not check
+ * flags in directory entries during page table walks, so
+ * only the physical address is needed — no mapping flags.
+ */
+ pn = paddr >> PAGE_PFN_SHIFT;
+ entry = pn << L3_SHIFT;
+ pmap_store(l2, entry);
+
+ pmap_invalidate_page(kernel_pmap, kernel_vm_end);
+
+ kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
+ if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
+ kernel_vm_end = vm_map_max(kernel_map);
+ break;
+ }
+ }
+ return (KERN_SUCCESS);
+}
+
+
+/***************************************************
+ * page management routines.
+ ***************************************************/
+
+static const uint64_t pc_freemask[_NPCM] = {
+ [0 ... _NPCM - 2] = PC_FREEN,
+ [_NPCM - 1] = PC_FREEL
+};
+
+#ifdef PV_STATS
+static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
+
+SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
+ "Current number of pv entry chunks");
+SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
+ "Current number of pv entry chunks allocated");
+SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
+ "Current number of pv entry chunks frees");
+SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD,
+ &pc_chunk_tryfail, 0,
+ "Number of times tried to get a chunk page but failed.");
+
+static long pv_entry_frees, pv_entry_allocs, pv_entry_count;
+static int pv_entry_spare;
+
+SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
+ "Current number of pv entry frees");
+SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD,
+ &pv_entry_allocs, 0,
+ "Current number of pv entry allocs");
+SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
+ "Current number of pv entries");
+SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
+ "Current number of spare pv entries");
+#endif
+
+/*
+ * We are in a serious low memory condition. Resort to
+ * drastic measures to free some pages so we can allocate
+ * another pv entry chunk.
+ *
+ * Returns NULL if PV entries were reclaimed from the specified pmap.
+ *
+ * We do not, however, unmap 2mpages because subsequent accesses will
+ * allocate per-page pv entries until repromotion occurs, thereby
+ * exacerbating the shortage of free pv entries.
+ */
+static vm_page_t
+reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
+{
+
+ panic("LoongArch Todo: reclaim_pv_chunk");
+}
+
+/*
+ * free the pv_entry back to the free list
+ */
+static void
+free_pv_entry(pmap_t pmap, pv_entry_t pv)
+{
+ struct pv_chunk *pc;
+ int idx, field, bit;
+
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ PV_STAT(atomic_add_long(&pv_entry_frees, 1));
+ PV_STAT(atomic_add_int(&pv_entry_spare, 1));
+ PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
+ pc = pv_to_chunk(pv);
+ idx = pv - &pc->pc_pventry[0];
+ field = idx / 64;
+ bit = idx % 64;
+ pc->pc_map[field] |= 1ul << bit;
+ if (!pc_is_free(pc)) {
+ /* 98% of the time, pc is already at the head of the list. */
+ if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
+ }
+ return;
+ }
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ free_pv_chunk(pc);
+}
+
+static void
+free_pv_chunk(struct pv_chunk *pc)
+{
+ vm_page_t m;
+
+ mtx_lock(&pv_chunks_mutex);
+ TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
+ mtx_unlock(&pv_chunks_mutex);
+ PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
+ PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
+ PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
+ /* entire chunk is free, return it */
+ m = DMAP_TO_VM_PAGE(pc);
+ dump_drop_page(m->phys_addr);
+ vm_page_unwire_noq(m);
+ vm_page_free(m);
+}
+
+/*
+ * Returns a new PV entry, allocating a new PV chunk from the system when
+ * needed. If this PV chunk allocation fails and a PV list lock pointer was
+ * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is
+ * returned.
+ *
+ * The given PV list lock may be released.
+ */
+static pv_entry_t
+get_pv_entry(pmap_t pmap, struct rwlock **lockp)
+{
+ int bit, field;
+ pv_entry_t pv;
+ struct pv_chunk *pc;
+ vm_page_t m;
+
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
+retry:
+ pc = TAILQ_FIRST(&pmap->pm_pvchunk);
+ if (pc != NULL) {
+ for (field = 0; field < _NPCM; field++) {
+ if (pc->pc_map[field]) {
+ bit = ffsl(pc->pc_map[field]) - 1;
+ break;
+ }
+ }
+ if (field < _NPCM) {
+ pv = &pc->pc_pventry[field * 64 + bit];
+ pc->pc_map[field] &= ~(1ul << bit);
+ /* If this was the last item, move it to tail */
+ if (pc_is_full(pc)) {
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
+ pc_list);
+ }
+ PV_STAT(atomic_add_long(&pv_entry_count, 1));
+ PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
+ return (pv);
+ }
+ }
+ /* No free items, allocate another chunk */
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
+ if (m == NULL) {
+ if (lockp == NULL) {
+ PV_STAT(pc_chunk_tryfail++);
+ return (NULL);
+ }
+ m = reclaim_pv_chunk(pmap, lockp);
+ if (m == NULL)
+ goto retry;
+ }
+ PV_STAT(atomic_add_int(&pc_chunk_count, 1));
+ PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
+ dump_add_page(m->phys_addr);
+ pc = VM_PAGE_TO_DMAP(m);
+ pc->pc_pmap = pmap;
+ pc->pc_map[0] = PC_FREEN & ~1ul; /* preallocated bit 0 */
+ pc->pc_map[1] = PC_FREEN;
+ pc->pc_map[2] = PC_FREEL;
+ mtx_lock(&pv_chunks_mutex);
+ TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
+ mtx_unlock(&pv_chunks_mutex);
+ pv = &pc->pc_pventry[0];
+ TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
+ PV_STAT(atomic_add_long(&pv_entry_count, 1));
+ PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
+ return (pv);
+}
+
+/*
+ * Ensure that the number of spare PV entries in the specified pmap meets or
+ * exceeds the given count, "needed".
+ *
+ * The given PV list lock may be released.
+ */
+static void
+reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
+{
+ struct pch new_tail;
+ struct pv_chunk *pc;
+ vm_page_t m;
+ int avail, free;
+ bool reclaimed;
+
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
+
+ /*
+ * Newly allocated PV chunks must be stored in a private list until
+ * the required number of PV chunks have been allocated. Otherwise,
+ * reclaim_pv_chunk() could recycle one of these chunks. In
+ * contrast, these chunks must be added to the pmap upon allocation.
+ */
+ TAILQ_INIT(&new_tail);
+retry:
+ avail = 0;
+ TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
+ bit_count((bitstr_t *)pc->pc_map, 0,
+ sizeof(pc->pc_map) * NBBY, &free);
+ if (free == 0)
+ break;
+ avail += free;
+ if (avail >= needed)
+ break;
+ }
+ for (reclaimed = false; avail < needed; avail += _NPCPV) {
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
+ if (m == NULL) {
+ m = reclaim_pv_chunk(pmap, lockp);
+ if (m == NULL)
+ goto retry;
+ reclaimed = true;
+ }
+ PV_STAT(atomic_add_int(&pc_chunk_count, 1));
+ PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
+ dump_add_page(m->phys_addr);
+ pc = VM_PAGE_TO_DMAP(m);
+ pc->pc_pmap = pmap;
+ pc->pc_map[0] = PC_FREEN;
+ pc->pc_map[1] = PC_FREEN;
+ pc->pc_map[2] = PC_FREEL;
+ TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
+ TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
+ PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
+
+ /*
+ * The reclaim might have freed a chunk from the current pmap.
+ * If that chunk contained available entries, we need to
+ * re-count the number of available entries.
+ */
+ if (reclaimed)
+ goto retry;
+ }
+ if (!TAILQ_EMPTY(&new_tail)) {
+ mtx_lock(&pv_chunks_mutex);
+ TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
+ mtx_unlock(&pv_chunks_mutex);
+ }
+}
+
+/*
+ * First find and then remove the pv entry for the specified pmap and virtual
+ * address from the specified pv list. Returns the pv entry if found and NULL
+ * otherwise. This operation can be performed on pv lists for either 4KB or
+ * 2MB page mappings.
+ */
+static __inline pv_entry_t
+pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
+{
+ pv_entry_t pv;
+
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
+ if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
+ TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
+ break;
+ }
+ }
+ return (pv);
+}
+
+/*
+ * First find and then destroy the pv entry for the specified pmap and virtual
+ * address. This operation can be performed on pv lists for either 4KB or 2MB
+ * page mappings.
+ */
+static void
+pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
+{
+ pv_entry_t pv;
+
+ pv = pmap_pvh_remove(pvh, pmap, va);
+
+ KASSERT(pv != NULL, ("pmap_pvh_free: pv not found for %#lx", va));
+ free_pv_entry(pmap, pv);
+}
+
+/*
+ * Conditionally create the PV entry for a 4KB page mapping if the required
+ * memory can be allocated without resorting to reclamation.
+ */
+static bool
+pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ struct rwlock **lockp)
+{
+ pv_entry_t pv;
+
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ /* Pass NULL instead of the lock pointer to disable reclamation. */
+ if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
+ pv->pv_va = va;
+ CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
+ return (true);
+ } else
+ return (false);
+}
+
+/*
+ * After demotion from a 2MB page mapping to 512 4KB page mappings,
+ * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
+ * entries for each of the 4KB page mappings.
+ */
+static void __unused
+pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
+ struct rwlock **lockp)
+{
+ struct md_page *pvh;
+ struct pv_chunk *pc;
+ pv_entry_t pv;
+ vm_page_t m;
+ vm_offset_t va_last;
+ int bit, field;
+
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
+
+ /*
+ * Transfer the 2mpage's pv entry for this mapping to the first
+ * page's pv list. Once this transfer begins, the pv list lock
+ * must not be released until the last pv entry is reinstantiated.
+ */
+ pvh = pa_to_pvh(pa);
+ va &= ~L2_OFFSET;
+ pv = pmap_pvh_remove(pvh, pmap, va);
+ KASSERT(pv != NULL, ("pmap_pv_demote_l2: pv not found"));
+ m = PHYS_TO_VM_PAGE(pa);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
+ /* Instantiate the remaining 511 pv entries. */
+ PV_STAT(atomic_add_long(&pv_entry_allocs, Ln_ENTRIES - 1));
+ va_last = va + L2_SIZE - PAGE_SIZE;
+ for (;;) {
+ pc = TAILQ_FIRST(&pmap->pm_pvchunk);
+ KASSERT(!pc_is_full(pc), ("pmap_pv_demote_l2: missing spare"));
+ for (field = 0; field < _NPCM; field++) {
+ while (pc->pc_map[field] != 0) {
+ bit = ffsl(pc->pc_map[field]) - 1;
+ pc->pc_map[field] &= ~(1ul << bit);
+ pv = &pc->pc_pventry[field * 64 + bit];
+ va += PAGE_SIZE;
+ pv->pv_va = va;
+ m++;
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("pmap_pv_demote_l2: page %p is not managed", m));
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
+ if (va == va_last)
+ goto out;
+ }
+ }
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
+ }
+out:
+ if (pc_is_full(pc)) {
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
+ }
+ PV_STAT(atomic_add_long(&pv_entry_count, Ln_ENTRIES - 1));
+ PV_STAT(atomic_add_int(&pv_entry_spare, -(Ln_ENTRIES - 1)));
+}
+
+#if VM_NRESERVLEVEL > 0
+static void
+pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
+ struct rwlock **lockp)
+{
+ struct md_page *pvh;
+ pv_entry_t pv;
+ vm_page_t m;
+ vm_offset_t va_last;
+
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ KASSERT((pa & L2_OFFSET) == 0,
+ ("pmap_pv_promote_l2: misaligned pa %#lx", pa));
+
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
+
+ m = PHYS_TO_VM_PAGE(pa);
+ va = va & ~L2_OFFSET;
+ pv = pmap_pvh_remove(&m->md, pmap, va);
+ KASSERT(pv != NULL, ("pmap_pv_promote_l2: pv for %#lx not found", va));
+ pvh = pa_to_pvh(pa);
+ TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
+
+ va_last = va + L2_SIZE - PAGE_SIZE;
+ do {
+ m++;
+ va += PAGE_SIZE;
+ pmap_pvh_free(&m->md, pmap, va);
+ } while (va < va_last);
+}
+#endif /* VM_NRESERVLEVEL > 0 */
+
+/*
+ * Create the PV entry for a 2MB page mapping. Always returns true unless the
+ * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns
+ * false if the PV entry cannot be allocated without resorting to reclamation.
+ */
+static bool
+pmap_pv_insert_l2(pmap_t pmap, vm_offset_t va, pd_entry_t l2e, u_int flags,
+ struct rwlock **lockp)
+{
+ struct md_page *pvh;
+ pv_entry_t pv;
+ vm_paddr_t pa;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ /* Pass NULL instead of the lock pointer to disable reclamation. */
+ if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
+ NULL : lockp)) == NULL)
+ return (false);
+ pv->pv_va = va;
+ pa = PTE_TO_PHYS(l2e);
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
+ pvh = pa_to_pvh(pa);
+ TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
+ return (true);
+}
+
+static void
+pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)
+{
+ pt_entry_t newl2, oldl2 __diagused;
+ vm_page_t ml3;
+ vm_paddr_t ml3pa;
+
+ KASSERT(!VIRT_IN_DMAP(va), ("removing direct mapping of %#lx", va));
+ KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+ ml3 = pmap_remove_pt_page(pmap, va);
+ if (ml3 == NULL) {
+ /*
+ * No tracked L3 page-table page: this 2MB kernel mapping was
+ * entered directly by pmap_enter_l2() -- a fresh superpage that
+ * was never promoted from 4KB pages, so there is no PT page
+ * kept for demotion. pmap_remove_l2() already cleared the L2
+ * entry and invalidated the TLB; nothing more to do. This is
+ * the same "no PT page" case that pmap_demote_l2_locked()
+ * anticipates in its success path (it allocates a fresh PT page
+ * there). Without this, removing/demoting a direct kernel
+ * superpage panicked.
+ */
+ static int diag;
+ if (diag < 4)
+ printf("pmap_remove_kernel_l2: direct 2MB superpage at "
+ "va=%#lx (no tracked PT page) [%d]\n", va, ++diag);
+ return;
+ }
+
+ ml3pa = VM_PAGE_TO_PHYS(ml3);
+ newl2 = (ml3pa >> L3_SHIFT) << L3_SHIFT;
+
+ /*
+ * If this page table page was unmapped by a promotion, then it
+ * contains valid mappings. Zero it to invalidate those mappings.
+ */
+ if (vm_page_any_valid(ml3))
+ pagezero(PHYS_TO_DMAP(ml3pa));
+
+ /*
+ * Demote the mapping.
+ */
+ oldl2 = pmap_load_store(l2, newl2);
+ KASSERT(oldl2 == invalid_pmd_table,
+ ("%s: found existing mapping at %p: %#lx",
+ __func__, l2, oldl2));
+}
+
+/*
+ * pmap_remove_l2: Do the things to unmap a level 2 superpage.
+ */
+static int
+pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva,
+ pd_entry_t l1e, struct spglist *free, struct rwlock **lockp)
+{
+ struct md_page *pvh;
+ pt_entry_t oldl2;
+ vm_offset_t eva, va;
+ vm_page_t m, ml3;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ KASSERT((sva & L2_OFFSET) == 0, ("pmap_remove_l2: sva is not aligned"));
+ oldl2 = pmap_pmd_entry(l2);
+ pmap_store(l2, invalid_pmd_table);
+ KASSERT((oldl2 & PTE_HUGE),
+ ("pmap_remove_l2: L2e %lx is not a superpage mapping", oldl2));
+
+ /*
+ * An invtlb specifying a single address within a superpage mapping is
+ * sufficient. However, since we do not perform any invalidation upon
+ * promotion, TLBs may still be caching 4KB mappings within the
+ * superpage, so we must invalidate the entire range.
+ */
+ pmap_invalidate_range(pmap, sva, sva + L2_SIZE);
+
+ if ((oldl2 & PTE_SW_WIRED) != 0)
+ pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE;
+ pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE);
+
+ if ((oldl2 & PTE_SW_MANAGED) != 0) {
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, PTE_TO_PHYS(oldl2));
+ pvh = pa_to_pvh(PTE_TO_PHYS(oldl2));
+ pmap_pvh_free(pvh, pmap, sva);
+ eva = sva + L2_SIZE;
+ for (va = sva, m = PTE_TO_VM_PAGE(oldl2);
+ va < eva; va += PAGE_SIZE, m++) {
+ if ((oldl2 & PTE_D) != 0)
+ vm_page_dirty(m);
+ if ((oldl2 & PTE_A) != 0)
+ vm_page_aflag_set(m, PGA_REFERENCED);
+ if (TAILQ_EMPTY(&m->md.pv_list) &&
+ TAILQ_EMPTY(&pvh->pv_list))
+ vm_page_aflag_clear(m, PGA_WRITEABLE);
+ }
+ }
+ if (pmap == kernel_pmap) {
+ pmap_remove_kernel_l2(pmap, l2, sva);
+ } else {
+ ml3 = pmap_remove_pt_page(pmap, sva);
+ if (ml3 != NULL) {
+ KASSERT(vm_page_any_valid(ml3),
+ ("pmap_remove_l2: l3 page not promoted"));
+ pmap_resident_count_dec(pmap, 1);
+ KASSERT(ml3->ref_count == Ln_ENTRIES,
+ ("pmap_remove_l2: l3 page ref count error"));
+ ml3->ref_count = 1;
+ vm_page_unwire_noq(ml3);
+ pmap_add_delayed_free_list(ml3, free, false);
+ }
+ }
+ return (pmap_unuse_pt(pmap, sva, l1e, free));
+}
+
+/*
+ * pmap_remove_l3: do the things to unmap a page in a process
+ */
+static int
+pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,
+ pd_entry_t l2e, struct spglist *free, struct rwlock **lockp,
+ bool invalidate)
+{
+ struct md_page *pvh;
+ pt_entry_t old_l3;
+ vm_page_t m;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ old_l3 = pmap_load_clear(l3);
+ /*
+ * Caller-driven: pmap_remove batches invalidation across the whole run
+ * (pmap_invalidate_range/all in its outer loop), so it passes false to
+ * avoid a redundant whole-ASID shootdown per page. pmap_enter_l2 has
+ * no surrounding batch, so it passes true to invalidate each evicted
+ * 4KB entry here.
+ */
+ if (invalidate)
+ pmap_invalidate_page(pmap, va);
+
+ if (old_l3 & PTE_SW_WIRED)
+ pmap->pm_stats.wired_count -= 1;
+ pmap_resident_count_dec(pmap, 1);
+
+ if (old_l3 & PTE_SW_MANAGED) {
+ m = PTE_TO_VM_PAGE(old_l3);
+ if ((old_l3 & PTE_D) != 0)
+ vm_page_dirty(m);
+ if (old_l3 & PTE_A)
+ vm_page_aflag_set(m, PGA_REFERENCED);
+ CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
+ pmap_pvh_free(&m->md, pmap, va);
+ if (TAILQ_EMPTY(&m->md.pv_list) &&
+ (m->flags & PG_FICTITIOUS) == 0) {
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ if (TAILQ_EMPTY(&pvh->pv_list))
+ vm_page_aflag_clear(m, PGA_WRITEABLE);
+ }
+ }
+
+ return (pmap_unuse_pt(pmap, va, l2e, free));
+}
+
+/*
+ * Remove the given range of addresses from the specified map.
+ *
+ * It is assumed that the start and end are properly
+ * rounded to the page size.
+ */
+void
+pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+ struct spglist free;
+ struct rwlock *lock;
+ vm_offset_t va, va_next;
+ pd_entry_t *l0, *l1, *l2, l2e;
+ pt_entry_t *l3;
+ int nflush;
+ bool flush_all;
+
+ /*
+ * Perform an unsynchronized read. This is, however, safe.
+ */
+ if (pmap->pm_stats.resident_count == 0)
+ return;
+
+ SLIST_INIT(&free);
+
+ rw_rlock(&pvh_global_lock);
+ PMAP_LOCK(pmap);
+
+ lock = NULL;
+ nflush = 0;
+ flush_all = false;
+ for (; sva < eva; sva = va_next) {
+ if (pmap->pm_stats.resident_count == 0)
+ break;
+
+ l0 = pmap_l0(pmap, sva);
+ if (!pmap_pgd_valid(l0)) {
+ va_next = (sva + L0_SIZE) & ~L0_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+ continue;
+ }
+
+ l1 = pmap_l0_to_l1(l0, sva);
+
+ if (!pmap_pud_valid(l1)) {
+ va_next = (sva + L1_SIZE) & ~L1_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+ continue;
+ }
+
+ /*
+ * Calculate index for next page table.
+ */
+ va_next = (sva + L2_SIZE) & ~L2_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+
+ l2 = pmap_l1_to_l2(l1, sva);
+ if (!pmap_pmd_valid(l2))
+ continue;
+ l2e = pmap_pmd_entry(l2);
+ if (l2e & PTE_HUGE) {
+ if (sva + L2_SIZE == va_next && eva >= va_next) {
+ (void)pmap_remove_l2(pmap, l2, sva,
+ pmap_pud_entry(l1), &free, &lock);
+ continue;
+ } else if (!pmap_demote_l2_locked(pmap, l2, sva,
+ &lock)) {
+ /*
+ * The large page mapping was destroyed.
+ */
+ continue;
+ }
+ l2e = pmap_pmd_entry(l2);
+ }
+
+ /*
+ * Limit our scan to either the end of the va represented
+ * by the current page table page, or to the end of the
+ * range being removed.
+ */
+ if (va_next > eva)
+ va_next = eva;
+
+ va = va_next;
+ for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++,
+ sva += L3_SIZE) {
+ if (!pmap_pte_valid(l3)) {
+ if (va != va_next) {
+ nflush += (sva - va) / L3_SIZE;
+ if (pmap_tlb_ipi_threshold > 0 &&
+ nflush >= pmap_tlb_ipi_threshold)
+ flush_all = true;
+ if (!flush_all)
+ pmap_invalidate_range(pmap,
+ va, sva);
+ va = va_next;
+ }
+ continue;
+ }
+ if (va == va_next)
+ va = sva;
+ if (pmap_remove_l3(pmap, l3, sva, l2e, &free, &lock,
+ false)) {
+ sva += L3_SIZE;
+ break;
+ }
+ }
+ if (va != va_next) {
+ nflush += (sva - va) / L3_SIZE;
+ if (pmap_tlb_ipi_threshold > 0 &&
+ nflush >= pmap_tlb_ipi_threshold)
+ flush_all = true;
+ if (!flush_all)
+ pmap_invalidate_range(pmap, va, sva);
+ }
+ }
+ if (flush_all)
+ pmap_invalidate_all(pmap);
+ if (lock != NULL)
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+ vm_page_free_pages_toq(&free, false);
+}
+
+/*
+ * Routine: pmap_remove_all
+ * Function:
+ * Removes this physical page from
+ * all physical maps in which it resides.
+ * Reflects back modify bits to the pager.
+ *
+ * Notes:
+ * Original versions of this routine were very
+ * inefficient because they iteratively called
+ * pmap_remove (slow...)
+ */
+
+void
+pmap_remove_all(vm_page_t m)
+{
+ struct spglist free;
+ struct md_page *pvh;
+ pmap_t pmap;
+ pt_entry_t *l3, l3e;
+ pd_entry_t *l2, l2e __diagused;
+ pv_entry_t pv;
+ vm_offset_t va;
+
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("pmap_remove_all: page %p is not managed", m));
+ SLIST_INIT(&free);
+ pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
+ pa_to_pvh(VM_PAGE_TO_PHYS(m));
+
+ rw_wlock(&pvh_global_lock);
+ while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
+ pmap = PV_PMAP(pv);
+ PMAP_LOCK(pmap);
+ va = pv->pv_va;
+ l2 = pmap_l2(pmap, va);
+ (void)pmap_demote_l2(pmap, l2, va);
+ PMAP_UNLOCK(pmap);
+ }
+ while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
+ pmap = PV_PMAP(pv);
+ PMAP_LOCK(pmap);
+ pmap_resident_count_dec(pmap, 1);
+ l2 = pmap_l2(pmap, pv->pv_va);
+ KASSERT(l2 != NULL, ("pmap_remove_all: no l2 table found"));
+ l2e = pmap_pmd_entry(l2);
+
+ KASSERT(!(l2e & PTE_HUGE),
+ ("pmap_remove_all: found a superpage in %p's pv list", m));
+
+ l3 = pmap_l2_to_l3(l2, pv->pv_va);
+ l3e = pmap_load_clear(l3);
+ pmap_invalidate_page(pmap, pv->pv_va);
+ if (l3e & PTE_SW_WIRED)
+ pmap->pm_stats.wired_count--;
+ if ((l3e & PTE_A) != 0)
+ vm_page_aflag_set(m, PGA_REFERENCED);
+
+ /*
+ * Update the vm_page_t clean and reference bits.
+ */
+ if ((l3e & PTE_D) != 0)
+ vm_page_dirty(m);
+ pmap_unuse_pt(pmap, pv->pv_va, pmap_pmd_entry(l2), &free);
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
+ free_pv_entry(pmap, pv);
+ PMAP_UNLOCK(pmap);
+ }
+ vm_page_aflag_clear(m, PGA_WRITEABLE);
+ rw_wunlock(&pvh_global_lock);
+ vm_page_free_pages_toq(&free, false);
+}
+
+/*
+ * Set the physical protection on the
+ * specified range of this map as requested.
+ */
+void
+pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
+{
+ pd_entry_t *l0, *l1, *l2, l2e;
+ pt_entry_t *l3, l3e, mask, setmask;
+ vm_page_t m, mt;
+ vm_paddr_t pa;
+ vm_offset_t va_next;
+ bool anychanged, pv_lists_locked;
+
+ if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
+ pmap_remove(pmap, sva, eva);
+ return;
+ }
+
+ if ((prot & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
+ (VM_PROT_WRITE | VM_PROT_EXECUTE))
+ return;
+
+ anychanged = false;
+ pv_lists_locked = false;
+ mask = 0;
+ setmask = 0;
+ if ((prot & VM_PROT_WRITE) == 0)
+ mask |= PTE_W | PTE_D;
+ if ((prot & VM_PROT_EXECUTE) == 0)
+ setmask |= PTE_NX;
+ else
+ mask |= PTE_NX;
+resume:
+ PMAP_LOCK(pmap);
+ for (; sva < eva; sva = va_next) {
+ l0 = pmap_l0(pmap, sva);
+ if (!pmap_pgd_valid(l0)) {
+ va_next = (sva + L0_SIZE) & ~L0_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+ continue;
+ }
+
+ l1 = pmap_l0_to_l1(l0, sva);
+
+ if (!pmap_pud_valid(l1)) {
+ va_next = (sva + L1_SIZE) & ~L1_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+ continue;
+ }
+
+ va_next = (sva + L2_SIZE) & ~L2_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+
+ l2 = pmap_l1_to_l2(l1, sva);
+ if (!pmap_pmd_valid(l2))
+ continue;
+ l2e = pmap_pmd_entry(l2);
+ if (l2e & PTE_HUGE) {
+ if (sva + L2_SIZE == va_next && eva >= va_next) {
+retryl2:
+ if ((prot & VM_PROT_WRITE) == 0 &&
+ (l2e & (PTE_SW_MANAGED | PTE_D)) ==
+ (PTE_SW_MANAGED | PTE_D)) {
+ pa = PTE_TO_PHYS(l2e);
+ m = PHYS_TO_VM_PAGE(pa);
+ for (mt = m; mt < &m[Ln_ENTRIES]; mt++)
+ vm_page_dirty(mt);
+ }
+ if (!pmap_cmpset(l2, &l2e,
+ (l2e & ~mask) | setmask))
+ goto retryl2;
+ anychanged = true;
+ continue;
+ } else {
+ if (!pv_lists_locked) {
+ pv_lists_locked = true;
+ if (!rw_try_rlock(&pvh_global_lock)) {
+ if (anychanged)
+ pmap_invalidate_all(
+ pmap);
+ PMAP_UNLOCK(pmap);
+ rw_rlock(&pvh_global_lock);
+ goto resume;
+ }
+ }
+ if (!pmap_demote_l2(pmap, l2, sva)) {
+ /*
+ * The large page mapping was destroyed.
+ */
+ continue;
+ }
+ }
+ }
+
+ if (va_next > eva)
+ va_next = eva;
+
+ for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++,
+ sva += L3_SIZE) {
+ l3e = pmap_pte_entry(l3);
+retryl3:
+ if (!pmap_pte_valid(l3))
+ continue;
+ if ((prot & VM_PROT_WRITE) == 0 &&
+ (l3e & (PTE_SW_MANAGED | PTE_D)) ==
+ (PTE_SW_MANAGED | PTE_D)) {
+ m = PTE_TO_VM_PAGE(l3e);
+ vm_page_dirty(m);
+ }
+ if (!pmap_cmpset(l3, &l3e, (l3e & ~mask) | setmask))
+ goto retryl3;
+ anychanged = true;
+ }
+ }
+ if (anychanged)
+ pmap_invalidate_all(pmap);
+ if (pv_lists_locked)
+ rw_runlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+}
+
+/*
+ * Install the leaf PTE for va directly into the TLB, bypassing the hardware
+ * page walker.
+ * Most callers (pmap_enter, pmap_fault) hold PMAP_LOCK. pmap_kenter and
+ * pmap_qenter do not; they rely on the caller guaranteeing exclusive access
+ * to the VA range being mapped.
+ */
+bool
+pmap_update_tlb(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *leaf;
+ pt_entry_t elo0, elo1;
+ uint64_t ehi, idx, ps;
+ register_t flags;
+
+ if (pmap != kernel_pmap && pmap != PCPU_GET(curpmap))
+ goto out;
+
+ /*
+ * Walk to the faulting leaf and validate IT -- not its even partner. An
+ * odd-page fault frequently has the even partner unmapped; validating
+ * the partner would make us return without installing anything while
+ * pmap_fault() reports the fault resolved, so the access re-faults
+ * forever (observed as a hang at init). After validating the faulting
+ * leaf, step back to the even partner so tlbfill installs the correct
+ * even/odd pair (the partner slot may legitimately be invalid).
+ */
+ leaf = pmap_l1(pmap, va);
+ if (leaf != NULL && pmap_pud_valid(leaf) &&
+ (pmap_pud_entry(leaf) & PTE_HUGE) != 0) {
+ ps = 30;
+ if ((va & (1UL << 30)) != 0)
+ leaf--;
+ } else {
+ leaf = pmap_l2(pmap, va);
+ if (leaf == NULL || !pmap_pmd_valid(leaf))
+ goto out;
+ if ((pmap_pmd_entry(leaf) & PTE_HUGE) != 0) {
+ ps = 21;
+ if ((va & (1UL << 21)) != 0)
+ leaf--;
+ } else {
+ leaf = pmap_l3(pmap, va);
+ if (leaf == NULL || !pmap_pte_valid(leaf))
+ goto out;
+ ps = 12;
+ if ((va & L3_SIZE) != 0)
+ leaf--;
+ }
+ }
+ elo0 = pmap_load(leaf);
+ elo1 = pmap_load(leaf + 1);
+ if (ps > 12) {
+ if ((elo0 & PTE_HUGE_G) == 0)
+ elo0 &= ~PTE_G;
+
+ if ((elo1 & PTE_HUGE_G) == 0)
+ elo1 &= ~PTE_G;
+ }
+
+ flags = intr_disable();
+ ehi = va & ~((2UL << ps) - 1);
+ __asm __volatile("csrwr %0, %1" : "+r"(ehi) :
+ "i"(LOONGARCH_CSR_TLBEHI));
+ __asm __volatile("tlbsrch");
+ __asm __volatile("csrrd %0, %1" : "=r"(idx) :
+ "i"(LOONGARCH_CSR_TLBIDX));
+ idx = (idx & ~CSR_TLBIDX_PS) | (ps << CSR_TLBIDX_PS_SHIFT);
+ __asm __volatile("csrwr %0, %1" : "+r"(idx) :
+ "i"(LOONGARCH_CSR_TLBIDX));
+ __asm __volatile("csrwr %0, %1" : "+r"(elo0) :
+ "i"(LOONGARCH_CSR_TLBELO0));
+ __asm __volatile("csrwr %0, %1" : "+r"(elo1) :
+ "i"(LOONGARCH_CSR_TLBELO1));
+ if ((idx & CSR_TLBIDX_EHINV) != 0) {
+ __asm __volatile("tlbfill");
+ TLB_DBG("pmap_update_tlb: cpu%d pid=%d (%s) tlbfill"
+ " va=%#lx ps=%lu ELO0=%#lx ELO1=%#lx\n",
+ PCPU_GET(cpuid), curproc->p_pid, curproc->p_comm,
+ va, ps, (unsigned long)elo0, (unsigned long)elo1);
+ } else {
+ __asm __volatile("tlbwr");
+ TLB_DBG("pmap_update_tlb: cpu%d pid=%d (%s) tlbwr"
+ " va=%#lx ps=%lu ELO0=%#lx ELO1=%#lx\n",
+ PCPU_GET(cpuid), curproc->p_pid, curproc->p_comm,
+ va, ps, (unsigned long)elo0, (unsigned long)elo1);
+ }
+ intr_restore(flags);
+ return (true);
+
+out:
+ /* PTE gone — evict any stale TLB entry for va.
+ * invtlb's op (sub-code) is an asm "i" immediate, so the kernel/user
+ * op must be a compile-time constant -- branch in C, not via ?: . */
+ if (pmap == kernel_pmap)
+ invtlb(INVTLB_ADDR_GTRUE_OR_ASID, pmap_current_asid(), va);
+ else
+ invtlb(INVTLB_ADDR_GFALSE_AND_ASID,
+ pmap_get_asid(pmap, PCPU_GET(cpuid)), va);
+ return (false);
+}
+
+int
+pmap_fault(pmap_t pmap, vm_offset_t va, vm_prot_t ftype)
+{
+ pd_entry_t *l1, *l2, l2e;
+ pt_entry_t bits, *pte, oldpte;
+ int rv;
+
+ KASSERT(VIRT_IS_VALID(va), ("pmap_fault: invalid va %#lx", va));
+
+ rv = 0;
+ PMAP_LOCK(pmap);
+ l1 = pmap_l1(pmap, va);
+ if (l1 != NULL && pmap_pud_valid(l1) &&
+ (pmap_pud_entry(l1) & PTE_HUGE) != 0) {
+ /* 1GB leaf at L1 (e.g. the DMAP). */
+ pte = l1;
+ oldpte = pmap_pud_entry(l1);
+ } else {
+ l2 = pmap_l2(pmap, va);
+ if (l2 == NULL || !pmap_pmd_valid(l2))
+ goto done;
+ l2e = pmap_pmd_entry(l2);
+ if (!(l2e & PTE_HUGE)) {
+ pte = pmap_l2_to_l3(l2, va);
+ oldpte = pmap_pte_entry(pte);
+ if (!pmap_pte_valid(pte))
+ goto done;
+ } else {
+ pte = l2;
+ oldpte = l2e;
+ }
+ }
+
+ if ((pmap != kernel_pmap && (oldpte & PTE_U) == 0) ||
+ (ftype == VM_PROT_WRITE && (oldpte & PTE_W) == 0) ||
+ (ftype == VM_PROT_EXECUTE && (oldpte & PTE_NX) == PTE_NX) ||
+ (ftype == VM_PROT_READ && (oldpte & PTE_NR) == PTE_NR))
+ goto done;
+
+ bits = PTE_A;
+ if (ftype == VM_PROT_WRITE)
+ bits |= PTE_D;
+
+ /*
+ * Spurious faults can occur if the implementation caches invalid
+ * entries in the TLB, or if simultaneous accesses on multiple CPUs
+ * race with each other.
+ */
+ if ((oldpte & bits) != bits)
+ pmap_store_bits(pte, bits);
+
+ pmap_update_tlb(pmap, va);
+
+ rv = 1;
+done:
+ PMAP_UNLOCK(pmap);
+ /*
+ * Sync the i-cache on all cpus after updating the PTE
+ * if the new PTE is executable. Must be done outside the
+ * PMAP lock to avoid deadlock with remote CPUs that may be
+ * spinning on PV list locks with interrupts disabled.
+ */
+ if (ftype & VM_PROT_EXECUTE)
+ pmap_sync_icache(pmap, va, PAGE_SIZE);
+ return (rv);
+}
+
+/*
+ * Demote the specified L1 page to separate L2 pages.
+ * Currently only used for DMAP entries.
+ */
+static bool
+pmap_demote_l1(pmap_t pmap, pd_entry_t *l1, vm_offset_t va)
+{
+ vm_page_t m;
+ pt_entry_t *l2, oldl1, newl2;
+ pd_entry_t newl1;
+ vm_paddr_t l2phys;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+ oldl1 = pmap_pud_entry(l1);
+ KASSERT((oldl1 & PTE_HUGE) != 0,
+ ("pmap_demote_l1: oldl1 is not a leaf PTE"));
+ KASSERT((oldl1 & PTE_A) != 0,
+ ("pmap_demote_l1: oldl1 is missing PTE_A"));
+ KASSERT((oldl1 & (PTE_D | PTE_W)) != PTE_W,
+ ("pmap_demote_l1: not dirty!"));
+ KASSERT((oldl1 & PTE_SW_MANAGED) == 0,
+ ("pmap_demote_l1: L1 table shouldn't be managed"));
+ KASSERT(VIRT_IN_DMAP(va),
+ ("pmap_demote_l1: is unsupported for non-DMAP va=%#lx", va));
+
+ /* Demoting L1 means we need to allocate a new page-table page. */
+ m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED);
+ if (m == NULL) {
+ CTR2(KTR_PMAP, "pmap_demote_l1: failure for va %#lx in pmap %p",
+ va, pmap);
+ return (false);
+ }
+
+ l2phys = VM_PAGE_TO_PHYS(m);
+ l2 = (pt_entry_t *)PHYS_TO_DMAP(l2phys);
+
+ /*
+ * Create new entries, relying on the fact that only the low bits
+ * (index) of the physical address are changing.
+ */
+ newl2 = oldl1;
+ for (int i = 0; i < Ln_ENTRIES; i++)
+ pmap_store(&l2[i], newl2 | (i << L2_SHIFT));
+
+ /*
+ * And update the L1 entry.
+ *
+ * NB: flushing the TLB is the responsibility of the caller. Cached
+ * translations are still "correct" for demoted mappings until some
+ * subset of the demoted range is modified.
+ */
+ newl1 = ((l2phys >> PAGE_PFN_SHIFT) << L3_SHIFT);
+ pmap_store(l1, newl1);
+
+ counter_u64_add(pmap_l1_demotions, 1);
+ CTR2(KTR_PMAP, "pmap_demote_l1: success for va %#lx in pmap %p",
+ va, pmap);
+ return (true);
+}
+
+static bool
+pmap_demote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va)
+{
+ struct rwlock *lock;
+ bool rv;
+
+ lock = NULL;
+ rv = pmap_demote_l2_locked(pmap, l2, va, &lock);
+ if (lock != NULL)
+ rw_wunlock(lock);
+ return (rv);
+}
+
+/*
+ * Tries to demote a 2MB page mapping. If demotion fails, the 2MB page
+ * mapping is invalidated.
+ */
+static bool
+pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2, vm_offset_t va,
+ struct rwlock **lockp)
+{
+ struct spglist free;
+ vm_page_t mpte;
+ pd_entry_t newl2, oldl2;
+ pt_entry_t *firstl3, newl3;
+ vm_paddr_t mptepa;
+ int i;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+ oldl2 = pmap_pmd_entry(l2);
+ KASSERT((oldl2 & PTE_HUGE),
+ ("pmap_demote_l2_locked: oldl2 is not a leaf entry"));
+ if ((oldl2 & PTE_A) == 0 ||
+ (mpte = pmap_remove_pt_page(pmap, va)) == NULL) {
+ KASSERT((oldl2 & PTE_SW_WIRED) == 0,
+ ("pmap_demote_l2_locked: page table page for a "
+ "wired mapping is missing"));
+ if ((oldl2 & PTE_A) == 0 || (mpte = vm_page_alloc_noobj(
+ (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : 0) |
+ VM_ALLOC_WIRED)) == NULL) {
+ SLIST_INIT(&free);
+ pmap_remove_l2(pmap, l2, va & ~L2_OFFSET,
+ pmap_pud_entry(pmap_l1(pmap, va)), &free, lockp);
+ vm_page_free_pages_toq(&free, true);
+ CTR2(KTR_PMAP, "pmap_demote_l2_locked: "
+ "failure for va %#lx in pmap %p", va, pmap);
+ return (false);
+ }
+ mpte->pindex = pmap_l2_pindex(va);
+ if (va < VM_MAXUSER_ADDRESS) {
+ mpte->ref_count = Ln_ENTRIES;
+ pmap_resident_count_inc(pmap, 1);
+ }
+ }
+ mptepa = VM_PAGE_TO_PHYS(mpte);
+ firstl3 = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
+ newl2 = ((mptepa >> PAGE_PFN_SHIFT) << L3_SHIFT);
+ KASSERT((oldl2 & PTE_A) != 0,
+ ("pmap_demote_l2_locked: oldl2 is missing PTE_A"));
+ KASSERT((oldl2 & (PTE_D | PTE_W)) != PTE_W,
+ ("pmap_demote_l2_locked: oldl2 is missing PTE_D"));
+
+ newl3 = oldl2;
+ newl3 &= ~PTE_HUGE;
+ newl3 &= ~PTE_HUGE_G;
+ if (oldl2 & PTE_HUGE_G) {
+ newl3 |= PTE_G;
+ }
+
+ /*
+ * If the page table page is not leftover from an earlier promotion,
+ * initialize it.
+ */
+ if (!vm_page_all_valid(mpte)) {
+ for (i = 0; i < Ln_ENTRIES; i++)
+ pmap_store(firstl3 + i, newl3 + (i << L3_SHIFT));
+ }
+ KASSERT(PTE_TO_PHYS(pmap_pte_entry(firstl3)) == PTE_TO_PHYS(newl3),
+ ("pmap_demote_l2_locked: firstl3 and newl3 map different physical "
+ "addresses"));
+
+ /*
+ * If the mapping has changed attributes, update the PTEs.
+ */
+ if ((pmap_pte_entry(firstl3) & PTE_PROMOTE) != (newl3 & PTE_PROMOTE))
+ for (i = 0; i < Ln_ENTRIES; i++)
+ pmap_store(firstl3 + i, newl3 + (i << L3_SHIFT));
+
+ /*
+ * The spare PV entries must be reserved prior to demoting the
+ * mapping, that is, prior to changing the L2 entry. Otherwise, the
+ * state of the L2 entry and the PV lists will be inconsistent, which
+ * can result in reclaim_pv_chunk() attempting to remove a PV entry from
+ * the wrong PV list and pmap_pv_demote_l2() failing to find the
+ * expected PV entry for the 2MB page mapping that is being demoted.
+ */
+
+ if ((oldl2 & PTE_SW_MANAGED) != 0)
+ reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp);
+
+ /*
+ * Demote the mapping. Unlike arm64, we cannot use break-before-
+ * make here: LoongArch directory entries are bare physical
+ * addresses with no valid bit, so writing the sentinel makes the
+ * HW page walker descend into the sentinel page and fault. If
+ * the faulting address is a kernel mapping whose pmap lock we
+ * already hold, that fault deadlocks. Instead, atomically replace
+ * the L2 entry and invalidate the stale superpage TLB afterwards.
+ * The new L3 table was fully populated above, so any concurrent
+ * walker observes a correct 4KB mapping once it reloads the L2.
+ */
+ pmap_store(l2, newl2);
+ pmap_invalidate_range(pmap, va & ~L2_OFFSET,
+ (va & ~L2_OFFSET) + L2_SIZE);
+
+ /*
+ * Demote the PV entry.
+ */
+ if ((oldl2 & PTE_SW_MANAGED) != 0)
+ pmap_pv_demote_l2(pmap, va, PTE_TO_PHYS(oldl2), lockp);
+
+ atomic_add_long(&pmap_l2_demotions, 1);
+ CTR2(KTR_PMAP, "pmap_demote_l2_locked: success for va %#lx in pmap %p",
+ va, pmap);
+ return (true);
+}
+
+#if VM_NRESERVLEVEL > 0
+static bool
+pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t ml3,
+ struct rwlock **lockp)
+{
+ pt_entry_t *firstl3, firstl3e, *l3, l3e;
+ pt_entry_t newl2, l3_all_PTE_A;
+ vm_paddr_t pa;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ if (!pmap_ps_enabled(pmap))
+ return (false);
+
+ KASSERT(!(pmap_pmd_entry(l2) & PTE_HUGE),
+ ("pmap_promote_l2: invalid l2 entry %p", l2));
+
+ /*
+ * Examine the first L3E in the specified PTP. Abort if this L3E is
+ * ineligible for promotion or does not map the first 4KB physical page
+ * within a 2MB page.
+ */
+ firstl3 = (pt_entry_t *)PHYS_TO_DMAP(PTE_TO_PHYS(pmap_pmd_entry(l2)));
+ firstl3e = pmap_pte_entry(firstl3);
+ pa = PTE_TO_PHYS(firstl3e);
+ if ((pa & L2_OFFSET) != 0) {
+ CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p",
+ va, pmap);
+ atomic_add_long(&pmap_l2_p_failures, 1);
+ return (false);
+ }
+
+ /*
+ * Downgrade a clean, writable mapping to read-only to ensure that the
+ * hardware does not set PTE_D while we are comparing PTEs.
+ *
+ * Upon a write access to a clean mapping, the implementation will
+ * either atomically check protections and set PTE_D, or raise a page
+ * fault. In the latter case, the pmap lock provides atomicity. Thus,
+ * we do not issue an invtlb here and instead rely on pmap_fault()
+ * to do so lazily.
+ */
+ while ((firstl3e & (PTE_W | PTE_D)) == PTE_W) {
+ if (pmap_cmpset(firstl3, &firstl3e, firstl3e & ~PTE_W)) {
+ firstl3e &= ~PTE_W;
+ break;
+ }
+ }
+
+ pa += L2_SIZE - PAGE_SIZE;
+ l3_all_PTE_A = firstl3e & PTE_A;
+ for (l3 = firstl3 + Ln_ENTRIES - 1; l3 > firstl3; l3--) {
+ l3e = pmap_pte_entry(l3);
+ if (PTE_TO_PHYS(l3e) != pa) {
+ CTR2(KTR_PMAP,
+ "pmap_promote_l2: failure for va %#lx pmap %p",
+ va, pmap);
+ atomic_add_long(&pmap_l2_p_failures, 1);
+ return (false);
+ }
+ while ((l3e & (PTE_W | PTE_D)) == PTE_W) {
+ if (pmap_cmpset(l3, &l3e, l3e & ~PTE_W)) {
+ l3e &= ~PTE_W;
+ break;
+ }
+ }
+ if ((l3e & PTE_PROMOTE) != (firstl3e & PTE_PROMOTE)) {
+ CTR2(KTR_PMAP,
+ "pmap_promote_l2: failure for va %#lx pmap %p",
+ va, pmap);
+ atomic_add_long(&pmap_l2_p_failures, 1);
+ return (false);
+ }
+ pa -= PAGE_SIZE;
+ l3_all_PTE_A &= l3e;
+ }
+
+ /*
+ * Unless all PTEs have PTE_A set, clear it from the superpage
+ * mapping, so that promotions triggered by speculative mappings,
+ * such as pmap_enter_quick(), don't automatically mark the
+ * underlying pages as referenced.
+ */
+ firstl3e &= ~PTE_A | l3_all_PTE_A;
+
+ /*
+ * Save the page table page in its current state until the L2
+ * mapping the superpage is demoted by pmap_demote_l2() or
+ * destroyed by pmap_remove_l3().
+ */
+ if (ml3 == NULL)
+ ml3 = PTE_TO_VM_PAGE(pmap_pmd_entry(l2));
+ KASSERT(ml3->pindex == pmap_l2_pindex(va),
+ ("pmap_promote_l2: page table page's pindex is wrong"));
+ if (pmap_insert_pt_page(pmap, ml3, true, l3_all_PTE_A != 0)) {
+ CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p",
+ va, pmap);
+ atomic_add_long(&pmap_l2_p_failures, 1);
+ return (false);
+ }
+
+ if ((firstl3e & PTE_SW_MANAGED) != 0)
+ pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(firstl3e), lockp);
+
+ /*
+ * Promote an L3 leaf PTE into a 2MiB L2 leaf.
+ * LoongArch's L2-leaf marker (PTE_HUGE, bit 6) is NOT present
+ * on L3 entries, so it must be added here; otherwise the
+ * promoted L2 entry is mistaken for a page-table pointer.
+ * Convert the L3 global bit (PTE_G) to the huge-page global
+ * marker (PTE_HUGE_G), the inverse of pmap_demote_l2_locked.
+ */
+
+ newl2 = firstl3e;
+
+ if ((firstl3e & PTE_G) != 0) {
+ newl2 &= ~PTE_G;
+ newl2 |= PTE_HUGE_G;
+ KASSERT(va >= VM_MIN_KERNEL_ADDRESS,
+ ("user pmap with G bit, pmap: %p, va: 0x%lx\n",
+ pmap, va));
+ }
+ newl2 |= PTE_HUGE;
+
+ pmap_store(l2, newl2);
+ KASSERT((pmap_pmd_entry(l2) & PTE_HUGE) != 0,
+ ("pmap_promote_l2: promoted L2 leaf missing PTE_HUGE, "
+ "va %#lx l2e %#lx", va, pmap_pmd_entry(l2)));
+
+ pmap_invalidate_range(pmap, va & ~L2_OFFSET,
+ (va & ~L2_OFFSET) + L2_SIZE);
+
+ atomic_add_long(&pmap_l2_promotions, 1);
+ CTR2(KTR_PMAP, "pmap_promote_l2: success for va %#lx in pmap %p", va,
+ pmap);
+
+ return (true);
+}
+#endif
+
+/*
+ * Insert the given physical page (p) at
+ * the specified virtual address (v) in the
+ * target physical map with the protection requested.
+ *
+ * If specified, the page will be wired down, meaning
+ * that the related pte can not be reclaimed.
+ *
+ * NB: This is the only routine which MAY NOT lazy-evaluate
+ * or lose information. That is, this routine must actually
+ * insert this page into the given map NOW.
+ */
+
+int
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
+ u_int flags, int8_t psind)
+{
+ struct rwlock *lock;
+ pd_entry_t *l2;
+ pt_entry_t new_l3, orig_l3;
+ pt_entry_t *l3;
+ pv_entry_t pv;
+ vm_paddr_t opa, pa;
+ vm_page_t mpte, om;
+ pn_t pn;
+ int rv;
+ bool nosleep;
+
+ va = trunc_page(va);
+ if ((m->oflags & VPO_UNMANAGED) == 0)
+ VM_PAGE_OBJECT_BUSY_ASSERT(m);
+ pa = VM_PAGE_TO_PHYS(m);
+ pn = pa >> PAGE_PFN_SHIFT;
+
+ new_l3 = PTE_V | PTE_P | PTE_CC | PTE_NX | PTE_K;
+ if (prot & VM_PROT_EXECUTE)
+ new_l3 &= ~PTE_NX;
+ if (flags & VM_PROT_WRITE)
+ new_l3 |= PTE_D;
+ if (prot & VM_PROT_WRITE)
+ new_l3 |= PTE_W;
+ if (va < VM_MAX_USER_ADDRESS)
+ new_l3 |= PTE_U;
+
+ /* for kernel mapping, add G bit */
+ if (pmap == kernel_pmap)
+ new_l3 |= PTE_G;
+
+ new_l3 |= (pn << L3_SHIFT);
+ if ((flags & PMAP_ENTER_WIRED) != 0)
+ new_l3 |= PTE_SW_WIRED;
+
+ /*
+ * Set modified bit gratuitously for writeable mappings if
+ * the page is unmanaged. We do not want to take a fault
+ * to do the dirty bit accounting for these mappings.
+ */
+ if ((m->oflags & VPO_UNMANAGED) != 0) {
+ if (prot & VM_PROT_WRITE)
+ new_l3 |= PTE_D;
+ } else
+ new_l3 |= PTE_SW_MANAGED;
+
+ CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa);
+
+ lock = NULL;
+ mpte = NULL;
+ rw_rlock(&pvh_global_lock);
+ PMAP_LOCK(pmap);
+ if (psind == 1) {
+ /* Assert the required virtual and physical alignment. */
+ KASSERT((va & L2_OFFSET) == 0,
+ ("pmap_enter: va %#lx unaligned", va));
+ KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
+ rv = pmap_enter_l2(pmap, va, new_l3, flags, m, &lock);
+ goto out;
+ }
+
+ l2 = pmap_l2(pmap, va);
+ if (l2 != NULL && pmap_pmd_valid(l2) &&
+ (!(pmap_pmd_entry(l2) & PTE_HUGE) ||
+ pmap_demote_l2_locked(pmap, l2, va, &lock))) {
+ l3 = pmap_l2_to_l3(l2, va);
+ if (va < VM_MAXUSER_ADDRESS) {
+ mpte = PTE_TO_VM_PAGE(pmap_pmd_entry(l2));
+ mpte->ref_count++;
+ }
+ } else if (va < VM_MAXUSER_ADDRESS) {
+ nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
+ mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock);
+ if (mpte == NULL && nosleep) {
+ CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
+ if (lock != NULL)
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+ l3 = pmap_l3(pmap, va);
+ } else {
+ panic("pmap_enter: missing L3 table for kernel va %#lx", va);
+ }
+
+ if (l3 == NULL) {
+ panic("L3 is null: va: 0x%lx\n", va);
+ }
+ orig_l3 = pmap_pte_entry(l3);
+ opa = PTE_TO_PHYS(orig_l3);
+ pv = NULL;
+
+ /*
+ * Is the specified virtual address already mapped?
+ */
+ if (pmap_pte_valid(l3)) {
+ /*
+ * Wiring change, just update stats. We don't worry about
+ * wiring PT pages as they remain resident as long as there
+ * are valid mappings in them. Hence, if a user page is wired,
+ * the PT page will be also.
+ */
+ if ((flags & PMAP_ENTER_WIRED) != 0 &&
+ (orig_l3 & PTE_SW_WIRED) == 0)
+ pmap->pm_stats.wired_count++;
+ else if ((flags & PMAP_ENTER_WIRED) == 0 &&
+ (orig_l3 & PTE_SW_WIRED) != 0)
+ pmap->pm_stats.wired_count--;
+
+ /*
+ * Remove the extra PT page reference.
+ */
+ if (mpte != NULL) {
+ mpte->ref_count--;
+ KASSERT(mpte->ref_count > 0,
+ ("pmap_enter: missing reference to page table page,"
+ " va: 0x%lx", va));
+ }
+
+ /*
+ * Has the physical page changed?
+ */
+ if (opa == pa) {
+ /*
+ * No, might be a protection or wiring change.
+ */
+ if ((orig_l3 & PTE_SW_MANAGED) != 0 &&
+ (new_l3 & PTE_W) != 0)
+ vm_page_aflag_set(m, PGA_WRITEABLE);
+ goto validate;
+ }
+
+ /*
+ * The physical page has changed. Temporarily invalidate
+ * the mapping. This ensures that all threads sharing the
+ * pmap keep a consistent view of the mapping, which is
+ * necessary for the correct handling of COW faults. It
+ * also permits reuse of the old mapping's PV entry,
+ * avoiding an allocation.
+ *
+ * For consistency, handle unmanaged mappings the same way.
+ */
+ orig_l3 = pmap_load_clear(l3);
+ KASSERT(PTE_TO_PHYS(orig_l3) == opa,
+ ("pmap_enter: unexpected pa update for %#lx", va));
+ if ((orig_l3 & PTE_SW_MANAGED) != 0) {
+ om = PHYS_TO_VM_PAGE(opa);
+
+ /*
+ * The pmap lock is sufficient to synchronize with
+ * concurrent calls to pmap_page_test_mappings() and
+ * pmap_ts_referenced().
+ */
+ if ((orig_l3 & PTE_D) != 0)
+ vm_page_dirty(om);
+ if ((orig_l3 & PTE_A) != 0)
+ vm_page_aflag_set(om, PGA_REFERENCED);
+ CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
+ pv = pmap_pvh_remove(&om->md, pmap, va);
+ KASSERT(pv != NULL,
+ ("pmap_enter: no PV entry for %#lx", va));
+ if ((new_l3 & PTE_SW_MANAGED) == 0)
+ free_pv_entry(pmap, pv);
+ if ((om->a.flags & PGA_WRITEABLE) != 0 &&
+ TAILQ_EMPTY(&om->md.pv_list) &&
+ ((om->flags & PG_FICTITIOUS) != 0 ||
+ TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
+ vm_page_aflag_clear(om, PGA_WRITEABLE);
+ }
+ pmap_invalidate_page(pmap, va);
+ orig_l3 = 0;
+ } else {
+ /*
+ * Increment the counters.
+ */
+ if ((new_l3 & PTE_SW_WIRED) != 0)
+ pmap->pm_stats.wired_count++;
+ pmap_resident_count_inc(pmap, 1);
+ }
+ /*
+ * Enter on the PV list if part of our managed memory.
+ */
+ if ((new_l3 & PTE_SW_MANAGED) != 0) {
+ if (pv == NULL) {
+ pv = get_pv_entry(pmap, &lock);
+ pv->pv_va = va;
+ }
+ CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
+ if ((new_l3 & PTE_W) != 0)
+ vm_page_aflag_set(m, PGA_WRITEABLE);
+ }
+
+validate:
+ /*
+ * Update the L3 entry.
+ */
+ if (pmap_pte_valid(l3)) {
+ orig_l3 = pmap_load_store(l3, new_l3);
+ pmap_invalidate_page(pmap, va);
+ KASSERT(PTE_TO_PHYS(orig_l3) == pa,
+ ("pmap_enter: invalid update"));
+ if ((orig_l3 & (PTE_D | PTE_SW_MANAGED)) ==
+ (PTE_D | PTE_SW_MANAGED))
+ vm_page_dirty(m);
+ } else {
+ pmap_store(l3, new_l3);
+ pmap_invalidate_page(pmap, va);
+ }
+
+#if VM_NRESERVLEVEL > 0
+ if (mpte != NULL && mpte->ref_count == Ln_ENTRIES &&
+ (m->flags & PG_FICTITIOUS) == 0 &&
+ vm_reserv_level_iffullpop(m) == 0)
+ (void)pmap_promote_l2(pmap, l2, va, mpte, &lock);
+#endif
+
+ rv = KERN_SUCCESS;
+out:
+ if (lock != NULL)
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
+
+ /*
+ * Install the TLB entry while holding PMAP_LOCK. If this runs
+ * outside the lock, a concurrent pmap_enter on another CPU can
+ * modify the PTE and shoot down this CPU between the PTE read
+ * (pmap_load) and the tlbfill/tlbwr — installing a stale PA.
+ */
+ pmap_update_tlb(pmap, va);
+
+ PMAP_UNLOCK(pmap);
+
+ /*
+ * Sync the i-cache on all cpus after updating the PTE
+ * if the new PTE is executable. Must be done outside the
+ * PMAP lock to avoid deadlock with remote CPUs that may be
+ * spinning on PV list locks with interrupts disabled.
+ */
+ if (prot & VM_PROT_EXECUTE)
+ pmap_sync_icache(pmap, va, PAGE_SIZE);
+ return (rv);
+}
+
+/*
+ * Release a page table page reference after a failed attempt to create a
+ * mapping.
+ */
+static void
+pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t l2pg)
+{
+ struct spglist free;
+
+ SLIST_INIT(&free);
+ if (pmap_unwire_ptp(pmap, va, l2pg, &free)) {
+ /*
+ * Although "va" is not mapped, paging-structure
+ * caches could nonetheless have entries that
+ * refer to the freed page table pages.
+ * Invalidate those entries.
+ */
+ pmap_invalidate_page(pmap, va);
+ vm_page_free_pages_toq(&free, true);
+ }
+}
+
+/*
+ * Tries to create a read- and/or execute-only 2MB page mapping. Returns
+ * KERN_SUCCESS if the mapping was created. Otherwise, returns an error
+ * value. See pmap_enter_l2() for the possible error values when "no sleep",
+ * "no replace", and "no reclaim" are specified.
+ */
+static int
+pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
+ struct rwlock **lockp)
+{
+ pd_entry_t new_l2;
+ pn_t pn;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+ pn = VM_PAGE_TO_PHYS(m) >> PAGE_PFN_SHIFT;
+ new_l2 = (pd_entry_t)((pn << L3_SHIFT) |
+ PTE_V | PTE_CC | PTE_P | PTE_NX);
+ if ((m->oflags & VPO_UNMANAGED) == 0)
+ new_l2 |= PTE_SW_MANAGED;
+ if ((prot & VM_PROT_EXECUTE) != 0)
+ new_l2 &= ~PTE_NX;
+ if (va < VM_MAXUSER_ADDRESS)
+ new_l2 |= PTE_U;
+ if (pmap == kernel_pmap)
+ new_l2 |= PTE_G;
+ return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP |
+ PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp));
+}
+
+/*
+ * Returns true if every page table entry in the specified page table is
+ * zero.
+ */
+static bool
+pmap_every_pte_zero(vm_paddr_t pa)
+{
+ pt_entry_t *pt_end, *pte;
+
+ KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned"));
+ pte = (pt_entry_t *)PHYS_TO_DMAP(pa);
+ for (pt_end = pte + Ln_ENTRIES; pte < pt_end; pte++) {
+ if (*pte != 0)
+ return (false);
+ }
+ return (true);
+}
+
+/*
+ * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
+ * the mapping was created, and one of KERN_FAILURE, KERN_NO_SPACE, or
+ * KERN_RESOURCE_SHORTAGE otherwise. Returns KERN_FAILURE if
+ * PMAP_ENTER_NOREPLACE was specified and a 4KB page mapping already exists
+ * within the 2MB virtual address range starting at the specified virtual
+ * address. Returns KERN_NO_SPACE if PMAP_ENTER_NOREPLACE was specified and a
+ * 2MB page mapping already exists at the specified virtual address. Returns
+ * KERN_RESOURCE_SHORTAGE if either (1) PMAP_ENTER_NOSLEEP was specified and a
+ * page table page allocation failed or (2) PMAP_ENTER_NORECLAIM was specified
+ * and a PV entry allocation failed.
+ *
+ * The parameter "m" is only used when creating a managed, writeable mapping.
+ */
+static int
+pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,
+ vm_page_t m, struct rwlock **lockp)
+{
+ struct spglist free;
+ pd_entry_t *l2, *l3, oldl2;
+ vm_offset_t sva;
+ vm_page_t l2pg, mt;
+ vm_page_t uwptpg;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+ if ((l2pg = pmap_alloc_l2(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
+ NULL : lockp)) == NULL) {
+ CTR2(KTR_PMAP, "pmap_enter_l2: failed to allocate PT page"
+ " for va %#lx in pmap %p", va, pmap);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+
+ l2 = VM_PAGE_TO_DMAP(l2pg);
+ l2 = &l2[pmap_l2_index(va)];
+ if (pmap_pmd_valid(l2)) {
+ oldl2 = pmap_pmd_entry(l2);
+ KASSERT(l2pg->ref_count > 1,
+ ("pmap_enter_l2: l2pg's ref count is too low"));
+ if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
+ if (oldl2 & PTE_HUGE) {
+ l2pg->ref_count--;
+ CTR2(KTR_PMAP,
+ "pmap_enter_l2: no space for va %#lx"
+ " in pmap %p", va, pmap);
+ return (KERN_NO_SPACE);
+ } else if (va < VM_MAXUSER_ADDRESS ||
+ !pmap_every_pte_zero(L2PTE_TO_PHYS(oldl2))) {
+ l2pg->ref_count--;
+ CTR2(KTR_PMAP, "pmap_enter_l2:"
+ " failed to replace existing mapping"
+ " for va %#lx in pmap %p", va, pmap);
+ return (KERN_FAILURE);
+ }
+ }
+ SLIST_INIT(&free);
+ if (oldl2 & PTE_HUGE)
+ (void)pmap_remove_l2(pmap, l2, va,
+ pmap_pud_entry(pmap_l1(pmap, va)), &free, lockp);
+ else
+ for (sva = va; sva < va + L2_SIZE; sva += PAGE_SIZE) {
+ l3 = pmap_l2_to_l3(l2, sva);
+ if (pmap_pte_valid(l3) &&
+ pmap_remove_l3(pmap, l3, sva, oldl2, &free,
+ lockp, true) != 0)
+ break;
+ }
+ vm_page_free_pages_toq(&free, true);
+ if (va >= VM_MAXUSER_ADDRESS) {
+ /*
+ * Both pmap_remove_l2() and pmap_remove_l3() will
+ * leave the kernel page table page zero filled.
+ */
+ mt = PTE_TO_VM_PAGE(pmap_pmd_entry(l2));
+ if (pmap_insert_pt_page(pmap, mt, false, false))
+ panic("pmap_enter_l2: trie insert failed");
+ } else
+ KASSERT(!pmap_pmd_valid(l2),
+ ("pmap_enter_l2: non-zero L2 entry %p", l2));
+ }
+
+ /*
+ * Allocate leaf ptpage for wired userspace pages.
+ */
+ uwptpg = NULL;
+ if ((new_l2 & PTE_SW_WIRED) != 0 && pmap != kernel_pmap) {
+ uwptpg = vm_page_alloc_noobj(VM_ALLOC_WIRED);
+ if (uwptpg == NULL) {
+ pmap_abort_ptp(pmap, va, l2pg);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+ uwptpg->pindex = pmap_l2_pindex(va);
+ if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {
+ vm_page_unwire_noq(uwptpg);
+ vm_page_free(uwptpg);
+ pmap_abort_ptp(pmap, va, l2pg);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+ pmap_resident_count_inc(pmap, 1);
+ uwptpg->ref_count = Ln_ENTRIES;
+ }
+
+ if ((new_l2 & PTE_SW_MANAGED) != 0) {
+ /*
+ * Abort this mapping if its PV entry could not be created.
+ */
+ if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) {
+ pmap_abort_ptp(pmap, va, l2pg);
+ if (uwptpg != NULL) {
+ mt = pmap_remove_pt_page(pmap, va);
+ KASSERT(mt == uwptpg,
+ ("removed pt page %p, expected %p", mt,
+ uwptpg));
+ pmap_resident_count_dec(pmap, 1);
+ uwptpg->ref_count = 1;
+ vm_page_unwire_noq(uwptpg);
+ vm_page_free(uwptpg);
+ }
+ CTR2(KTR_PMAP,
+ "pmap_enter_l2: failed to create PV entry"
+ " for va %#lx in pmap %p", va, pmap);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+ if ((new_l2 & PTE_W) != 0)
+ for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++)
+ vm_page_aflag_set(mt, PGA_WRITEABLE);
+ }
+
+ /*
+ * Increment counters.
+ */
+ if ((new_l2 & PTE_SW_WIRED) != 0)
+ pmap->pm_stats.wired_count += L2_SIZE / PAGE_SIZE;
+ pmap->pm_stats.resident_count += L2_SIZE / PAGE_SIZE;
+
+ /*
+ * Map the superpage.
+ *
+ * PTE_HUGE (bit 6) is the marker that tells the LoongArch HW page
+ * walker that this L2 entry is a 2MB leaf, not a directory pointer to
+ * an L3 table. Without it, the walker follows the PFN as a bare-phys
+ * directory pointer -- reading the data page's contents as L3 PTEs,
+ * producing the garbage-TLB corruption.
+ */
+
+ if (new_l2 & PTE_G) {
+ new_l2 &= ~PTE_G;
+ new_l2 |= PTE_HUGE_G;
+ KASSERT(va >= VM_MIN_KERNEL_ADDRESS,
+ ("user pmap with G bit, pmap: %p, va: 0x%lx\n",
+ pmap, va));
+ }
+ new_l2 |= PTE_HUGE;
+
+ pmap_store(l2, new_l2);
+ pmap_invalidate_page(pmap, va);
+
+ atomic_add_long(&pmap_l2_mappings, 1);
+ CTR2(KTR_PMAP, "pmap_enter_l2: success for va %#lx in pmap %p",
+ va, pmap);
+
+ return (KERN_SUCCESS);
+}
+
+/*
+ * Maps a sequence of resident pages belonging to the same object.
+ * The sequence begins with the given page m_start. This page is
+ * mapped at the given virtual address start. Each subsequent page is
+ * mapped at a virtual address that is offset from start by the same
+ * amount as the page is offset from m_start within the object. The
+ * last page in the sequence is the page with the largest offset from
+ * m_start that can be mapped at a virtual address less than the given
+ * virtual address end. Not every virtual page between start and end
+ * is mapped; only those for which a resident page exists with the
+ * corresponding offset from m_start are mapped.
+ */
+void
+pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
+ vm_page_t m_start, vm_prot_t prot)
+{
+ struct pctrie_iter pages;
+ struct rwlock *lock;
+ vm_offset_t va;
+ vm_page_t m, mpte;
+ int rv;
+
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
+ mpte = NULL;
+ vm_page_iter_limit_init(&pages, m_start->object,
+ m_start->pindex + atop(end - start));
+ m = vm_radix_iter_lookup(&pages, m_start->pindex);
+ lock = NULL;
+ rw_rlock(&pvh_global_lock);
+ PMAP_LOCK(pmap);
+ while (m != NULL) {
+ va = start + ptoa(m->pindex - m_start->pindex);
+ if ((va & L2_OFFSET) == 0 && va + L2_SIZE <= end &&
+ m->psind == 1 && pmap_ps_enabled(pmap) &&
+ ((rv = pmap_enter_2mpage(pmap, va, m, prot, &lock)) ==
+ KERN_SUCCESS || rv == KERN_NO_SPACE)) {
+ m = vm_radix_iter_jump(&pages, L2_SIZE / PAGE_SIZE);
+ } else {
+ mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte,
+ &lock);
+ m = vm_radix_iter_step(&pages);
+ }
+ }
+ if (lock != NULL)
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+}
+
+/*
+ * this code makes some *MAJOR* assumptions:
+ * 1. Current pmap & pmap exists.
+ * 2. Not wired.
+ * 3. Read access.
+ * 4. No page table pages.
+ * but is *MUCH* faster than pmap_enter...
+ */
+
+void
+pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
+{
+ struct rwlock *lock;
+
+ lock = NULL;
+ rw_rlock(&pvh_global_lock);
+ PMAP_LOCK(pmap);
+ (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);
+ if (lock != NULL)
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+ if (prot & VM_PROT_EXECUTE)
+ pmap_sync_icache(pmap, va, PAGE_SIZE);
+}
+
+static vm_page_t
+pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp)
+{
+ struct spglist free;
+ pd_entry_t *l2;
+ pt_entry_t *l3, newl3;
+
+ KASSERT(!VA_IS_CLEANMAP(va) ||
+ (m->oflags & VPO_UNMANAGED) != 0,
+ ("pmap_enter_quick_locked: managed mapping within the "
+ "clean submap"));
+ rw_assert(&pvh_global_lock, RA_LOCKED);
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ l2 = NULL;
+
+ CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);
+ /*
+ * In the case that a page table page is not
+ * resident, we are creating it here.
+ */
+ if (va < VM_MAXUSER_ADDRESS) {
+ vm_pindex_t l2pindex;
+
+ /*
+ * Calculate pagetable page index
+ */
+ l2pindex = pmap_l2_pindex(va);
+ if (mpte && (mpte->pindex == l2pindex)) {
+ mpte->ref_count++;
+ } else {
+ /*
+ * Get the l2 entry
+ */
+ l2 = pmap_l2(pmap, va);
+
+ /*
+ * If the page table page is mapped, we just increment
+ * the hold count, and activate it. Otherwise, we
+ * attempt to allocate a page table page. If this
+ * attempt fails, we don't retry. Instead, we give up.
+ */
+ if (l2 != NULL && pmap_pmd_valid(l2)) {
+ if (pmap_pmd_entry(l2) & PTE_HUGE)
+ return (NULL);
+ mpte = PTE_TO_VM_PAGE(pmap_pmd_entry(l2));
+ mpte->ref_count++;
+ } else {
+ /*
+ * Pass NULL instead of the PV list lock
+ * pointer, because we don't intend to sleep.
+ */
+ mpte = _pmap_alloc_l3(pmap, l2pindex, NULL);
+ if (mpte == NULL)
+ return (mpte);
+ }
+ }
+ l3 = VM_PAGE_TO_DMAP(mpte);
+ l3 = &l3[pmap_l3_index(va)];
+ } else {
+ mpte = NULL;
+ l3 = pmap_l3(kernel_pmap, va);
+ }
+ if (l3 == NULL)
+ panic("pmap_enter_quick_locked: No l3");
+ if (pmap_pte_valid(l3)) {
+ if (mpte != NULL)
+ mpte->ref_count--;
+ return (NULL);
+ }
+
+ /*
+ * Enter on the PV list if part of our managed memory.
+ */
+ if ((m->oflags & VPO_UNMANAGED) == 0 &&
+ !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
+ if (mpte != NULL) {
+ SLIST_INIT(&free);
+ if (pmap_unwire_ptp(pmap, va, mpte, &free))
+ vm_page_free_pages_toq(&free, false);
+ }
+ return (NULL);
+ }
+
+ /*
+ * Increment counters
+ */
+ pmap_resident_count_inc(pmap, 1);
+
+ newl3 = ((VM_PAGE_TO_PHYS(m) / PAGE_SIZE) << L3_SHIFT) |
+ PTE_V | PTE_CC | PTE_P | PTE_NX;
+ if ((prot & VM_PROT_EXECUTE) != 0)
+ newl3 &= ~PTE_NX;
+ if ((m->oflags & VPO_UNMANAGED) == 0)
+ newl3 |= PTE_SW_MANAGED;
+ if (va < VM_MAX_USER_ADDRESS)
+ newl3 |= PTE_U;
+ else
+ newl3 |= PTE_G;
+
+ pmap_store(l3, newl3);
+ pmap_invalidate_page(pmap, va);
+ pmap_update_tlb(pmap, va);
+
+#if VM_NRESERVLEVEL > 0
+ /*
+ * If both the PTP and the reservation are fully populated, then attempt
+ * promotion.
+ */
+ if ((prot & VM_PROT_NO_PROMOTE) == 0 &&
+ (mpte == NULL || mpte->ref_count == Ln_ENTRIES) &&
+ (m->flags & PG_FICTITIOUS) == 0 &&
+ vm_reserv_level_iffullpop(m) == 0) {
+ if (l2 == NULL)
+ l2 = pmap_l2(pmap, va);
+
+ /*
+ * If promotion succeeds, then the next call to this function
+ * should not be given the unmapped PTP as a hint.
+ */
+ if (pmap_promote_l2(pmap, l2, va, mpte, lockp))
+ mpte = NULL;
+ }
+#endif
+
+ return (mpte);
+}
+
+/*
+ * This code maps large physical mmap regions into the
+ * processor address space. Note that some shortcuts
+ * are taken, but the code works.
+ */
+void
+pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
+ vm_pindex_t pindex, vm_size_t size)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
+ ("pmap_object_init_pt: non-device object"));
+}
+
+/*
+ * Clear the wired attribute from the mappings for the specified range of
+ * addresses in the given pmap. Every valid mapping within that range
+ * must have the wired attribute set. In contrast, invalid mappings
+ * cannot have the wired attribute set, so they are ignored.
+ *
+ * The wired attribute of the page table entry is not a hardware feature,
+ * so there is no need to invalidate any TLB entries.
+ */
+void
+pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+ vm_offset_t va_next;
+ pd_entry_t *l0, *l1, *l2, l2e;
+ pt_entry_t *l3, l3e;
+ bool pv_lists_locked;
+
+ pv_lists_locked = false;
+retry:
+ PMAP_LOCK(pmap);
+ for (; sva < eva; sva = va_next) {
+
+ l0 = pmap_l0(pmap, sva);
+ if (!pmap_pgd_valid(l0)) {
+ va_next = (sva + L0_SIZE) & ~L0_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+ continue;
+ }
+ l1 = pmap_l0_to_l1(l0, sva);
+
+ if (!pmap_pud_valid(l1)) {
+ va_next = (sva + L1_SIZE) & ~L1_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+ continue;
+ }
+
+ va_next = (sva + L2_SIZE) & ~L2_OFFSET;
+ if (va_next < sva)
+ va_next = eva;
+
+ l2 = pmap_l1_to_l2(l1, sva);
+ if (!pmap_pmd_valid(l2))
+ continue;
+ l2e = pmap_pmd_entry(l2);
+ if (l2e & PTE_HUGE) {
+ if (sva + L2_SIZE == va_next && eva >= va_next) {
+ if ((l2e & PTE_SW_WIRED) == 0)
+ panic("pmap_unwire: l2 %#jx is missing "
+ "PTE_SW_WIRED", (uintmax_t)l2e);
+ pmap_clear_bits(l2, PTE_SW_WIRED);
+ continue;
+ } else {
+ if (!pv_lists_locked) {
+ pv_lists_locked = true;
+ if (!rw_try_rlock(&pvh_global_lock)) {
+ PMAP_UNLOCK(pmap);
+ rw_rlock(&pvh_global_lock);
+ /* Repeat sva. */
+ goto retry;
+ }
+ }
+ if (!pmap_demote_l2(pmap, l2, sva))
+ panic("pmap_unwire: demotion failed");
+ }
+ }
+
+ if (va_next > eva)
+ va_next = eva;
+ for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++,
+ sva += L3_SIZE) {
+ if (!pmap_pte_valid(l3))
+ continue;
+ l3e = pmap_pte_entry(l3);
+ if ((l3e & PTE_SW_WIRED) == 0)
+ panic("pmap_unwire: l3 %#jx is missing "
+ "PTE_SW_WIRED", (uintmax_t)l3e);
+
+ /*
+ * PG_W must be cleared atomically. Although the pmap
+ * lock synchronizes access to PG_W, another processor
+ * could be setting PG_M and/or PG_A concurrently.
+ */
+ pmap_clear_bits(l3, PTE_SW_WIRED);
+ pmap->pm_stats.wired_count--;
+ }
+ }
+ if (pv_lists_locked)
+ rw_runlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+}
+
+/*
+ * Copy the range specified by src_addr/len
+ * from the source map to the range dst_addr/len
+ * in the destination map.
+ *
+ * This routine is only advisory and need not do anything.
+ */
+
+void
+pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
+ vm_offset_t src_addr)
+{
+
+}
+
+/*
+ * pmap_zero_page zeros the specified hardware page by mapping
+ * the page into KVM and using bzero to clear its contents.
+ */
+void
+pmap_zero_page(vm_page_t m)
+{
+ vm_offset_t va = (vm_offset_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
+
+ pagezero((void *)va);
+}
+
+/*
+ * pmap_zero_page_area zeros the specified hardware page by mapping
+ * the page into KVM and using bzero to clear its contents.
+ *
+ * off and size may not cover an area beyond a single hardware page.
+ */
+void
+pmap_zero_page_area(vm_page_t m, int off, int size)
+{
+ vm_offset_t va = (vm_offset_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
+
+ if (off == 0 && size == PAGE_SIZE)
+ pagezero((void *)va);
+ else
+ bzero((char *)va + off, size);
+}
+
+/*
+ * pmap_copy_page copies the specified (machine independent)
+ * page by mapping the page into virtual memory and using
+ * bcopy to copy the page, one machine dependent page at a
+ * time.
+ */
+void
+pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
+{
+ vm_offset_t src = (vm_offset_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
+ vm_offset_t dst = (vm_offset_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
+
+ pagecopy((void *)src, (void *)dst);
+}
+
+int unmapped_buf_allowed = 1;
+
+void
+pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
+ vm_offset_t b_offset, int xfersize)
+{
+ void *a_cp, *b_cp;
+ vm_page_t m_a, m_b;
+ vm_paddr_t p_a, p_b;
+ vm_offset_t a_pg_offset, b_pg_offset;
+ int cnt;
+
+ while (xfersize > 0) {
+ a_pg_offset = a_offset & PAGE_MASK;
+ m_a = ma[a_offset >> PAGE_SHIFT];
+ p_a = m_a->phys_addr;
+ b_pg_offset = b_offset & PAGE_MASK;
+ m_b = mb[b_offset >> PAGE_SHIFT];
+ p_b = m_b->phys_addr;
+ cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
+ cnt = min(cnt, PAGE_SIZE - b_pg_offset);
+ if (__predict_false(!PHYS_IN_DMAP(p_a))) {
+ panic("!DMAP a %lx", p_a);
+ } else {
+ a_cp = (char *)PHYS_TO_DMAP(p_a) + a_pg_offset;
+ }
+ if (__predict_false(!PHYS_IN_DMAP(p_b))) {
+ panic("!DMAP b %lx", p_b);
+ } else {
+ b_cp = (char *)PHYS_TO_DMAP(p_b) + b_pg_offset;
+ }
+ bcopy(a_cp, b_cp, cnt);
+ a_offset += cnt;
+ b_offset += cnt;
+ xfersize -= cnt;
+ }
+}
+
+void *
+pmap_quick_enter_page(vm_page_t m)
+{
+
+ return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
+}
+
+void
+pmap_quick_remove_page(void *addr)
+{
+}
+
+/*
+ * Returns true if the pmap's pv is one of the first
+ * 16 pvs linked to from this page. This count may
+ * be changed upwards or downwards in the future; it
+ * is only necessary that true be returned for a small
+ * subset of pmaps for proper page aging.
+ */
+bool
+pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
+{
+ struct md_page *pvh;
+ struct rwlock *lock;
+ pv_entry_t pv;
+ int loops = 0;
+ bool rv;
+
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = false;
+ rw_rlock(&pvh_global_lock);
+ lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+ rw_rlock(lock);
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
+ if (PV_PMAP(pv) == pmap) {
+ rv = true;
+ break;
+ }
+ loops++;
+ if (loops >= 16)
+ break;
+ }
+ if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
+ if (PV_PMAP(pv) == pmap) {
+ rv = true;
+ break;
+ }
+ loops++;
+ if (loops >= 16)
+ break;
+ }
+ }
+ rw_runlock(lock);
+ rw_runlock(&pvh_global_lock);
+ return (rv);
+}
+
+/*
+ * pmap_page_wired_mappings:
+ *
+ * Return the number of managed mappings to the given physical page
+ * that are wired.
+ */
+int
+pmap_page_wired_mappings(vm_page_t m)
+{
+ struct md_page *pvh;
+ struct rwlock *lock;
+ pmap_t pmap;
+ pd_entry_t *l2;
+ pt_entry_t *l3;
+ pv_entry_t pv;
+ int count, md_gen, pvh_gen;
+
+ if ((m->oflags & VPO_UNMANAGED) != 0)
+ return (0);
+ rw_rlock(&pvh_global_lock);
+ lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+ rw_rlock(lock);
+restart:
+ count = 0;
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ md_gen = m->md.pv_gen;
+ rw_runlock(lock);
+ PMAP_LOCK(pmap);
+ rw_rlock(lock);
+ if (md_gen != m->md.pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto restart;
+ }
+ }
+ l2 = pmap_l2(pmap, pv->pv_va);
+ KASSERT(!(pmap_pmd_entry(l2) & PTE_HUGE),
+ ("%s: found a 2mpage in page %p's pv list", __func__, m));
+
+ l3 = pmap_l2_to_l3(l2, pv->pv_va);
+ if ((pmap_pte_entry(l3) & PTE_SW_WIRED) != 0)
+ count++;
+ PMAP_UNLOCK(pmap);
+ }
+ if ((m->flags & PG_FICTITIOUS) == 0) {
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ md_gen = m->md.pv_gen;
+ pvh_gen = pvh->pv_gen;
+ rw_runlock(lock);
+ PMAP_LOCK(pmap);
+ rw_rlock(lock);
+ if (md_gen != m->md.pv_gen ||
+ pvh_gen != pvh->pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto restart;
+ }
+ }
+ l2 = pmap_l2(pmap, pv->pv_va);
+ if ((pmap_pmd_entry(l2) & PTE_SW_WIRED) != 0)
+ count++;
+ PMAP_UNLOCK(pmap);
+ }
+ }
+ rw_runlock(lock);
+ rw_runlock(&pvh_global_lock);
+ return (count);
+}
+
+/*
+ * Returns true if the given page is mapped individually or as part of
+ * a 2mpage. Otherwise, returns false.
+ */
+bool
+pmap_page_is_mapped(vm_page_t m)
+{
+ struct rwlock *lock;
+ bool rv;
+
+ if ((m->oflags & VPO_UNMANAGED) != 0)
+ return (false);
+ lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+ rw_rlock(lock);
+ rv = !TAILQ_EMPTY(&m->md.pv_list) ||
+ ((m->flags & PG_FICTITIOUS) == 0 &&
+ !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
+ rw_runlock(lock);
+ return (rv);
+}
+
+static void
+pmap_remove_pages_pv(pmap_t pmap, vm_page_t m, pv_entry_t pv,
+ struct spglist *free, bool superpage)
+{
+ struct md_page *pvh;
+ vm_page_t mpte, mt;
+
+ if (superpage) {
+ pmap_resident_count_dec(pmap, Ln_ENTRIES);
+ pvh = pa_to_pvh(m->phys_addr);
+ TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
+ if (TAILQ_EMPTY(&pvh->pv_list)) {
+ for (mt = m; mt < &m[Ln_ENTRIES]; mt++)
+ if (TAILQ_EMPTY(&mt->md.pv_list) &&
+ (mt->a.flags & PGA_WRITEABLE) != 0)
+ vm_page_aflag_clear(mt, PGA_WRITEABLE);
+ }
+ mpte = pmap_remove_pt_page(pmap, pv->pv_va);
+ if (mpte != NULL) {
+ KASSERT(vm_page_any_valid(mpte),
+ ("pmap_remove_pages: pte page not promoted"));
+ pmap_resident_count_dec(pmap, 1);
+ KASSERT(mpte->ref_count == Ln_ENTRIES,
+ ("pmap_remove_pages: pte page ref count error"));
+ /*
+ * Restore the single real wiring (VM_ALLOC_WIRED at
+ * demote time) before dropping it, matching
+ * pmap_remove_l2(): a bare ref_count = 0 would free the
+ * page without vm_wire_sub(1), leaking the global
+ * wired-page counter for every promoted 2MB page-table
+ * page removed via this (process-exit) path.
+ */
+ mpte->ref_count = 1;
+ vm_page_unwire_noq(mpte);
+ pmap_add_delayed_free_list(mpte, free, false);
+ }
+ } else {
+ pmap_resident_count_dec(pmap, 1);
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
+ if (TAILQ_EMPTY(&m->md.pv_list) &&
+ (m->a.flags & PGA_WRITEABLE) != 0) {
+ pvh = pa_to_pvh(m->phys_addr);
+ if (TAILQ_EMPTY(&pvh->pv_list))
+ vm_page_aflag_clear(m, PGA_WRITEABLE);
+ }
+ }
+}
+
+int
+pmap_growkernel(vm_offset_t addr)
+{
+ int rv;
+
+ rv = pmap_growkernel_nopanic(addr);
+ if (rv != KERN_SUCCESS && pmap_growkernel_panic)
+ panic("pmap_growkernel: no memory to grow kernel");
+ return (rv);
+}
+
+/*
+ * Destroy all managed, non-wired mappings in the given user-space
+ * pmap. This pmap cannot be active on any processor besides the
+ * caller.
+ *
+ * This function cannot be applied to the kernel pmap. Moreover, it
+ * is not intended for general use. It is only to be used during
+ * process termination. Consequently, it can be implemented in ways
+ * that make it faster than pmap_remove(). First, it can more quickly
+ * destroy mappings by iterating over the pmap's collection of PV
+ * entries, rather than searching the page table. Second, it doesn't
+ * have to test and clear the page table entries atomically, because
+ * no processor is currently accessing the user address space. In
+ * particular, a page table entry's dirty bit won't change state once
+ * this function starts.
+ */
+void
+pmap_remove_pages(pmap_t pmap)
+{
+ struct spglist free;
+ pd_entry_t ptepde;
+ pt_entry_t *pte, tpte;
+ vm_page_t m, mt;
+ pv_entry_t pv;
+ struct pv_chunk *pc, *npc;
+ struct rwlock *lock;
+ int64_t bit;
+ uint64_t inuse, bitmask;
+ int allfree, field, freed __pv_stat_used, idx;
+ bool superpage;
+
+ lock = NULL;
+
+ SLIST_INIT(&free);
+ rw_rlock(&pvh_global_lock);
+ PMAP_LOCK(pmap);
+ TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
+ allfree = 1;
+ freed = 0;
+ for (field = 0; field < _NPCM; field++) {
+ inuse = ~pc->pc_map[field] & pc_freemask[field];
+ while (inuse != 0) {
+ bit = ffsl(inuse) - 1;
+ bitmask = 1UL << bit;
+ idx = field * 64 + bit;
+ pv = &pc->pc_pventry[idx];
+ inuse &= ~bitmask;
+
+ pte = pmap_l1(pmap, pv->pv_va);
+ KASSERT(pte != NULL,
+ ("%s: l1 address is NULL", __func__));
+ ptepde = pmap_pud_entry(pte);
+ pte = pmap_l1_to_l2(pte, pv->pv_va);
+ tpte = pmap_pmd_entry(pte);
+
+ KASSERT(pmap_pmd_valid(pte),
+ ("L2 PTE is invalid... bogus PV entry? "
+ "va=%#lx, pte=%#lx", pv->pv_va, tpte));
+ if (tpte & PTE_HUGE) {
+ superpage = true;
+ } else {
+ ptepde = tpte;
+ pte = pmap_l2_to_l3(pte, pv->pv_va);
+ tpte = pmap_pte_entry(pte);
+ superpage = false;
+ }
+
+ /*
+ * We cannot remove wired pages from a
+ * process' mapping at this time.
+ */
+ if (tpte & PTE_SW_WIRED) {
+ allfree = 0;
+ continue;
+ }
+
+ m = PTE_TO_VM_PAGE(tpte);
+ KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
+ m < &vm_page_array[vm_page_array_size],
+ ("pmap_remove_pages: bad pte %#jx",
+ (uintmax_t)tpte));
+
+ /*
+ * For a 2MB superpage, pte is the L2
+ * (directory) entry: clear it to the sentinel,
+ * NOT 0. A directory slot of 0 makes the HW
+ * page walker follow phys 0 and read garbage,
+ * which is the user-space corruption we have
+ * been chasing. (pmap_remove_l2() does the same
+ * with pmap_store(l2, invalid_pmd_table).) For
+ * a 4KB page, pte is the L3 (leaf) PTE,
+ * correctly cleared to 0.
+ */
+ if (superpage)
+ pmap_store(pte, invalid_pmd_table);
+ else
+ pmap_clear(pte);
+
+ /*
+ * Update the vm_page_t clean/reference bits.
+ */
+ if ((tpte & (PTE_D | PTE_W)) ==
+ (PTE_D | PTE_W)) {
+ if (superpage)
+ for (mt = m;
+ mt < &m[Ln_ENTRIES]; mt++)
+ vm_page_dirty(mt);
+ else
+ vm_page_dirty(m);
+ }
+
+ CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
+
+ /* Mark free */
+ pc->pc_map[field] |= bitmask;
+
+ pmap_remove_pages_pv(pmap, m, pv, &free,
+ superpage);
+ pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
+ freed++;
+ }
+ }
+ PV_STAT(atomic_add_long(&pv_entry_frees, freed));
+ PV_STAT(atomic_add_int(&pv_entry_spare, freed));
+ PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
+ if (allfree) {
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ free_pv_chunk(pc);
+ }
+ }
+ if (lock != NULL)
+ rw_wunlock(lock);
+ pmap_invalidate_all(pmap);
+ rw_runlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+ vm_page_free_pages_toq(&free, false);
+}
+
+static bool
+pmap_page_test_mappings(vm_page_t m, bool accessed, bool modified)
+{
+ struct md_page *pvh;
+ struct rwlock *lock;
+ pd_entry_t *l2;
+ pt_entry_t *l3, mask;
+ pv_entry_t pv;
+ pmap_t pmap;
+ int md_gen, pvh_gen;
+ bool rv;
+
+ mask = 0;
+ if (modified)
+ mask |= PTE_D;
+
+ if (accessed)
+ mask |= PTE_A;
+
+ rv = false;
+ rw_rlock(&pvh_global_lock);
+ lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+ rw_rlock(lock);
+restart:
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ md_gen = m->md.pv_gen;
+ rw_runlock(lock);
+ PMAP_LOCK(pmap);
+ rw_rlock(lock);
+ if (md_gen != m->md.pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto restart;
+ }
+ }
+ l2 = pmap_l2(pmap, pv->pv_va);
+ KASSERT(!(pmap_pmd_entry(l2) & PTE_HUGE),
+ ("%s: found a 2mpage in page %p's pv list", __func__, m));
+ l3 = pmap_l2_to_l3(l2, pv->pv_va);
+ rv = (pmap_pte_entry(l3) & mask) == mask;
+ PMAP_UNLOCK(pmap);
+ if (rv)
+ goto out;
+ }
+ if ((m->flags & PG_FICTITIOUS) == 0) {
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ md_gen = m->md.pv_gen;
+ pvh_gen = pvh->pv_gen;
+ rw_runlock(lock);
+ PMAP_LOCK(pmap);
+ rw_rlock(lock);
+ if (md_gen != m->md.pv_gen ||
+ pvh_gen != pvh->pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto restart;
+ }
+ }
+ l2 = pmap_l2(pmap, pv->pv_va);
+ rv = (pmap_pmd_entry(l2) & mask) == mask;
+ PMAP_UNLOCK(pmap);
+ if (rv)
+ goto out;
+ }
+ }
+out:
+ rw_runlock(lock);
+ rw_runlock(&pvh_global_lock);
+ return (rv);
+}
+
+/*
+ * pmap_is_modified:
+ *
+ * Return whether or not the specified physical page was modified
+ * in any physical maps.
+ */
+bool
+pmap_is_modified(vm_page_t m)
+{
+
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("pmap_is_modified: page %p is not managed", m));
+
+ /*
+ * If the page is not busied then this check is racy.
+ */
+ if (!pmap_page_is_write_mapped(m))
+ return (false);
+ return (pmap_page_test_mappings(m, false, true));
+}
+
+/*
+ * pmap_is_prefaultable:
+ *
+ * Return whether or not the specified virtual address is eligible
+ * for prefault.
+ */
+bool
+pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
+{
+ pt_entry_t *l3;
+ bool rv;
+
+ /*
+ * Return true if and only if the L3 entry for the specified virtual
+ * address is allocated but invalid.
+ */
+ rv = false;
+ PMAP_LOCK(pmap);
+ l3 = pmap_l3(pmap, addr);
+ if (l3 != NULL && !pmap_pte_valid(l3)) {
+ rv = true;
+ }
+ PMAP_UNLOCK(pmap);
+ return (rv);
+}
+
+/*
+ * pmap_is_referenced:
+ *
+ * Return whether or not the specified physical page was referenced
+ * in any physical maps.
+ */
+bool
+pmap_is_referenced(vm_page_t m)
+{
+
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("pmap_is_referenced: page %p is not managed", m));
+ return (pmap_page_test_mappings(m, true, false));
+}
+
+/*
+ * Clear the write and modified bits in each of the given page's mappings.
+ */
+void
+pmap_remove_write(vm_page_t m)
+{
+ struct md_page *pvh;
+ struct rwlock *lock;
+ pmap_t pmap;
+ pd_entry_t *l2;
+ pt_entry_t *l3, oldl3, newl3;
+ pv_entry_t next_pv, pv;
+ vm_offset_t va;
+ int md_gen, pvh_gen;
+
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("pmap_remove_write: page %p is not managed", m));
+ vm_page_assert_busied(m);
+
+ if (!pmap_page_is_write_mapped(m))
+ return;
+ lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+ pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
+ pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ rw_rlock(&pvh_global_lock);
+retry_pv_loop:
+ rw_wlock(lock);
+ TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ pvh_gen = pvh->pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen) {
+ PMAP_UNLOCK(pmap);
+ rw_wunlock(lock);
+ goto retry_pv_loop;
+ }
+ }
+ va = pv->pv_va;
+ l2 = pmap_l2(pmap, va);
+ if ((pmap_pmd_entry(l2) & PTE_W) != 0)
+ (void)pmap_demote_l2_locked(pmap, l2, va, &lock);
+ KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
+ ("inconsistent pv lock %p %p for page %p",
+ lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
+ PMAP_UNLOCK(pmap);
+ }
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ pvh_gen = pvh->pv_gen;
+ md_gen = m->md.pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
+ PMAP_UNLOCK(pmap);
+ rw_wunlock(lock);
+ goto retry_pv_loop;
+ }
+ }
+ l2 = pmap_l2(pmap, pv->pv_va);
+ KASSERT(!(pmap_pmd_entry(l2) & PTE_HUGE),
+ ("%s: found a 2mpage in page %p's pv list", __func__, m));
+ l3 = pmap_l2_to_l3(l2, pv->pv_va);
+ oldl3 = pmap_pte_entry(l3);
+retry:
+ if ((oldl3 & PTE_W) != 0) {
+ newl3 = oldl3 & ~(PTE_D | PTE_W);
+ if (!pmap_cmpset(l3, &oldl3, newl3))
+ goto retry;
+ if ((oldl3 & PTE_D) != 0)
+ vm_page_dirty(m);
+ pmap_invalidate_page(pmap, pv->pv_va);
+ }
+ PMAP_UNLOCK(pmap);
+ }
+ rw_wunlock(lock);
+ vm_page_aflag_clear(m, PGA_WRITEABLE);
+ rw_runlock(&pvh_global_lock);
+}
+
+/*
+ * pmap_ts_referenced:
+ *
+ * Return a count of reference bits for a page, clearing those bits.
+ * It is not necessary for every reference bit to be cleared, but it
+ * is necessary that 0 only be returned when there are truly no
+ * reference bits set.
+ *
+ * As an optimization, update the page's dirty field if a modified bit is
+ * found while counting reference bits. This opportunistic update can be
+ * performed at low cost and can eliminate the need for some future calls
+ * to pmap_is_modified(). However, since this function stops after
+ * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
+ * dirty pages. Those dirty pages will only be detected by a future call
+ * to pmap_is_modified().
+ */
+int
+pmap_ts_referenced(vm_page_t m)
+{
+ struct spglist free;
+ struct md_page *pvh;
+ struct rwlock *lock;
+ pv_entry_t pv, pvf;
+ pmap_t pmap;
+ pd_entry_t *l2, l2e;
+ pt_entry_t *l3, l3e;
+ vm_paddr_t pa;
+ vm_offset_t va;
+ int cleared, md_gen, not_cleared, pvh_gen;
+
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
+ SLIST_INIT(&free);
+ cleared = 0;
+ pa = VM_PAGE_TO_PHYS(m);
+ pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
+
+ lock = PHYS_TO_PV_LIST_LOCK(pa);
+ rw_rlock(&pvh_global_lock);
+ rw_wlock(lock);
+retry:
+ not_cleared = 0;
+ if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
+ goto small_mappings;
+ pv = pvf;
+ do {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ pvh_gen = pvh->pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto retry;
+ }
+ }
+ va = pv->pv_va;
+ l2 = pmap_l2(pmap, va);
+ l2e = pmap_pmd_entry(l2);
+ if ((l2e & (PTE_W | PTE_D)) == (PTE_W | PTE_D)) {
+ /*
+ * Although l2e is mapping a 2MB page, because
+ * this function is called at a 4KB page granularity,
+ * we only update the 4KB page under test.
+ */
+ vm_page_dirty(m);
+ }
+
+ if ((l2e & PTE_A) != 0) {
+ /*
+ * Since this reference bit is shared by 512 4KB
+ * pages, it should not be cleared every time it is
+ * tested. Apply a simple "hash" function on the
+ * physical page number, the virtual superpage number,
+ * and the pmap address to select one 4KB page out of
+ * the 512 on which testing the reference bit will
+ * result in clearing that reference bit. This
+ * function is designed to avoid the selection of the
+ * same 4KB page for every 2MB page mapping.
+ *
+ * On demotion, a mapping that hasn't been referenced
+ * is simply destroyed. To avoid the possibility of a
+ * subsequent page fault on a demoted wired mapping,
+ * always leave its reference bit set. Moreover,
+ * since the superpage is wired, the current state of
+ * its reference bit won't affect page replacement.
+ */
+ if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L2_SHIFT) ^
+ (uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 &&
+ (l2e & PTE_SW_WIRED) == 0) {
+ pmap_clear_bits(l2, PTE_V);
+ pmap_invalidate_range(pmap, va & ~L2_OFFSET,
+ (va & ~L2_OFFSET) + L2_SIZE);
+ cleared++;
+ } else
+ not_cleared++;
+ }
+ PMAP_UNLOCK(pmap);
+ /* Rotate the PV list if it has more than one entry. */
+ if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) {
+ TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
+ TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
+ }
+ if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX)
+ goto out;
+ } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
+small_mappings:
+ if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
+ goto out;
+ pv = pvf;
+ do {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ pvh_gen = pvh->pv_gen;
+ md_gen = m->md.pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto retry;
+ }
+ }
+ l2 = pmap_l2(pmap, pv->pv_va);
+
+ KASSERT(!(pmap_pmd_entry(l2) & PTE_HUGE),
+ ("pmap_ts_referenced: found an invalid l2 table"));
+
+ l3 = pmap_l2_to_l3(l2, pv->pv_va);
+ l3e = pmap_pte_entry(l3);
+
+ if ((l3e & PTE_D) != 0)
+ vm_page_dirty(m);
+ if ((l3e & PTE_A) != 0) {
+ if ((l3e & PTE_SW_WIRED) == 0) {
+ /*
+ * Wired pages cannot be paged out so
+ * doing accessed bit emulation for
+ * them is wasted effort. We do the
+ * hard work for unwired pages only.
+ */
+ pmap_clear_bits(l3, PTE_V);
+ pmap_invalidate_page(pmap, pv->pv_va);
+ cleared++;
+ } else
+ not_cleared++;
+ }
+ PMAP_UNLOCK(pmap);
+ /* Rotate the PV list if it has more than one entry. */
+ if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) {
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
+ }
+ } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
+ not_cleared < PMAP_TS_REFERENCED_MAX);
+out:
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
+ vm_page_free_pages_toq(&free, false);
+ return (cleared + not_cleared);
+}
+
+/*
+ * Apply the given advice to the specified range of addresses within the
+ * given pmap. Depending on the advice, clear the referenced and/or
+ * modified flags in each mapping and set the mapped page's dirty field.
+ */
+void
+pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
+{
+}
+
+/*
+ * Clear the modify bits on the specified physical page.
+ */
+void
+pmap_clear_modify(vm_page_t m)
+{
+ struct md_page *pvh;
+ struct rwlock *lock;
+ pmap_t pmap;
+ pv_entry_t next_pv, pv;
+ pd_entry_t *l2, oldl2;
+ pt_entry_t *l3;
+ vm_offset_t va;
+ int md_gen, pvh_gen;
+
+ KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+ ("%s: page %p is not managed", __func__, m));
+ vm_page_assert_busied(m);
+
+ if (!pmap_page_is_write_mapped(m))
+ return;
+
+ pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
+ pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+ rw_rlock(&pvh_global_lock);
+ rw_wlock(lock);
+restart:
+ TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ pvh_gen = pvh->pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto restart;
+ }
+ }
+ va = pv->pv_va;
+ l2 = pmap_l2(pmap, va);
+ oldl2 = pmap_pmd_entry(l2);
+ /* If oldl2 has PTE_W set, then it also has PTE_D set. */
+ if ((oldl2 & PTE_W) != 0 &&
+ pmap_demote_l2_locked(pmap, l2, va, &lock) &&
+ (oldl2 & PTE_SW_WIRED) == 0) {
+ /*
+ * Write protect the mapping to a single page so that
+ * a subsequent write access may repromote.
+ */
+ va += VM_PAGE_TO_PHYS(m) - PTE_TO_PHYS(oldl2);
+ l3 = pmap_l2_to_l3(l2, va);
+ pmap_clear_bits(l3, PTE_D | PTE_W);
+ vm_page_dirty(m);
+ pmap_invalidate_page(pmap, va);
+ }
+ PMAP_UNLOCK(pmap);
+ }
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
+ pmap = PV_PMAP(pv);
+ if (!PMAP_TRYLOCK(pmap)) {
+ md_gen = m->md.pv_gen;
+ pvh_gen = pvh->pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
+ PMAP_UNLOCK(pmap);
+ goto restart;
+ }
+ }
+ l2 = pmap_l2(pmap, pv->pv_va);
+ KASSERT(!(pmap_pmd_entry(l2) & PTE_HUGE),
+ ("%s: found a 2mpage in page %p's pv list", __func__, m));
+ l3 = pmap_l2_to_l3(l2, pv->pv_va);
+ if ((pmap_pte_entry(l3) & (PTE_D | PTE_W)) == (PTE_D | PTE_W)) {
+ pmap_clear_bits(l3, PTE_D | PTE_W);
+ pmap_invalidate_page(pmap, pv->pv_va);
+ }
+ PMAP_UNLOCK(pmap);
+ }
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
+}
+
+void *
+pmap_mapbios(vm_paddr_t pa, vm_size_t size)
+{
+ vm_offset_t off, va;
+
+ /* In the direct-map range: just use the DMAP. */
+ if (PHYS_IN_DMAP(pa) && PHYS_IN_DMAP(pa + size - 1))
+ return ((void *)PHYS_TO_DMAP(pa));
+
+ /* Out of DMAP (MMIO / device memory): allocate KVA and map. */
+ off = pa & PAGE_MASK;
+ size = roundup2(off + size, PAGE_SIZE);
+ va = (vm_offset_t)kva_alloc(size);
+ if (va == 0)
+ return (NULL);
+ pmap_kenter_device(va, size, pa & ~PAGE_MASK);
+ return ((void *)(va + off));
+}
+
+void
+pmap_unmapbios(void *p, vm_size_t size)
+{
+ vm_offset_t va, off;
+
+ va = (vm_offset_t)p;
+ off = va & PAGE_MASK;
+ /* DMAP addresses are permanent (no-op). */
+ if (VIRT_IN_DMAP(va))
+ return;
+ /* KVA mappings: remove and free. */
+ size = roundup2(off + size, PAGE_SIZE);
+ pmap_qremove((void *)(va - off), atop(size));
+ kva_free((void *)(va - off), size);
+}
+
+/*
+ * Sets the memory attribute for the specified page.
+ */
+void
+pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
+{
+ if (m->md.pv_memattr == ma)
+ return;
+
+ m->md.pv_memattr = ma;
+
+ /*
+ * If "m" is a normal page, update its direct mapping. This update
+ * can be relied upon to perform any cache operations that are
+ * required for data coherence.
+ */
+ if ((m->flags & PG_FICTITIOUS) == 0 &&
+ pmap_change_attr((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)),
+ PAGE_SIZE, m->md.pv_memattr) != 0)
+ panic("memory attribute change on the direct map failed");
+}
+
+/*
+ * Changes the specified virtual address range's memory type to that given by
+ * the parameter "mode". The specified virtual address range must be
+ * completely contained within either the direct map or the kernel map.
+ *
+ * Returns zero if the change completed successfully, and either EINVAL or
+ * ENOMEM if the change failed. Specifically, EINVAL is returned if some part
+ * of the virtual address range was not mapped, and ENOMEM is returned if
+ * there was insufficient memory available to complete the change. In the
+ * latter case, the memory type may have been changed on some part of the
+ * virtual address range.
+ */
+int
+pmap_change_attr(void *va, vm_size_t size, int mode)
+{
+ int error;
+
+ PMAP_LOCK(kernel_pmap);
+ error = pmap_change_attr_locked(va, size, mode);
+ PMAP_UNLOCK(kernel_pmap);
+ return (error);
+}
+
+/*
+ * Translate a VM_MEMATTR_* value into the corresponding PTE MAT field bits.
+ * The absence of MAT bits selects strongly-ordered uncached (PTE_SUC).
+ */
+static inline pt_entry_t
+pmap_memattr_bits(vm_memattr_t mode)
+{
+
+ switch (mode) {
+ case VM_MEMATTR_UNCACHEABLE:
+ case VM_MEMATTR_DEVICE:
+ return (PTE_SUC);
+ case VM_MEMATTR_WRITE_COMBINING:
+ return (PTE_WUC);
+ default:
+ return (PTE_CC);
+ }
+}
+
+static int
+pmap_change_attr_locked(void *addr, vm_size_t size, int mode)
+{
+ vm_offset_t base, offset, tmpva, va;
+ vm_paddr_t phys;
+ pd_entry_t *l1, l1e;
+ pd_entry_t *l2, l2e;
+ pt_entry_t *l3, l3e;
+ pt_entry_t bits, mask;
+ bool anychanged = false;
+ int error = 0;
+
+ PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
+ va = (vm_offset_t)addr;
+ base = trunc_page(va);
+ offset = va & PAGE_MASK;
+ size = round_page(offset + size);
+
+ if (!VIRT_IN_DMAP(base) &&
+ !(base >= VM_MIN_KERNEL_ADDRESS && base < VM_MAX_KERNEL_ADDRESS))
+ return (EINVAL);
+
+ bits = pmap_memattr_bits(mode);
+ mask = PTE_MAT_MASK;
+
+ /* First loop: perform PTE validation and demotions as necessary. */
+ for (tmpva = base; tmpva < base + size; ) {
+ l1 = pmap_l1(kernel_pmap, tmpva);
+ if (l1 == NULL || !pmap_pud_valid(l1))
+ return (EINVAL);
+ l1e = pmap_pud_entry(l1);
+ if ((l1e & PTE_HUGE) != 0) {
+ /*
+ * If the existing entry has the correct attributes,
+ * then no need to demote.
+ */
+ if ((l1e & mask) == bits) {
+ tmpva = (tmpva & ~L1_OFFSET) + L1_SIZE;
+ continue;
+ }
+
+ /*
+ * If the 1GB page fits in the remaining range, we
+ * don't need to demote.
+ */
+ if ((tmpva & L1_OFFSET) == 0 &&
+ tmpva + L1_SIZE <= base + size) {
+ tmpva += L1_SIZE;
+ continue;
+ }
+
+ if (!pmap_demote_l1(kernel_pmap, l1, tmpva))
+ return (ENOMEM);
+ continue;
+ }
+ l2 = pmap_l1_to_l2(l1, tmpva);
+ if (!pmap_pmd_valid(l2))
+ return (EINVAL);
+ l2e = pmap_pmd_entry(l2);
+ if ((l2e & PTE_HUGE) != 0) {
+ /*
+ * If the existing entry has the correct attributes,
+ * then no need to demote.
+ */
+ if ((l2e & mask) == bits) {
+ tmpva = (tmpva & ~L2_OFFSET) + L2_SIZE;
+ continue;
+ }
+
+ /*
+ * If the 2MB page fits in the remaining range, we
+ * don't need to demote.
+ */
+ if ((tmpva & L2_OFFSET) == 0 &&
+ tmpva + L2_SIZE <= base + size) {
+ tmpva += L2_SIZE;
+ continue;
+ }
+
+ if (!pmap_demote_l2(kernel_pmap, l2, tmpva))
+ return (ENOMEM);
+ continue;
+ }
+ l3 = pmap_l2_to_l3(l2, tmpva);
+ if (!pmap_pte_valid(l3))
+ return (EINVAL);
+
+ tmpva += PAGE_SIZE;
+ }
+
+ /* Second loop: perform PTE updates. */
+ for (tmpva = base; tmpva < base + size; ) {
+ l1 = pmap_l1(kernel_pmap, tmpva);
+ l1e = pmap_pud_entry(l1);
+ if ((l1e & PTE_HUGE) != 0) {
+ /* Unchanged. */
+ if ((l1e & mask) == bits) {
+ tmpva += L1_SIZE;
+ continue;
+ }
+
+ pmap_store(l1, (l1e & ~mask) | bits);
+ anychanged = true;
+ tmpva += L1_SIZE;
+
+ /* Update the corresponding DMAP entry. */
+ phys = PTE_TO_PHYS(l1e);
+ if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(phys)) {
+ error = pmap_change_attr_locked(
+ (void *)PHYS_TO_DMAP(phys), L1_SIZE, mode);
+ if (error != 0)
+ break;
+ }
+ continue;
+ }
+
+ l2 = pmap_l1_to_l2(l1, tmpva);
+ l2e = pmap_pmd_entry(l2);
+ if ((l2e & PTE_HUGE) != 0) {
+ /* Unchanged. */
+ if ((l2e & mask) == bits) {
+ tmpva += L2_SIZE;
+ continue;
+ }
+
+ pmap_store(l2, (l2e & ~mask) | bits);
+ anychanged = true;
+ tmpva += L2_SIZE;
+
+ /* Update the corresponding DMAP entry. */
+ phys = PTE_TO_PHYS(l2e);
+ if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(phys)) {
+ error = pmap_change_attr_locked(
+ (void *)PHYS_TO_DMAP(phys), L2_SIZE, mode);
+ if (error != 0)
+ break;
+ }
+ continue;
+ }
+
+ l3 = pmap_l2_to_l3(l2, tmpva);
+ l3e = pmap_load(l3);
+
+ /* Unchanged. */
+ if ((l3e & mask) == bits) {
+ tmpva += PAGE_SIZE;
+ continue;
+ }
+
+ pmap_store(l3, (l3e & ~mask) | bits);
+ anychanged = true;
+ tmpva += PAGE_SIZE;
+
+ /* Update the corresponding DMAP entry. */
+ phys = PTE_TO_PHYS(l3e);
+ if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(phys)) {
+ error = pmap_change_attr_locked(
+ (void *)PHYS_TO_DMAP(phys), L3_SIZE, mode);
+ if (error != 0)
+ break;
+ }
+ }
+
+ if (anychanged) {
+ /*
+ * The second loop may advance "tmpva" past the end of the
+ * range by up to a superpage size when it skips an
+ * already-correct superpage; only invalidate what we changed.
+ */
+ pmap_invalidate_range(kernel_pmap, base,
+ MIN(tmpva, base + size));
+ /*
+ * No cache maintenance is required for a cached to uncached
+ * transition: LoongArch maintains cache coherency in
+ * hardware.
+ */
+ }
+
+ return (error);
+}
+
+/*
+ * Perform the pmap work for mincore(2). If the page is not both referenced and
+ * modified by this pmap, returns its physical address so that the caller can
+ * find other mappings.
+ */
+int
+pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)
+{
+ pt_entry_t *l2, *l3, tpte;
+ vm_paddr_t pa;
+ int val;
+ bool managed;
+
+ PMAP_LOCK(pmap);
+ l2 = pmap_l2(pmap, addr);
+ if (l2 != NULL && pmap_pmd_valid(l2)) {
+ tpte = pmap_pmd_entry(l2);
+ if (tpte & PTE_HUGE) {
+ pa = PTE_TO_PHYS(tpte) | (addr & L2_OFFSET);
+ val = MINCORE_INCORE | MINCORE_PSIND(1);
+ } else {
+ l3 = pmap_l2_to_l3(l2, addr);
+ if (!pmap_pte_valid(l3)) {
+ PMAP_UNLOCK(pmap);
+ return (0);
+ }
+ tpte = pmap_pte_entry(l3);
+ pa = PTE_TO_PHYS(tpte) | (addr & L3_OFFSET);
+ val = MINCORE_INCORE;
+ }
+
+ if ((tpte & PTE_D) != 0)
+ val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
+ if ((tpte & PTE_A) != 0)
+ val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
+ managed = (tpte & PTE_SW_MANAGED) == PTE_SW_MANAGED;
+ } else {
+ managed = false;
+ val = 0;
+ }
+ if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
+ (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) {
+ *pap = pa;
+ }
+ PMAP_UNLOCK(pmap);
+ return (val);
+}
+
+void
+pmap_activate_sw(struct thread *td)
+{
+ pmap_t oldpmap, pmap;
+ u_int cpuid;
+ bool need_tlb_flush = false;
+
+ oldpmap = PCPU_GET(curpmap);
+ pmap = vmspace_pmap(td->td_proc->p_vmspace);
+ if (pmap == oldpmap) {
+ la_watch_load(td);
+ return;
+ }
+
+ cpuid = PCPU_GET(cpuid);
+#ifdef SMP
+ CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
+#else
+ CPU_SET(cpuid, &pmap->pm_active);
+#endif
+ /*
+ * Invalidate oldpmap's ASID on this CPU BEFORE removing it from
+ * pm_active. This preserves the invariant that pm_active is exactly
+ * the set of CPUs holding a live ASID for a pmap. If we cleared
+ * pm_active first, a concurrent remote shooter checking pm_active
+ * would skip this CPU, but stale TLB entries with the old ASID could
+ * still match if the ASID is later recycled (ASID wraparound).
+ *
+ * Zeroing asid[cpuid] also forces a fresh ASID next time oldpmap is
+ * activated here, so the vfork shared-pmap case is still covered.
+ */
+ if (oldpmap != kernel_pmap) {
+ la_stale_add(pmap_get_asid(oldpmap, cpuid));
+ oldpmap->asid[cpuid] = 0;
+ }
+#ifdef SMP
+ CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
+#else
+ CPU_CLR(cpuid, &oldpmap->pm_active);
+#endif
+
+ if (!pmap_asid_valid(pmap, cpuid))
+ pmap_new_asid(pmap, cpuid, &need_tlb_flush);
+
+ /*
+ * On an ASID generation wrap, pmap_new_asid() recycled the low ASID
+ * bits, which may match stale TLB entries from the previous generation.
+ * Flush all G=0 entries BEFORE loading the new ASID: flush_tlb_user()
+ * is INVTLB_CURRENT_GFALSE (ASID-agnostic), so it evicts those stale
+ * entries regardless of which ASID is currently loaded, ensuring none
+ * can survive to collide with the freshly-installed ASID.
+ */
+ if (need_tlb_flush) {
+ flush_tlb_user();
+ /*
+ * The wrap evicted every G=0 entry on this CPU, so the
+ * stale-ASID list (old-generation numbers) is obsolete. Drop
+ * it now, before those numbers are recycled onto new pmaps and
+ * a later drain spuriously flushes a live address space.
+ */
+ la_stale_reset();
+ }
+
+ /*
+ * Switch the user-space page table base. PGDL is used for
+ * addresses with VA[63]=0 (user space). PGDH was set at boot
+ * to the kernel page table and never changes.
+ */
+ set_pgd_asid(pmap_get_asid(pmap, cpuid), pmap->pm_pgdl);
+
+ PCPU_SET(curpmap, pmap);
+
+ /*
+ * Program the incoming thread's watchpoints now that the new pmap's
+ * ASID is installed on this CPU. Runs on every switch (including the
+ * same-pmap thread switch above) so a previous thread's watchpoints
+ * cannot linger.
+ */
+ la_watch_load(td);
+}
+
+void
+pmap_activate(struct thread *td)
+{
+
+ critical_enter();
+ pmap_activate_sw(td);
+ critical_exit();
+}
+
+void
+pmap_activate_boot(pmap_t pmap)
+{
+ u_int cpuid;
+
+ cpuid = PCPU_GET(cpuid);
+#ifdef SMP
+ CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
+#else
+ CPU_SET(cpuid, &pmap->pm_active);
+#endif
+ PCPU_SET(curpmap, pmap);
+ csr_write64(pmap->pm_pgdl, LOONGARCH_CSR_PGDL);
+ csr_write64(pmap->pm_pgdh, LOONGARCH_CSR_PGDH);
+
+ /*
+ * Discard all TLB entries on this CPU. APs join pm_active after the
+ * BSP has already mapped kernel pages (dpcpu zones, boot stacks); any
+ * TLB entries filled during the AP's early boot (before it could
+ * receive shootdown IPIs) may be stale — even/odd pair entries where
+ * the partner half was valid when the pair was filled but later
+ * invalidated, or entries mapping the wrong PFN. A full flush forces
+ * the HW walker to re-walk the (now current) page tables on first use.
+ */
+ invtlb_all(INVTLB_CURRENT_ALL);
+}
+
+void
+pmap_active_cpus(pmap_t pmap, cpuset_t *res)
+{
+ *res = pmap->pm_active;
+}
+
+void
+pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)
+{
+
+ /*
+ * Flush the local icache to make any recent data stores visible
+ * as instructions. Remote cpus will flush their own icache on
+ * the next context switch or exception return. We do not issue
+ * remote IPIs here because tlb_invalidate_all_remote spin-waits
+ * with a global counter that cannot handle concurrent callers,
+ * and the icache flush is not urgent enough to warrant a complex
+ * synchronization protocol.
+ */
+ flush_icache();
+}
+
+/*
+ * Increase the starting virtual address of the given mapping if a
+ * different alignment might result in more superpage mappings.
+ */
+void
+pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
+ vm_offset_t *addr, vm_size_t size)
+{
+ vm_offset_t superpage_offset;
+
+ if (size < L2_SIZE)
+ return;
+ if (object != NULL && (object->flags & OBJ_COLORED) != 0)
+ offset += ptoa(object->pg_color);
+ superpage_offset = offset & L2_OFFSET;
+ if (size - ((L2_SIZE - superpage_offset) & L2_OFFSET) < L2_SIZE ||
+ (*addr & L2_OFFSET) == superpage_offset)
+ return;
+ if ((*addr & L2_OFFSET) < superpage_offset)
+ *addr = (*addr & ~L2_OFFSET) + superpage_offset;
+ else
+ *addr = ((*addr + L2_OFFSET) & ~L2_OFFSET) + superpage_offset;
+}
+
+/**
+ * Get the kernel virtual address of a set of physical pages. If there are
+ * physical addresses not covered by the DMAP perform a transient mapping
+ * that will be removed when calling pmap_unmap_io_transient.
+ *
+ * \param page The pages the caller wishes to obtain the virtual
+ * address on the kernel memory map.
+ * \param vaddr On return contains the kernel virtual memory address
+ * of the pages passed in the page parameter.
+ * \param count Number of pages passed in.
+ * \param can_fault true if the thread using the mapped pages can take
+ * page faults, false otherwise.
+ *
+ * \returns true if the caller must call pmap_unmap_io_transient when
+ * finished or false otherwise.
+ *
+ */
+bool
+pmap_map_io_transient(vm_page_t page[], void *vaddr[], int count,
+ bool can_fault)
+{
+ vm_paddr_t paddr;
+ vmem_addr_t addr;
+ bool needs_mapping;
+ int error __diagused, i;
+
+ /*
+ * Allocate any KVA space that we need, this is done in a separate
+ * loop to prevent calling vmem_alloc while pinned.
+ */
+ needs_mapping = false;
+ for (i = 0; i < count; i++) {
+ paddr = VM_PAGE_TO_PHYS(page[i]);
+ if (__predict_false(paddr >= DMAP_MAX_PHYSADDR)) {
+ error = vmem_alloc(kernel_arena, PAGE_SIZE,
+ M_BESTFIT | M_WAITOK, &addr);
+ KASSERT(error == 0, ("vmem_alloc failed: %d", error));
+ vaddr[i] = (void*)addr;
+ needs_mapping = true;
+ } else {
+ vaddr[i] = (void*)PHYS_TO_DMAP(paddr);
+ }
+ }
+
+ /* Exit early if everything is covered by the DMAP */
+ if (!needs_mapping)
+ return (false);
+
+ if (!can_fault)
+ sched_pin();
+ for (i = 0; i < count; i++) {
+ paddr = VM_PAGE_TO_PHYS(page[i]);
+ if (paddr >= DMAP_MAX_PHYSADDR) {
+ panic(
+ "pmap_map_io_transient: TODO: Map out of DMAP data");
+ }
+ }
+
+ return (needs_mapping);
+}
+
+void
+pmap_unmap_io_transient(vm_page_t page[], void *vaddr[], int count,
+ bool can_fault)
+{
+ vm_paddr_t paddr;
+ int i;
+
+ if (!can_fault)
+ sched_unpin();
+ for (i = 0; i < count; i++) {
+ paddr = VM_PAGE_TO_PHYS(page[i]);
+ if (paddr >= DMAP_MAX_PHYSADDR) {
+ panic("TODO: pmap_unmap_io_transient: Unmap data");
+ }
+ }
+}
+
+bool
+pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)
+{
+
+ return (mode >= VM_MEMATTR_DEVICE && mode <= VM_MEMATTR_WRITE_BACK);
+}
+
+bool
+pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l1, pd_entry_t **l2,
+ pt_entry_t **l3)
+{
+ pd_entry_t *l1p, *l2p;
+
+ /* Get l1 directory entry. */
+ l1p = pmap_l1(pmap, va);
+ *l1 = l1p;
+
+ if (l1p == NULL || !pmap_pud_valid(l1p))
+ return (false);
+
+ if (pmap_pud_entry(l1p) & PTE_HUGE) {
+ *l2 = NULL;
+ *l3 = NULL;
+ return (true);
+ }
+
+ /* Get l2 directory entry. */
+ l2p = pmap_l1_to_l2(l1p, va);
+ *l2 = l2p;
+
+ if (l2p == NULL || !pmap_pmd_valid(l2p))
+ return (false);
+
+ if (pmap_pmd_entry(l2p) & PTE_HUGE) {
+ *l3 = NULL;
+ return (true);
+ }
+
+ /* Get l3 page table entry. */
+ *l3 = pmap_l2_to_l3(l2p, va);
+
+ return (true);
+}
+
+/*
+ * Track a range of the kernel's virtual address space that is contiguous
+ * in various mapping attributes.
+ */
+struct pmap_kernel_map_range {
+ vm_offset_t sva;
+ pt_entry_t attrs;
+ int l3pages;
+ int l2pages;
+ int l1pages;
+};
+
+static void
+sysctl_kmaps_dump(struct sbuf *sb, struct pmap_kernel_map_range *range,
+ vm_offset_t eva)
+{
+
+ if (eva <= range->sva)
+ return;
+
+ sbuf_printf(sb, "0x%016lx-0x%016lx r%c%c%c%c %d %d %d\n",
+ range->sva, eva,
+ (range->attrs & PTE_W) == PTE_W ? 'w' : '-',
+ (range->attrs & PTE_NX) == 0 ? 'x' : '-',
+ (range->attrs & PTE_U) == PTE_U ? 'u' : 's',
+ (range->attrs & PTE_G) == PTE_G ? 'g' : '-',
+ range->l1pages, range->l2pages, range->l3pages);
+
+ /* Reset to sentinel value. */
+ range->sva = 0xfffffffffffffffful;
+}
+
+/*
+ * Determine whether the attributes specified by a page table entry match those
+ * being tracked by the current range.
+ */
+static bool
+sysctl_kmaps_match(struct pmap_kernel_map_range *range, pt_entry_t attrs)
+{
+
+ return (range->attrs == attrs);
+}
+
+static void
+sysctl_kmaps_reinit(struct pmap_kernel_map_range *range, vm_offset_t va,
+ pt_entry_t attrs)
+{
+
+ memset(range, 0, sizeof(*range));
+ range->sva = va;
+ range->attrs = attrs;
+}
+
+/*
+ * Given a leaf PTE, derive the mapping's attributes. If they do not match
+ * those of the current run, dump the address range and its attributes, and
+ * begin a new run.
+ */
+static void
+sysctl_kmaps_check(struct sbuf *sb, struct pmap_kernel_map_range *range,
+ vm_offset_t va, pd_entry_t l1e, pd_entry_t l2e, pt_entry_t l3e)
+{
+ pt_entry_t attrs;
+
+ /* The PTE global bit is inherited by lower levels. */
+ attrs = l1e & PTE_G;
+ if (l1e & PTE_HUGE)
+ attrs |= l1e & (PTE_W | PTE_U);
+ else if (l2e != 0)
+ attrs |= l2e & PTE_G;
+ if (l2e & PTE_HUGE)
+ attrs |= l2e & (PTE_W | PTE_U);
+ else if (l3e != 0)
+ attrs |= l3e & (PTE_W | PTE_U | PTE_G);
+
+ if (range->sva > va || !sysctl_kmaps_match(range, attrs)) {
+ sysctl_kmaps_dump(sb, range, va);
+ sysctl_kmaps_reinit(range, va, attrs);
+ }
+}
+
+static int
+sysctl_kmaps(SYSCTL_HANDLER_ARGS)
+{
+ struct pmap_kernel_map_range range;
+ struct sbuf sbuf, *sb;
+ pd_entry_t *l1, l1e, *l2, l2e;
+ pt_entry_t *l3, l3e;
+ vm_offset_t sva;
+ vm_paddr_t pa;
+ int error, i, j, k;
+
+ error = sysctl_wire_old_buffer(req, 0);
+ if (error != 0)
+ return (error);
+ sb = &sbuf;
+ sbuf_new_for_sysctl(sb, NULL, PAGE_SIZE, req);
+
+ /* Sentinel value. */
+ range.sva = 0xfffffffffffffffful;
+
+ /*
+ * Iterate over the kernel page tables without holding the kernel pmap
+ * lock. Kernel page table pages are never freed, so at worst we will
+ * observe inconsistencies in the output.
+ */
+ sva = VM_MIN_KERNEL_ADDRESS;
+ for (i = pmap_l1_index(sva); i < Ln_ENTRIES; i++) {
+ if (i == pmap_l1_index(DMAP_MIN_ADDRESS))
+ sbuf_printf(sb, "\nDirect map:\n");
+ else if (i == pmap_l1_index(VM_MIN_KERNEL_ADDRESS))
+ sbuf_printf(sb, "\nKernel map:\n");
+
+ l1 = pmap_l1(kernel_pmap, sva);
+ if (!pmap_pud_valid(l1)) {
+ sysctl_kmaps_dump(sb, &range, sva);
+ sva += L1_SIZE;
+ continue;
+ }
+ l1e = pmap_pud_entry(l1);
+ if (l1e & PTE_HUGE) {
+ sysctl_kmaps_check(sb, &range, sva, l1e, 0, 0);
+ range.l1pages++;
+ sva += L1_SIZE;
+ continue;
+ }
+ pa = PTE_TO_PHYS(l1e);
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(pa);
+
+ for (j = pmap_l2_index(sva); j < Ln_ENTRIES; j++) {
+ if (!pmap_pmd_valid(&l2[j])) {
+ sysctl_kmaps_dump(sb, &range, sva);
+ sva += L2_SIZE;
+ continue;
+ }
+ l2e = pmap_pmd_entry(&l2[j]);
+ if (l2e & PTE_HUGE) {
+ sysctl_kmaps_check(sb, &range, sva,
+ l1e, l2e, 0);
+ range.l2pages++;
+ sva += L2_SIZE;
+ continue;
+ }
+ pa = PTE_TO_PHYS(l2e);
+ l3 = (pd_entry_t *)PHYS_TO_DMAP(pa);
+
+ for (k = pmap_l3_index(sva); k < Ln_ENTRIES; k++,
+ sva += L3_SIZE) {
+ if (!pmap_pte_valid(&l3[k])) {
+ sysctl_kmaps_dump(sb, &range, sva);
+ continue;
+ }
+ l3e = pmap_pte_entry(&l3[k]);
+ sysctl_kmaps_check(sb, &range, sva,
+ l1e, l2e, l3e);
+ range.l3pages++;
+ }
+ }
+ }
+
+ error = sbuf_finish(sb);
+ sbuf_delete(sb);
+ return (error);
+}
+SYSCTL_OID(_vm_pmap, OID_AUTO, kernel_maps,
+ CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_SKIP,
+ NULL, 0, sysctl_kmaps, "A",
+ "Dump kernel address layout");
diff --git a/sys/loongarch/loongarch/ptrace_machdep.c b/sys/loongarch/loongarch/ptrace_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/ptrace_machdep.c
@@ -0,0 +1,85 @@
+/*-
+ * Copyright (c) 2014 Andrew Turner
+ * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/exec.h>
+#include <sys/imgact.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/ptrace.h>
+#include <sys/reg.h>
+#include <sys/rwlock.h>
+#include <sys/sched.h>
+#include <sys/signalvar.h>
+#include <sys/syscallsubr.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
+#include <sys/ucontext.h>
+
+#include <machine/cpu.h>
+#include <machine/pcb.h>
+#include <machine/pte.h>
+#include <machine/loongarchreg.h>
+#include <machine/trap.h>
+
+int
+ptrace_set_pc(struct thread *td, u_long addr)
+{
+
+ td->td_frame->tf_era = addr;
+ return (0);
+}
+
+int
+ptrace_single_step(struct thread *td)
+{
+
+ /*
+ * Arm a single step: la_watch_load() will program IB0 at the current
+ * PC and set FWPS.SKIP on the next switch-in, so exactly one user
+ * instruction executes before EXCCODE_WATCH fires.
+ */
+ td->td_md.md_ss_addr = td->td_frame->tf_era;
+ td->td_md.md_ss_active = 1;
+ return (0);
+}
+
+int
+ptrace_clear_single_step(struct thread *td)
+{
+
+ td->td_md.md_ss_active = 0;
+ return (0);
+}
+
diff --git a/sys/loongarch/loongarch/stack_machdep.c b/sys/loongarch/loongarch/stack_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/stack_machdep.c
@@ -0,0 +1,86 @@
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/stack.h>
+
+#include <machine/vmparam.h>
+#include <machine/pcb.h>
+#include <machine/stack.h>
+
+static void
+stack_capture(struct thread *td, struct stack *st, struct unwind_state *frame)
+{
+
+ stack_zero(st);
+
+ while (1) {
+ if (!unwind_frame(td, frame))
+ break;
+ if (!INKERNEL((vm_offset_t)frame->pc))
+ break;
+ if (stack_put(st, frame->pc) == -1)
+ break;
+ }
+}
+
+int
+stack_save_td(struct stack *st, struct thread *td)
+{
+ struct unwind_state frame;
+
+ THREAD_LOCK_ASSERT(td, MA_OWNED);
+
+ if (TD_IS_RUNNING(td))
+ return (EOPNOTSUPP);
+
+ frame.sp = td->td_pcb->pcb_sp;
+ frame.fp = td->td_pcb->fp;
+ frame.pc = td->td_pcb->pcb_ra;
+
+ stack_capture(td, st, &frame);
+ return (0);
+}
+
+void
+stack_save(struct stack *st)
+{
+ struct unwind_state frame;
+ uintptr_t sp;
+
+ __asm __volatile("move %0, $sp" : "=&r" (sp));
+
+ frame.sp = sp;
+ frame.fp = (uintptr_t)__builtin_frame_address(0);
+ frame.pc = (uintptr_t)stack_save;
+
+ stack_capture(curthread, st, &frame);
+}
diff --git a/sys/loongarch/loongarch/support.S b/sys/loongarch/loongarch/support.S
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/support.S
@@ -0,0 +1,269 @@
+/*-
+ * Copyright (c) 2015-2020 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+#include <machine/setjmp.h>
+#include <machine/loongarchreg.h>
+
+#include "assym.inc"
+
+/*
+ * One of the fu* or su* functions failed, return -1.
+ */
+ENTRY(fsu_fault)
+ SET_FAULT_HANDLER($zero, $a1) /* Reset the handler function */
+ EXIT_USER_ACCESS($a1)
+fsu_fault_nopcb:
+ li.d $a0, -1
+ jr $ra
+END(fsu_fault)
+
+/*
+ * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
+ */
+ENTRY(casueword32)
+ li.d $a4, (VM_MAXUSER_ADDRESS-3)
+ bgeu $a0, $a4, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a4) /* And set it */
+ ENTER_USER_ACCESS($a4)
+ move $t0, $a3 /* save newval: sc's rd is both value and result */
+ li.d $a5, 1 /* rv = 1 (CAS did not apply) by default */
+1: ll.w $a4, $a0, 0 /* a4 = observed old value */
+ bne $a4, $a1, 2f /* mismatch: leave rv = 1 */
+ move $a3, $t0 /* reload newval before sc (sc clobbers rd) */
+ sc.w $a3, $a0, 0 /* store newval; a3 = 1 (ok) / 0 (lost race) */
+ beqz $a3, 1b /* lost the ll/sc race -> retry */
+ li.d $a5, 0 /* matched and stored: rv = 0 (success) */
+2: EXIT_USER_ACCESS($a6)
+ SET_FAULT_HANDLER($zero, $a6) /* Reset the fault handler */
+ st.w $a4, $a2, 0 /* Store the read data */
+ move $a0, $a5 /* 0 = success, 1 = CAS did not apply */
+ jr $ra /* Return */
+END(casueword32)
+
+/*
+ * int casueword(volatile u_long *, u_long, u_long *, u_long)
+ */
+ENTRY(casueword)
+ li.d $a4, (VM_MAXUSER_ADDRESS-7)
+ bgeu $a0, $a4, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a4) /* And set it */
+ ENTER_USER_ACCESS($a4)
+ move $t0, $a3 /* save newval: sc's rd is both value and result */
+ li.d $a5, 1 /* rv = 1 (CAS did not apply) by default */
+1: ll.d $a4, $a0, 0 /* a4 = observed old value */
+ bne $a4, $a1, 2f /* mismatch: leave rv = 1 */
+ move $a3, $t0 /* reload newval before sc (sc clobbers rd) */
+ sc.d $a3, $a0, 0 /* store newval; a3 = 1 (ok) / 0 (lost race) */
+ beqz $a3, 1b /* lost the ll/sc race -> retry */
+ li.d $a5, 0 /* matched and stored: rv = 0 (success) */
+2: EXIT_USER_ACCESS($a6)
+ SET_FAULT_HANDLER($zero, $a6) /* Reset the fault handler */
+ st.d $a4, $a2, 0 /* Store the read data */
+ move $a0, $a5 /* 0 = success, 1 = CAS did not apply */
+ jr $ra /* Return */
+END(casueword)
+
+/*
+ * int fubyte(volatile const void *)
+ */
+ENTRY(fubyte)
+ li.d $a1, VM_MAXUSER_ADDRESS
+ bgeu $a0, $a1, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a1) /* And set it */
+ ENTER_USER_ACCESS($a1)
+ ld.bu $a0, $a0, 0 /* Try loading the data */
+ EXIT_USER_ACCESS($a1)
+ SET_FAULT_HANDLER($zero, $a1) /* Reset the fault handler */
+ jr $ra /* Return */
+END(fubyte)
+
+/*
+ * int fuword(volatile const void *)
+ */
+ENTRY(fuword16)
+ li.d $a1, (VM_MAXUSER_ADDRESS-1)
+ bgeu $a0, $a1, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a1) /* And set it */
+ ENTER_USER_ACCESS($a1)
+ ld.hu $a0, $a0, 0 /* Try loading the data */
+ EXIT_USER_ACCESS($a1)
+ SET_FAULT_HANDLER($zero, $a1) /* Reset the fault handler */
+ jr $ra /* Return */
+END(fuword16)
+
+/*
+ * int32_t fueword32(volatile const void *, int32_t *)
+ */
+ENTRY(fueword32)
+ li.d $a2, (VM_MAXUSER_ADDRESS-3)
+ bgeu $a0, $a2, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a2) /* And set it */
+ ENTER_USER_ACCESS($a2)
+ ld.w $a0, $a0, 0 /* Try loading the data */
+ EXIT_USER_ACCESS($a2)
+ SET_FAULT_HANDLER($zero, $a2) /* Reset the fault handler */
+ st.w $a0, $a1, 0 /* Save the data in kernel space */
+ li.d $a0, 0 /* Success */
+ jr $ra /* Return */
+END(fueword32)
+
+/*
+ * long fueword(volatile const void *, int64_t *)
+ * int64_t fueword64(volatile const void *, int64_t *)
+ */
+ENTRY(fueword)
+EENTRY(fueword64)
+ li.d $a2, (VM_MAXUSER_ADDRESS-7)
+ bgeu $a0, $a2, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a2) /* And set it */
+ ENTER_USER_ACCESS($a2)
+ ld.d $a0, $a0, 0 /* Try loading the data */
+ EXIT_USER_ACCESS($a2)
+ SET_FAULT_HANDLER($zero, $a2) /* Reset the fault handler */
+ st.d $a0, $a1, 0 /* Save the data in kernel space */
+ li.d $a0, 0 /* Success */
+ jr $ra /* Return */
+EEND(fueword64)
+END(fueword)
+
+/*
+ * int subyte(volatile void *, int)
+ */
+ENTRY(subyte)
+ li.d $a2, VM_MAXUSER_ADDRESS
+ bgeu $a0, $a2, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a2) /* And set it */
+ ENTER_USER_ACCESS($a2)
+ st.b $a1, $a0, 0 /* Try storing the data */
+ EXIT_USER_ACCESS($a2)
+ SET_FAULT_HANDLER($zero, $a2) /* Reset the fault handler */
+ li.d $a0, 0 /* Success */
+ jr $ra /* Return */
+END(subyte)
+
+/*
+ * int suword16(volatile void *, int)
+ */
+ENTRY(suword16)
+ li.d $a2, (VM_MAXUSER_ADDRESS-1)
+ bgeu $a0, $a2, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a2) /* And set it */
+ ENTER_USER_ACCESS($a2)
+ st.h $a1, $a0, 0 /* Try storing the data */
+ EXIT_USER_ACCESS($a2)
+ SET_FAULT_HANDLER($zero, $a2) /* Reset the fault handler */
+ li.d $a0, 0 /* Success */
+ jr $ra /* Return */
+END(suword16)
+
+/*
+ * int suword32(volatile void *, int)
+ */
+ENTRY(suword32)
+ li.d $a2, (VM_MAXUSER_ADDRESS-3)
+ bgeu $a0, $a2, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a2) /* And set it */
+ ENTER_USER_ACCESS($a2)
+ st.w $a1, $a0, 0 /* Try storing the data */
+ EXIT_USER_ACCESS($a2)
+ SET_FAULT_HANDLER($zero, $a2) /* Reset the fault handler */
+ li.d $a0, 0 /* Success */
+ jr $ra /* Return */
+END(suword32)
+
+/*
+ * int suword(volatile void *, long)
+ */
+ENTRY(suword)
+EENTRY(suword64)
+ li.d $a2, (VM_MAXUSER_ADDRESS-7)
+ bgeu $a0, $a2, fsu_fault_nopcb
+ la.pcrel $a6, _C_LABEL(fsu_fault) /* Load the fault handler */
+ SET_FAULT_HANDLER($a6, $a2) /* And set it */
+ ENTER_USER_ACCESS($a2)
+ st.d $a1, $a0, 0 /* Try storing the data */
+ EXIT_USER_ACCESS($a2)
+ SET_FAULT_HANDLER($zero, $a2) /* Reset the fault handler */
+ li.d $a0, 0 /* Success */
+ jr $ra /* Return */
+EEND(suword64)
+END(suword)
+
+ENTRY(setjmp)
+ /* Store the stack pointer */
+ st.d $sp, $a0, 0
+ addi.d $a0, $a0, 8
+
+ /* Store the general purpose registers and ra */
+ st.d $fp, $a0, (0 * 8)
+ st.d $s0, $a0, (1 * 8)
+ st.d $s1, $a0, (2 * 8)
+ st.d $s2, $a0, (3 * 8)
+ st.d $s3, $a0, (4 * 8)
+ st.d $s4, $a0, (5 * 8)
+ st.d $s5, $a0, (6 * 8)
+ st.d $s6, $a0, (7 * 8)
+ st.d $s7, $a0, (8 * 8)
+ st.d $s8, $a0, (9 * 8)
+ st.d $ra, $a0, (10 * 8)
+
+ /* Return value */
+ li.d $a0, 0
+ jr $ra
+END(setjmp)
+
+ENTRY(longjmp)
+ /* Restore the stack pointer */
+ ld.d $sp, $a0, 0
+ addi.d $a0, $a0, 8
+
+ /* Restore the general purpose registers and ra */
+ ld.d $fp, $a0, (0 * 8)
+ ld.d $s0, $a0, (1 * 8)
+ ld.d $s1, $a0, (2 * 8)
+ ld.d $s2, $a0, (3 * 8)
+ ld.d $s3, $a0, (4 * 8)
+ ld.d $s4, $a0, (5 * 8)
+ ld.d $s5, $a0, (6 * 8)
+ ld.d $s6, $a0, (7 * 8)
+ ld.d $s7, $a0, (8 * 8)
+ ld.d $s8, $a0, (9 * 8)
+ ld.d $ra, $a0, (10 * 8)
+
+ /* Load the return value */
+ move $a0, $a1
+ jr $ra
+END(longjmp)
diff --git a/sys/loongarch/loongarch/swtch.S b/sys/loongarch/loongarch/swtch.S
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/swtch.S
@@ -0,0 +1,451 @@
+/*-
+ * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "assym.inc"
+#include "opt_sched.h"
+
+#include <machine/param.h>
+#include <machine/asm.h>
+#include <machine/loongarchreg.h>
+#include <machine/pte.h>
+
+#define RSIZE 8
+
+.macro strel_mb
+ dbar 0x12
+.endm
+
+.macro ldacq_mb
+ dbar 0x14
+.endm
+
+.macro EX inst, reg, src, offset
+ li.d $t0, \offset
+ \inst \reg, \src, $t0
+.endm
+
+.macro __fpe_state_save p
+
+ /* Store registers */
+ EX fstx.d, $fa0, \p, (PCB_F + 0 * RSIZE)
+ EX fstx.d, $fa1, \p, (PCB_F + 1 * RSIZE)
+ EX fstx.d, $fa2, \p, (PCB_F + 2 * RSIZE)
+ EX fstx.d, $fa3, \p, (PCB_F + 3 * RSIZE)
+ EX fstx.d, $fa4, \p, (PCB_F + 4 * RSIZE)
+ EX fstx.d, $fa5, \p, (PCB_F + 5 * RSIZE)
+ EX fstx.d, $fa6, \p, (PCB_F + 6 * RSIZE)
+ EX fstx.d, $fa7, \p, (PCB_F + 7 * RSIZE)
+
+ EX fstx.d, $ft0, \p, (PCB_F + 8 * RSIZE)
+ EX fstx.d, $ft1, \p, (PCB_F + 9 * RSIZE)
+ EX fstx.d, $ft2, \p, (PCB_F + 10 * RSIZE)
+ EX fstx.d, $ft3, \p, (PCB_F + 11 * RSIZE)
+ EX fstx.d, $ft4, \p, (PCB_F + 12 * RSIZE)
+ EX fstx.d, $ft5, \p, (PCB_F + 13 * RSIZE)
+ EX fstx.d, $ft6, \p, (PCB_F + 14 * RSIZE)
+ EX fstx.d, $ft7, \p, (PCB_F + 15 * RSIZE)
+ EX fstx.d, $ft8, \p, (PCB_F + 16 * RSIZE)
+ EX fstx.d, $ft9, \p, (PCB_F + 17 * RSIZE)
+ EX fstx.d, $ft10, \p, (PCB_F + 18 * RSIZE)
+ EX fstx.d, $ft11, \p, (PCB_F + 19 * RSIZE)
+ EX fstx.d, $ft12, \p, (PCB_F + 20 * RSIZE)
+ EX fstx.d, $ft13, \p, (PCB_F + 21 * RSIZE)
+ EX fstx.d, $ft14, \p, (PCB_F + 22 * RSIZE)
+ EX fstx.d, $ft15, \p, (PCB_F + 23 * RSIZE)
+
+ EX fstx.d, $fs0, \p, (PCB_F + 24 * RSIZE)
+ EX fstx.d, $fs1, \p, (PCB_F + 25 * RSIZE)
+ EX fstx.d, $fs2, \p, (PCB_F + 26 * RSIZE)
+ EX fstx.d, $fs3, \p, (PCB_F + 27 * RSIZE)
+ EX fstx.d, $fs4, \p, (PCB_F + 28 * RSIZE)
+ EX fstx.d, $fs5, \p, (PCB_F + 29 * RSIZE)
+ EX fstx.d, $fs6, \p, (PCB_F + 30 * RSIZE)
+ EX fstx.d, $fs7, \p, (PCB_F + 31 * RSIZE)
+
+ movfcsr2gr $t1, $fcsr0
+ li.d $t0, (PCB_F + 32 * RSIZE) /* Push the FCSR0 register. */
+ stx.d $t1, \p, $t0
+
+.endm
+
+.macro __fpe_state_load p
+
+ /* Restore registers */
+ EX fldx.d, $fa0, \p, (PCB_F + 0 * RSIZE)
+ EX fldx.d, $fa1, \p, (PCB_F + 1 * RSIZE)
+ EX fldx.d, $fa2, \p, (PCB_F + 2 * RSIZE)
+ EX fldx.d, $fa3, \p, (PCB_F + 3 * RSIZE)
+ EX fldx.d, $fa4, \p, (PCB_F + 4 * RSIZE)
+ EX fldx.d, $fa5, \p, (PCB_F + 5 * RSIZE)
+ EX fldx.d, $fa6, \p, (PCB_F + 6 * RSIZE)
+ EX fldx.d, $fa7, \p, (PCB_F + 7 * RSIZE)
+
+ EX fldx.d, $ft0, \p, (PCB_F + 8 * RSIZE)
+ EX fldx.d, $ft1, \p, (PCB_F + 9 * RSIZE)
+ EX fldx.d, $ft2, \p, (PCB_F + 10 * RSIZE)
+ EX fldx.d, $ft3, \p, (PCB_F + 11 * RSIZE)
+ EX fldx.d, $ft4, \p, (PCB_F + 12 * RSIZE)
+ EX fldx.d, $ft5, \p, (PCB_F + 13 * RSIZE)
+ EX fldx.d, $ft6, \p, (PCB_F + 14 * RSIZE)
+ EX fldx.d, $ft7, \p, (PCB_F + 15 * RSIZE)
+ EX fldx.d, $ft8, \p, (PCB_F + 16 * RSIZE)
+ EX fldx.d, $ft9, \p, (PCB_F + 17 * RSIZE)
+ EX fldx.d, $ft10, \p, (PCB_F + 18 * RSIZE)
+ EX fldx.d, $ft11, \p, (PCB_F + 19 * RSIZE)
+ EX fldx.d, $ft12, \p, (PCB_F + 20 * RSIZE)
+ EX fldx.d, $ft13, \p, (PCB_F + 21 * RSIZE)
+ EX fldx.d, $ft14, \p, (PCB_F + 22 * RSIZE)
+ EX fldx.d, $ft15, \p, (PCB_F + 23 * RSIZE)
+
+ EX fldx.d, $fs0, \p, (PCB_F + 24 * RSIZE)
+ EX fldx.d, $fs1, \p, (PCB_F + 25 * RSIZE)
+ EX fldx.d, $fs2, \p, (PCB_F + 26 * RSIZE)
+ EX fldx.d, $fs3, \p, (PCB_F + 27 * RSIZE)
+ EX fldx.d, $fs4, \p, (PCB_F + 28 * RSIZE)
+ EX fldx.d, $fs5, \p, (PCB_F + 29 * RSIZE)
+ EX fldx.d, $fs6, \p, (PCB_F + 30 * RSIZE)
+ EX fldx.d, $fs7, \p, (PCB_F + 31 * RSIZE)
+
+ li.d $t0, (PCB_F + 32 * RSIZE)
+ ldx.d $t0, \p, $t0
+ movgr2fcsr $fcsr0, $t0 /* Pop the fcsr0 register. */
+
+.endm
+
+/*
+ * void
+ * fpe_state_save(struct thread *td)
+ */
+ENTRY(fpe_state_save)
+ /*
+ * Only save FPU state if FPU is enabled. fpe_state_save is
+ * called from fill_fpregs (ptrace) and get_fpcontext (signal
+ * delivery). After a context switch the hardware FPEN may have
+ * been disabled by another thread even though PCB_FP_STARTED is
+ * set for this thread. Accessing FPU registers with FPU disabled
+ * raises exception 0xf, which is fatal in kernel mode.
+ */
+ csrrd $t0, LOONGARCH_CSR_EUEN
+ andi $t0, $t0, CSR_EUEN_FPEN
+ beqz $t0, 1f
+ ld.d $a0, $a0, TD_PCB
+ __fpe_state_save $a0
+1:
+ jr $ra
+END(fpe_state_save)
+
+/*
+ * void
+ * fpe_state_clear(void)
+ */
+ENTRY(fpe_state_clear)
+ /*
+ * Only clear FPU registers if FPU is enabled. fpe_state_clear
+ * may be called from the FPDIS trap handler before FPU is enabled
+ * for the current thread. Accessing FPU registers with FPU
+ * disabled raises exception 0xf.
+ */
+ csrrd $t1, LOONGARCH_CSR_EUEN
+ andi $t1, $t1, CSR_EUEN_FPEN
+ beqz $t1, 1f
+
+ li.w $t0, 0
+ movgr2fr.d $fa0, $t0
+ movgr2fr.d $fa1, $t0
+ movgr2fr.d $fa2, $t0
+ movgr2fr.d $fa3, $t0
+ movgr2fr.d $fa4, $t0
+ movgr2fr.d $fa5, $t0
+ movgr2fr.d $fa6, $t0
+ movgr2fr.d $fa7, $t0
+
+ movgr2fr.d $ft0, $t0
+ movgr2fr.d $ft1, $t0
+ movgr2fr.d $ft2, $t0
+ movgr2fr.d $ft3, $t0
+ movgr2fr.d $ft4, $t0
+ movgr2fr.d $ft5, $t0
+ movgr2fr.d $ft6, $t0
+ movgr2fr.d $ft7, $t0
+ movgr2fr.d $ft8, $t0
+ movgr2fr.d $ft9, $t0
+ movgr2fr.d $ft10, $t0
+ movgr2fr.d $ft11, $t0
+ movgr2fr.d $ft12, $t0
+ movgr2fr.d $ft13, $t0
+ movgr2fr.d $ft14, $t0
+ movgr2fr.d $ft15, $t0
+
+ movgr2fr.d $fs0, $t0
+ movgr2fr.d $fs1, $t0
+ movgr2fr.d $fs2, $t0
+ movgr2fr.d $fs3, $t0
+ movgr2fr.d $fs4, $t0
+ movgr2fr.d $fs5, $t0
+ movgr2fr.d $fs6, $t0
+ movgr2fr.d $fs7, $t0
+
+1:
+ jr $ra
+END(fpe_state_clear)
+
+/*
+ * void cpu_throw(struct thread *old __unused, struct thread *new)
+ */
+ENTRY(cpu_throw)
+ /* Activate the new thread's pmap. */
+ move $s0, $a1
+ move $a0, $a1
+ bl _C_LABEL(pmap_activate_sw)
+ move $a0, $s0
+
+ /* Store the new curthread */
+ st.d $a0, $r21, PC_CURTHREAD
+ /* And the new pcb */
+ ld.d $a3, $a0, TD_PCB
+ st.d $a3, $r21, PC_CURPCB
+
+ /* s[0-8] */
+ ld.d $s0, $a3, (PCB_S + 0 * 8)
+ ld.d $s1, $a3, (PCB_S + 1 * 8)
+ ld.d $s2, $a3, (PCB_S + 2 * 8)
+ ld.d $s3, $a3, (PCB_S + 3 * 8)
+ ld.d $s4, $a3, (PCB_S + 4 * 8)
+ ld.d $s5, $a3, (PCB_S + 5 * 8)
+ ld.d $s6, $a3, (PCB_S + 6 * 8)
+ ld.d $s7, $a3, (PCB_S + 7 * 8)
+ ld.d $s8, $a3, (PCB_S + 8 * 8)
+
+ /* Load registers */
+ ld.d $ra, $a3, PCB_RA
+ ld.d $fp, $a3, PCB_FP
+ ld.d $tp, $a3, PCB_TP
+ ld.d $sp, $a3, PCB_SP
+
+ jr $ra
+END(cpu_throw)
+
+/*
+ * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
+ *
+ * a0 = old
+ * a1 = new
+ * a2 = mtx
+ */
+ENTRY(cpu_switch)
+ /* Save the old context. */
+ ld.d $a3, $a0, TD_PCB
+
+ /* Store ra, sp, fp and the callee-saved registers */
+ st.d $ra, $a3, PCB_RA
+ st.d $sp, $a3, PCB_SP
+ st.d $tp, $a3, PCB_TP
+ st.d $fp, $a3, PCB_FP
+
+ /* s[0-8] */
+ st.d $s0, $a3, (PCB_S + 0 * 8)
+ st.d $s1, $a3, (PCB_S + 1 * 8)
+ st.d $s2, $a3, (PCB_S + 2 * 8)
+ st.d $s3, $a3, (PCB_S + 3 * 8)
+ st.d $s4, $a3, (PCB_S + 4 * 8)
+ st.d $s5, $a3, (PCB_S + 5 * 8)
+ st.d $s6, $a3, (PCB_S + 6 * 8)
+ st.d $s7, $a3, (PCB_S + 7 * 8)
+ st.d $s8, $a3, (PCB_S + 8 * 8)
+
+ /* Activate the new thread's pmap */
+ move $s0, $a0
+ move $s1, $a1
+ move $s2, $a2
+ move $a0, $a1
+ bl _C_LABEL(pmap_activate_sw)
+ move $a1, $s1
+
+ /* Release the old thread */
+ strel_mb
+ st.d $s2, $s0, TD_LOCK
+#if defined(SCHED_ULE) && defined(SMP)
+ /* Spin if TD_LOCK points to a blocked_lock */
+ la.pcrel $s2, _C_LABEL(blocked_lock)
+1:
+ ld.d $t0, $a1, TD_LOCK
+ ldacq_mb
+ beq $t0, $s2, 1b
+#endif
+
+ /* Store the new curthread */
+ st.d $a1, $r21, PC_CURTHREAD
+ /* And the new pcb */
+ ld.d $a3, $a1, TD_PCB
+ st.d $a3, $r21, PC_CURPCB
+
+ /*
+ * Restore the saved context.
+ */
+
+ /* s[0-8] */
+ ld.d $s0, $a3, (PCB_S + 0 * 8)
+ ld.d $s1, $a3, (PCB_S + 1 * 8)
+ ld.d $s2, $a3, (PCB_S + 2 * 8)
+ ld.d $s3, $a3, (PCB_S + 3 * 8)
+ ld.d $s4, $a3, (PCB_S + 4 * 8)
+ ld.d $s5, $a3, (PCB_S + 5 * 8)
+ ld.d $s6, $a3, (PCB_S + 6 * 8)
+ ld.d $s7, $a3, (PCB_S + 7 * 8)
+ ld.d $s8, $a3, (PCB_S + 8 * 8)
+
+ /* Restore the registers */
+ ld.d $ra, $a3, PCB_RA
+ ld.d $fp, $a3, PCB_FP
+ ld.d $tp, $a3, PCB_TP
+ ld.d $sp, $a3, PCB_SP
+ jr $ra
+END(cpu_switch)
+
+/*
+ * fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
+ * struct trapframe *frame)
+ */
+
+ENTRY(fork_trampoline)
+
+ move $a0, $s0
+ move $a1, $s1
+ move $a2, $sp
+ bl _C_LABEL(fork_exit)
+
+ /*
+ * Disable interrupts IMMEDIATELY after fork_exit returns, before any
+ * register restore.
+ */
+ li.d $t0, 1
+ li.d $t1, CSR_CRMD_IE
+ csrxchg $t0, $t1, LOONGARCH_CSR_CRMD
+
+ /* Restore exception program counter */
+ ld.d $t0, $sp, TF_ERA
+ csrwr $t0, LOONGARCH_CSR_ERA
+
+ /* Restore PRMD */
+ ld.d $t0, $sp, TF_PRMD
+ csrwr $t0, LOONGARCH_CSR_PRMD
+
+ /* Restore EUEN if FPEN changed */
+ ld.d $t1, $sp, TF_EUEN
+ csrrd $t0, LOONGARCH_CSR_EUEN
+ andi $t0, $t0, CSR_EUEN_FPEN
+ andi $t1, $t1, CSR_EUEN_FPEN
+ beq $t0, $t1, 1f
+ ld.d $t0, $sp, TF_EUEN
+ csrwr $t0, LOONGARCH_CSR_EUEN
+1:
+
+ /* Restore the registers */
+ /* t0 and t1 will be used later */
+ ld.d $t2, $sp, (TF_T + 2 * RSIZE)
+ ld.d $t3, $sp, (TF_T + 3 * RSIZE)
+ ld.d $t4, $sp, (TF_T + 4 * RSIZE)
+ ld.d $t5, $sp, (TF_T + 5 * RSIZE)
+ ld.d $t6, $sp, (TF_T + 6 * RSIZE)
+ ld.d $t7, $sp, (TF_T + 7 * RSIZE)
+ ld.d $t8, $sp, (TF_T + 8 * RSIZE)
+
+ ld.d $s0, $sp, (TF_S + 0 * RSIZE)
+ ld.d $s1, $sp, (TF_S + 1 * RSIZE)
+ ld.d $s2, $sp, (TF_S + 2 * RSIZE)
+ ld.d $s3, $sp, (TF_S + 3 * RSIZE)
+ ld.d $s4, $sp, (TF_S + 4 * RSIZE)
+ ld.d $s5, $sp, (TF_S + 5 * RSIZE)
+ ld.d $s6, $sp, (TF_S + 6 * RSIZE)
+ ld.d $s7, $sp, (TF_S + 7 * RSIZE)
+ ld.d $s8, $sp, (TF_S + 8 * RSIZE)
+
+ ld.d $a0, $sp, (TF_A + 0 * RSIZE)
+ ld.d $a1, $sp, (TF_A + 1 * RSIZE)
+ ld.d $a2, $sp, (TF_A + 2 * RSIZE)
+ ld.d $a3, $sp, (TF_A + 3 * RSIZE)
+ ld.d $a4, $sp, (TF_A + 4 * RSIZE)
+ ld.d $a5, $sp, (TF_A + 5 * RSIZE)
+ ld.d $a6, $sp, (TF_A + 6 * RSIZE)
+ ld.d $a7, $sp, (TF_A + 7 * RSIZE)
+
+ addi.d $t0, $sp, TF_SIZE
+ /* Save kernel stack so we can use it doing a user trap */
+ st.d $t0, $r21, PC_KERNEL_SP
+
+ /* Fixup t0, t1. */
+ ld.d $t0, $sp, (TF_T + 0 * RSIZE)
+ ld.d $t1, $sp, (TF_T + 1 * RSIZE)
+
+ /* Load user ra */
+ ld.d $ra, $sp, TF_RA
+
+ /* Restore user tp, fp, r21 from trapframe */
+ ld.d $tp, $sp, TF_TP
+ ld.d $fp, $sp, TF_R22
+ ld.d $r21, $sp, TF_R21
+
+ /* Load user stack */
+ ld.d $sp, $sp, TF_SP
+
+ ertn
+END(fork_trampoline)
+
+ENTRY(savectx)
+ /* Store ra, sp tp, fp and the callee-saved registers */
+ st.d $ra, $a0, (PCB_RA)
+ st.d $sp, $a0, (PCB_SP)
+ st.d $tp, $a0, (PCB_TP)
+ st.d $fp, $a0, (PCB_FP)
+
+ /* s[0-8] */
+ st.d $s0, $a0, (PCB_S + 0 * RSIZE)
+ st.d $s1, $a0, (PCB_S + 1 * RSIZE)
+ st.d $s2, $a0, (PCB_S + 2 * RSIZE)
+ st.d $s3, $a0, (PCB_S + 3 * RSIZE)
+ st.d $s4, $a0, (PCB_S + 4 * RSIZE)
+ st.d $s5, $a0, (PCB_S + 5 * RSIZE)
+ st.d $s6, $a0, (PCB_S + 6 * RSIZE)
+ st.d $s7, $a0, (PCB_S + 7 * RSIZE)
+ st.d $s8, $a0, (PCB_S + 8 * RSIZE)
+
+ /*
+ * Only save FPU state if FPU is enabled. savectx is called from
+ * IPI_STOP and dump_savectx() which may fire while FPU is disabled
+ * (e.g. on an AP that hasn't entered a FPU-using thread yet).
+ * Accessing FPU registers with FPU disabled raises exception 0xf,
+ * which is fatal in an interrupt context.
+ */
+ ld.d $t0, $r21, PC_CURTHREAD
+ ld.d $t0, $t0, TD_FRAME
+ ld.d $t1, $t0, TF_EUEN
+ andi $t2, $t1, CSR_EUEN_FPEN
+ beqz $t2, 1f
+
+ __fpe_state_save $a0
+1:
+ jr $ra
+END(savectx)
diff --git a/sys/loongarch/loongarch/sys_machdep.c b/sys/loongarch/loongarch/sys_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/sys_machdep.c
@@ -0,0 +1,39 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/sysproto.h>
+
+#include <machine/sysarch.h>
+
+int
+sysarch(struct thread *td, struct sysarch_args *uap)
+{
+
+ return (ENOTSUP);
+}
diff --git a/sys/loongarch/loongarch/timer.c b/sys/loongarch/loongarch/timer.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/timer.c
@@ -0,0 +1,296 @@
+/*-
+ * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * LoongArch Timer
+ */
+
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/timeet.h>
+#include <sys/timetc.h>
+#include <sys/vdso.h>
+#include <sys/watchdog.h>
+
+#include <machine/cpufunc.h>
+#include <machine/intr.h>
+#include <machine/md_var.h>
+
+#include <dev/ofw/openfirm.h>
+
+struct loongarch_timer_softc {
+ void *ih;
+ uint32_t clkfreq;
+ struct eventtimer et;
+};
+static struct loongarch_timer_softc *loongarch_timer_sc = NULL;
+
+static timecounter_get_t loongarch_timer_tc_get_timecount;
+static timecounter_fill_vdso_timehands_t loongarch_timer_tc_fill_vdso_timehands;
+
+static struct timecounter loongarch_timer_timecount = {
+ .tc_name = "LoongArch Timecounter",
+ .tc_get_timecount = loongarch_timer_tc_get_timecount,
+ .tc_poll_pps = NULL,
+ .tc_counter_mask = ~0u,
+ .tc_frequency = 0,
+ .tc_quality = 1000,
+ .tc_fill_vdso_timehands = loongarch_timer_tc_fill_vdso_timehands,
+};
+
+static inline uint64_t
+get_timecount(void)
+{
+ return (rdtime());
+}
+
+static u_int
+loongarch_timer_tc_get_timecount(struct timecounter *tc __unused)
+{
+ return (get_timecount());
+}
+
+static uint32_t
+loongarch_timer_tc_fill_vdso_timehands(struct vdso_timehands *vdso_th,
+ struct timecounter *tc)
+{
+ vdso_th->th_algo = VDSO_TH_ALGO_LOONGARCH_RDTIME;
+ bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
+ return (1);
+}
+
+static int
+loongarch_timer_et_start(struct eventtimer *et, sbintime_t first,
+ sbintime_t period)
+{
+ uint64_t counts;
+ uint64_t val;
+
+ if (first == 0 && period == 0)
+ return (EINVAL);
+
+ if (first != 0) {
+ counts = ((uint32_t)et->et_frequency * first) >> 32;
+ } else {
+ counts = ((uint32_t)et->et_frequency * period) >> 32;
+ }
+
+ counts &= CSR_TCFG_VAL;
+ val = counts | CSR_TCFG_EN;
+
+ if (period) {
+ val |= CSR_TCFG_PERIOD;
+ } else {
+ val &= ~CSR_TCFG_PERIOD;
+ }
+
+ csr_write64(val, LOONGARCH_CSR_TCFG);
+
+ return (0);
+}
+
+static int
+loongarch_timer_et_stop(struct eventtimer *et)
+{
+ uint64_t val;
+
+ val = csr_read64(LOONGARCH_CSR_TCFG);
+ /* Disable timer */
+ val &= ~CSR_TCFG_EN;
+ csr_write64(val, LOONGARCH_CSR_TCFG);
+ return (0);
+}
+
+static int
+loongarch_timer_intr(void *arg)
+{
+ struct loongarch_timer_softc *sc;
+
+ sc = (struct loongarch_timer_softc *)arg;
+
+ /* Clear the timer interrupt. */
+ csr_write32(1, LOONGARCH_CSR_TINTCLR);
+
+ if (sc->et.et_active)
+ sc->et.et_event_cb(&sc->et, sc->et.et_arg);
+
+ return (FILTER_HANDLED);
+}
+
+static int
+loongarch_timer_get_timebase(device_t dev, uint32_t *freq)
+{
+ uint32_t res;
+ uint32_t base_freq;
+ uint32_t cfm, cfd;
+
+ res = read_cpucfg(LOONGARCH_CPUCFG2);
+ if (!(res & CPUCFG2_LLFTP))
+ return (ENXIO);
+ base_freq = read_cpucfg(LOONGARCH_CPUCFG4);
+ res = read_cpucfg(LOONGARCH_CPUCFG5);
+ cfm = res & 0xffff;
+ cfd = (res >> 16) & 0xffff;
+
+ if (!base_freq || !cfm || !cfd)
+ return (ENXIO);
+
+ *freq = (base_freq * cfm / cfd);
+
+ return (0);
+}
+
+static int
+loongarch_timer_probe(device_t dev)
+{
+
+ device_set_desc(dev, "LoongArch Timer");
+
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+loongarch_timer_attach(device_t dev)
+{
+ struct loongarch_timer_softc *sc;
+ int error;
+
+ sc = device_get_softc(dev);
+ if (loongarch_timer_sc != NULL)
+ return (ENXIO);
+
+ if (device_get_unit(dev) != 0)
+ return (ENXIO);
+
+ if (loongarch_timer_get_timebase(dev, &sc->clkfreq) != 0) {
+ device_printf(dev, "No clock frequency specified\n");
+ return (ENXIO);
+ }
+
+ loongarch_timer_sc = sc;
+
+ /*
+ * Register the timer interrupt handler on the cpuic IRQ_TI ISRC via
+ * the cpuic driver's CPU-local helper. bus_setup_intr can't be used
+ * here because the timer has no FDT interrupt node.
+ */
+ error = cpuic_setup_cpu_intr(device_get_nameunit(dev),
+ loongarch_timer_intr, NULL, sc, IRQ_TI, INTR_TYPE_CLK,
+ &sc->ih);
+ if (error) {
+ device_printf(dev, "Unable to alloc int resource.\n");
+ return (ENXIO);
+ }
+
+ loongarch_timer_timecount.tc_frequency = sc->clkfreq;
+ loongarch_timer_timecount.tc_priv = sc;
+ tc_init(&loongarch_timer_timecount);
+
+ sc->et.et_name = "LoongArch Eventtimer";
+ sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERIODIC |
+ ET_FLAGS_PERCPU;
+ sc->et.et_quality = 1000;
+
+ sc->et.et_frequency = sc->clkfreq;
+ sc->et.et_min_period = (0x00000002LLU << 32) / sc->et.et_frequency;
+ sc->et.et_max_period = (0xfffffffeLLU << 32) / sc->et.et_frequency;
+ sc->et.et_start = loongarch_timer_et_start;
+ sc->et.et_stop = loongarch_timer_et_stop;
+ sc->et.et_priv = sc;
+ et_register(&sc->et);
+
+ set_cputicker(get_timecount, sc->clkfreq, false);
+
+ return (0);
+}
+
+static device_method_t loongarch_timer_methods[] = {
+ DEVMETHOD(device_probe, loongarch_timer_probe),
+ DEVMETHOD(device_attach, loongarch_timer_attach),
+ { 0, 0 }
+};
+
+static driver_t loongarch_timer_driver = {
+ "timer",
+ loongarch_timer_methods,
+ sizeof(struct loongarch_timer_softc),
+};
+
+EARLY_DRIVER_MODULE(timer, nexus, loongarch_timer_driver, 0, 0,
+ BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE);
+
+void
+DELAY(int usec)
+{
+ int64_t counts, counts_per_usec;
+ uint64_t first, last;
+
+ /*
+ * Check the timers are setup, if not just
+ * use a for loop for the meantime
+ */
+ if (loongarch_timer_sc == NULL) {
+ for (; usec > 0; usec--)
+ for (counts = 200; counts > 0; counts--)
+ /*
+ * Prevent the compiler from optimizing
+ * out the loop
+ */
+ cpufunc_nullop();
+ return;
+ }
+ TSENTER();
+
+ /* Get the number of times to count */
+ counts_per_usec = ((loongarch_timer_timecount.tc_frequency /
+ 1000000) + 1);
+
+ /*
+ * Clamp the timeout at a maximum value (about 32 seconds with
+ * a 66MHz clock). *Nobody* should be delay()ing for anywhere
+ * near that length of time and if they are, they should be hung
+ * out to dry.
+ */
+ if (usec >= (0x80000000U / counts_per_usec))
+ counts = (0x80000000U / counts_per_usec) - 1;
+ else
+ counts = usec * counts_per_usec;
+
+ first = get_timecount();
+
+ while (counts > 0) {
+ last = get_timecount();
+ counts -= (int64_t)(last - first);
+ first = last;
+ }
+ TSEXIT();
+}
diff --git a/sys/loongarch/loongarch/trap.c b/sys/loongarch/loongarch/trap.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/trap.c
@@ -0,0 +1,567 @@
+/*-
+ * Copyright (c) 2024 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include "opt_ddb.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/linker.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/bus.h>
+#include <sys/proc.h>
+#include <sys/ptrace.h>
+#include <sys/syscall.h>
+#include <sys/sysent.h>
+#include <sys/smp.h>
+#ifdef KDB
+#include <sys/kdb.h>
+#endif
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+#include <vm/vm_param.h>
+#include <vm/vm_extern.h>
+
+#include <machine/fpe.h>
+#include <machine/frame.h>
+#include <machine/loongarchreg.h>
+#include <machine/pcb.h>
+#include <machine/pcpu.h>
+#include <machine/pte.h>
+#include <machine/watch.h>
+#include <machine/vmparam.h>
+
+#include <machine/resource.h>
+#include <machine/intr.h>
+#include <machine/machdep.h>
+#include <machine/cpufunc.h>
+
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+#endif
+
+#ifdef DDB
+#include <ddb/ddb.h>
+#include <ddb/db_sym.h>
+#endif
+
+#define EXC_CODE(estat) ((estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT)
+
+int (*dtrace_invop_jump_addr)(struct trapframe *);
+
+/* Called from exception.S */
+void do_trap_supervisor(struct trapframe *);
+void do_trap_user(struct trapframe *);
+
+static __inline void
+call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
+{
+ ksiginfo_t ksi;
+
+ ksiginfo_init_trap(&ksi);
+ ksi.ksi_signo = sig;
+ ksi.ksi_code = code;
+ ksi.ksi_addr = addr;
+ ksi.ksi_trapno = trapno;
+ trapsignal(td, &ksi);
+}
+
+int
+cpu_fetch_syscall_args(struct thread *td)
+{
+ struct proc *p;
+ syscallarg_t *ap, *dst_ap;
+ struct syscall_args *sa;
+
+ p = td->td_proc;
+ sa = &td->td_sa;
+
+ /* Save original a0 before it is overwritten by return value. */
+ td->td_frame->tf_a0 = td->td_frame->tf_a[0];
+
+ ap = &td->td_frame->tf_a[0];
+ dst_ap = &sa->args[0];
+
+ sa->code = td->td_frame->tf_a[7];
+ sa->original_code = sa->code;
+
+ if (__predict_false(sa->code == SYS_syscall ||
+ sa->code == SYS___syscall)) {
+ sa->code = *ap++;
+ } else {
+ *dst_ap++ = *ap++;
+ }
+
+ if (__predict_false(sa->code >= p->p_sysent->sv_size))
+ sa->callp = &p->p_sysent->sv_table[0];
+ else
+ sa->callp = &p->p_sysent->sv_table[sa->code];
+
+ KASSERT(sa->callp->sy_narg <= nitems(sa->args),
+ ("Syscall %d takes too many arguments", sa->code));
+
+ memcpy(dst_ap, ap, (NARGREG - 1) * sizeof(*dst_ap));
+
+ td->td_retval[0] = 0;
+ td->td_retval[1] = 0;
+
+ return (0);
+}
+
+#include "../../kern/subr_syscall.c"
+
+static void
+print_with_symbol(const char *name, uint64_t value)
+{
+#ifdef DDB
+ c_db_sym_t sym;
+ db_expr_t sym_value;
+ db_expr_t offset;
+ const char *sym_name;
+#endif
+
+ printf("%7s: 0x%016lx", name, value);
+
+#ifdef DDB
+ if (value >= VM_MIN_KERNEL_ADDRESS) {
+ sym = db_search_symbol(value, DB_STGY_ANY, &offset);
+ if (sym != C_DB_SYM_NULL) {
+ db_symbol_values(sym, &sym_name, &sym_value);
+ if (offset != 0)
+ printf(" (%s + 0x%lx)", sym_name, offset);
+ else
+ printf(" (%s)", sym_name);
+ }
+ }
+#endif
+ printf("\n");
+}
+
+static void
+dump_regs(struct trapframe *frame)
+{
+ char name[6];
+ int i;
+
+ for (i = 0; i < nitems(frame->tf_t); i++) {
+ snprintf(name, sizeof(name), "t[%d]", i);
+ print_with_symbol(name, frame->tf_t[i]);
+ }
+
+ for (i = 0; i < nitems(frame->tf_s); i++) {
+ snprintf(name, sizeof(name), "s[%d]", i);
+ print_with_symbol(name, frame->tf_s[i]);
+ }
+
+ for (i = 0; i < nitems(frame->tf_a); i++) {
+ snprintf(name, sizeof(name), "a[%d]", i);
+ print_with_symbol(name, frame->tf_a[i]);
+ }
+
+ print_with_symbol("ra", frame->tf_ra);
+ print_with_symbol("sp", frame->tf_sp);
+ print_with_symbol("tp", frame->tf_tp);
+ print_with_symbol("fp", frame->tf_fp);
+ printf("r21: 0x%016lx\n", frame->r21);
+ printf("era: 0x%016lx\n", frame->tf_era);
+ printf("crmd: 0x%016lx\n", frame->tf_crmd);
+ printf("prmd: 0x%016lx\n", frame->tf_prmd);
+ printf("euen: 0x%016lx\n", frame->tf_euen);
+ printf("ecfg: 0x%016lx\n", frame->tf_ecfg);
+ printf("estat: 0x%016lx\n", frame->tf_estat);
+ printf("badvaddr: 0x%016lx\n", frame->tf_badvaddr);
+}
+
+static void
+syscall_handler(void)
+{
+ struct thread *td;
+
+ td = curthread;
+
+ syscallenter(td);
+ syscallret(td);
+}
+
+static void
+page_fault_handler(struct trapframe *frame, int usermode)
+{
+ struct vm_map *map;
+ uint64_t evaddr;
+ struct thread *td;
+ struct pcb *pcb;
+ vm_prot_t ftype;
+ vm_offset_t va;
+ struct proc *p;
+ int error, sig, ucode;
+#ifdef KDB
+ bool handled;
+#endif
+
+#ifdef KDB
+ if (kdb_active) {
+ kdb_reenter();
+ return;
+ }
+#endif
+
+ td = curthread;
+ p = td->td_proc;
+ pcb = td->td_pcb;
+ evaddr = frame->tf_badvaddr;
+
+ if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 ||
+ WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
+ "Kernel page fault") != 0)
+ goto fatal;
+
+ if (usermode) {
+ if (!VIRT_IS_VALID(evaddr)) {
+ call_trapsignal(td, SIGSEGV, SEGV_MAPERR,
+ (void *)evaddr, EXC_CODE(frame->tf_estat));
+ goto done;
+ }
+ map = &p->p_vmspace->vm_map;
+ } else {
+ if (evaddr >= VM_MIN_KERNEL_ADDRESS) {
+ map = kernel_map;
+ } else {
+ if (pcb->pcb_onfault == 0)
+ goto fatal;
+ map = &p->p_vmspace->vm_map;
+ }
+ }
+
+ va = trunc_page(evaddr);
+
+ if (EXC_CODE(frame->tf_estat) == EXCCODE_TLBS ||
+ EXC_CODE(frame->tf_estat) == EXCCODE_TLBM) {
+ ftype = VM_PROT_WRITE;
+ } else if (EXC_CODE(frame->tf_estat) == EXCCODE_TLBI ||
+ EXC_CODE(frame->tf_estat) == EXCCODE_TLBNX) {
+ ftype = VM_PROT_EXECUTE;
+ } else {
+ ftype = VM_PROT_READ;
+ }
+
+ if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype))
+ goto done;
+
+ error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode);
+ if (error != KERN_SUCCESS) {
+ if (usermode) {
+ call_trapsignal(td, sig, ucode, (void *)evaddr,
+ EXC_CODE(frame->tf_estat));
+ } else {
+ if (pcb->pcb_onfault != 0) {
+ frame->tf_a[0] = error;
+ frame->tf_era = pcb->pcb_onfault;
+ return;
+ }
+ goto fatal;
+ }
+ }
+
+done:
+ if (usermode)
+ userret(td, frame);
+ return;
+
+fatal:
+ dump_regs(frame);
+#ifdef KDB
+ if (debugger_on_trap) {
+ kdb_why = KDB_WHY_TRAP;
+ handled = kdb_trap(EXC_CODE(frame->tf_estat), 0, frame);
+ kdb_why = KDB_WHY_UNSET;
+ if (handled)
+ return;
+ }
+#endif
+ panic("Fatal page fault at %#lx: %#016lx", frame->tf_era, evaddr);
+}
+
+void
+do_trap_supervisor(struct trapframe *frame)
+{
+ uint64_t exception;
+
+ /* Ensure we came from supervisor mode, interrupts disabled */
+ exception = EXC_CODE(frame->tf_estat);
+
+ if (EXC_CODE(frame->tf_estat) == EXCCODE_RSV) {
+ /* Interrupt */
+ intr_irq_handler(frame, INTR_ROOT_IRQ);
+ return;
+ }
+
+#ifdef KDTRACE_HOOKS
+ if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
+ return;
+#endif
+
+ switch (exception) {
+ case EXCCODE_ADE:
+ dump_regs(frame);
+ panic("Memory access exception at 0x%016lx\n", frame->tf_era);
+ break;
+ case EXCCODE_ALE:
+ dump_regs(frame);
+ panic("Misaligned address exception at %#016lx: %#016lx\n",
+ frame->tf_era, frame->tf_badvaddr);
+ break;
+ case EXCCODE_TLBL:
+ case EXCCODE_TLBS:
+ case EXCCODE_TLBI:
+ case EXCCODE_TLBM:
+ case EXCCODE_TLBNR:
+ case EXCCODE_TLBNX:
+ case EXCCODE_TLBPE:
+ if (curthread->td_md.md_spinlock_count == 0 &&
+ (frame->tf_prmd & CSR_PRMD_PIE) != 0)
+ intr_enable();
+ page_fault_handler(frame, 0);
+ break;
+ case EXCCODE_BP:
+#ifdef KDTRACE_HOOKS
+ if (dtrace_invop_jump_addr != NULL &&
+ dtrace_invop_jump_addr(frame) == 0)
+ break;
+#endif
+#ifdef KDB
+ kdb_trap(exception, 0, frame);
+#else
+ dump_regs(frame);
+ panic("No debugger in kernel.\n");
+#endif
+ break;
+ case EXCCODE_WATCH:
+ la_watch_kern_handler(frame);
+#ifdef KDB
+ kdb_trap(exception, 0, frame);
+#else
+ dump_regs(frame);
+ panic("Watchpoint exception in kernel at %#lx\n",
+ (unsigned long)frame->tf_era);
+#endif
+ break;
+ case EXCCODE_INE:
+ dump_regs(frame);
+ panic("Illegal instruction at 0x%016lx\n", frame->tf_era);
+ break;
+ case EXCCODE_FPDIS:
+ dump_regs(frame);
+ panic("FPU disabled in kernel at 0x%016lx\n", frame->tf_era);
+ break;
+ case EXCCODE_FPE:
+ dump_regs(frame);
+ panic("FPU exception in kernel at 0x%016lx\n", frame->tf_era);
+ break;
+ default:
+ dump_regs(frame);
+ panic("Unknown kernel exception %lx trap value %lx\n",
+ exception, frame->tf_badvaddr);
+ }
+}
+
+void
+do_trap_user(struct trapframe *frame)
+{
+ uint64_t exception;
+ struct thread *td;
+ struct pcb *pcb;
+
+ td = curthread;
+ pcb = td->td_pcb;
+
+ KASSERT(td->td_frame == frame,
+ ("%s: td_frame %p != frame %p", __func__, td->td_frame, frame));
+
+ /* Ensure we came from usermode, interrupts disabled */
+ exception = EXC_CODE(frame->tf_estat);
+ if (EXC_CODE(frame->tf_estat) == EXCCODE_RSV) {
+ /* Interrupt */
+ intr_irq_handler(frame, INTR_ROOT_IRQ);
+ return;
+ }
+
+ /*
+ * Re-enable interrupts for synchronous exceptions from userland
+ * (syscall, page faults, ...). The exception entry runs with IE=0
+ * (hardware clears CRMD.IE on trap); without this the whole syscall
+ * runs interrupts-disabled, so IPIs sent to the syscall CPU (rmlock
+ * smp_rendezvous, TLB shootdown) are deferred until ertn -- which
+ * deadlocks rmlock rendezvous (the reader spins in syscall context
+ * and never services IPI_RENDEZVOUS) and lets stale TLBs persist
+ * during the syscall (buffer-cache corruption). Mirrors arm64
+ * do_el0_sync(), which calls intr_enable() before the exception
+ * switch. Interrupts (EXCCODE_RSV, handled above) keep IE=0 until
+ * EOI and are unaffected.
+ */
+ intr_enable();
+
+ CTR4(KTR_TRAP, "%s: exception=%lu, pc=%lx, badvaddr=%lx", __func__,
+ exception, frame->tf_era, frame->tf_badvaddr);
+
+ switch (exception) {
+ case EXCCODE_ADE:
+ call_trapsignal(td, SIGBUS, BUS_ADRERR, (void *)frame->tf_badvaddr,
+ exception);
+ userret(td, frame);
+ break;
+ case EXCCODE_ALE:
+ call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_badvaddr,
+ exception);
+ userret(td, frame);
+ break;
+ case EXCCODE_BCE:
+ /*
+ * Bounds Check Error: raised by the asrtle/asrtgt and bounded
+ * load/store (ldle/ldgt/stle/stgt/...) instructions.
+ * Deliver SIGSEGV rather than taking the kernel down.
+ * tf_badvaddr holds the checked address for the
+ * load/store forms.
+ */
+ call_trapsignal(td, SIGSEGV, SEGV_ACCERR,
+ (void *)frame->tf_badvaddr, exception);
+ userret(td, frame);
+ break;
+ case EXCCODE_TLBL:
+ case EXCCODE_TLBS:
+ case EXCCODE_TLBI:
+ case EXCCODE_TLBM:
+ case EXCCODE_TLBNR:
+ case EXCCODE_TLBNX:
+ case EXCCODE_TLBPE:
+ page_fault_handler(frame, 1);
+ break;
+ case EXCCODE_SYS:
+ frame->tf_era += 4; /* Next instruction */
+ syscall_handler();
+ break;
+ case EXCCODE_INE:
+ call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_era,
+ exception);
+ userret(td, frame);
+ break;
+ case EXCCODE_IPE:
+ /* Privileged instruction executed from user mode */
+ call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_era,
+ exception);
+ userret(td, frame);
+ break;
+ case EXCCODE_FPDIS:
+ /*
+ * FPU Disabled trap. Enable FPU and retry the
+ * instruction. FPU may be disabled after fork()
+ * even though PCB_FP_STARTED is set, so always
+ * re-enable it here. STARTED only gates whether
+ * this is the first time the thread uses FPU.
+ */
+ if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0)
+ pcb->pcb_fpflags |= PCB_FP_STARTED;
+ frame->tf_euen |= CSR_EUEN_FPEN;
+ break;
+ case EXCCODE_LSXDIS:
+ /*
+ * LSX (128-bit SIMD) disabled: enable LSX+FP and retry.
+ * Vector state is saved/restored from tf_vregs by exception.S;
+ * first use finds it zeroed (exec_setregs memsets the
+ * trapframe).
+ */
+ pcb->pcb_fpflags |= PCB_FP_STARTED | PCB_FP_LSX_STARTED;
+ frame->tf_euen |= CSR_EUEN_FPEN | CSR_EUEN_LSXEN;
+ break;
+ case EXCCODE_LASXDIS:
+ /*
+ * LASX (256-bit SIMD) disabled: enable LASX (which implies
+ * LSX+FP) and retry.
+ */
+ pcb->pcb_fpflags |= PCB_FP_STARTED | PCB_FP_LSX_STARTED |
+ PCB_FP_LASX_STARTED;
+ frame->tf_euen |= CSR_EUEN_FPEN | CSR_EUEN_LSXEN |
+ CSR_EUEN_LASXEN;
+ break;
+ case EXCCODE_BTDIS:
+ /* LBT (binary translation) is not managed; SIGILL. */
+ call_trapsignal(td, SIGILL, ILL_ILLOPC, (void *)frame->tf_era,
+ exception);
+ userret(td, frame);
+ break;
+ case EXCCODE_FPE: {
+ uint64_t fcsr = frame->fcsr0;
+ uint64_t cause_mask;
+ int code;
+
+ /* Decode the FCSR0 Cause bits into si_code. */
+ if (fcsr & FPU_CSR_INV_X)
+ code = FPE_FLTINV;
+ else if (fcsr & FPU_CSR_DIV_X)
+ code = FPE_FLTDIV;
+ else if (fcsr & FPU_CSR_OVF_X)
+ code = FPE_FLTOVF;
+ else if (fcsr & FPU_CSR_UDF_X)
+ code = FPE_FLTUND;
+ else if (fcsr & FPU_CSR_INE_X)
+ code = FPE_FLTRES;
+ else
+ code = FPE_FLTINV;
+
+ /*
+ * Clear the enabled Cause bits in the saved FCSR0 so the
+ * faulting instruction does not re-trap FPE on ertn.
+ * Shift Enable bits [4:0] to align with Cause bits [28:24],
+ * keep only the enabled causes, and clear them.
+ */
+ cause_mask = (fcsr & FPU_CSR_ALL_E) <<
+ (ffsll(FPU_CSR_ALL_X) - ffsll(FPU_CSR_ALL_E));
+ frame->fcsr0 = fcsr & ~(cause_mask & FPU_CSR_ALL_X);
+
+ call_trapsignal(td, SIGFPE, code, (void *)frame->tf_era,
+ exception);
+ userret(td, frame);
+ break;
+ }
+ case EXCCODE_BP:
+ call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_era,
+ exception);
+ userret(td, frame);
+ break;
+ case EXCCODE_WATCH:
+ la_watch_handler(td, frame);
+ userret(td, frame);
+ break;
+ default:
+ dump_regs(frame);
+ panic("Unknown userland exception %lx, trap value %lx\n",
+ exception, frame->tf_badvaddr);
+ }
+}
diff --git a/sys/loongarch/loongarch/uio_machdep.c b/sys/loongarch/loongarch/uio_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/uio_machdep.c
@@ -0,0 +1,132 @@
+/*-
+ * Copyright (c) 2004 Alan L. Cox <alc@cs.rice.edu>
+ * Copyright (c) 1982, 1986, 1991, 1993
+ * The Regents of the University of California. All rights reserved.
+ * (c) UNIX System Laboratories, Inc.
+ * All or some portions of this file are derived from material licensed
+ * to the University of California by American Telephone and Telegraph
+ * Co. or Unix System Laboratories, Inc. and are reproduced herein with
+ * the permission of UNIX System Laboratories, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+
+#include <machine/vmparam.h>
+
+/*
+ * Implement uiomove(9) from physical memory using the direct map to
+ * avoid the creation and destruction of ephemeral mappings.
+ */
+int
+uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
+{
+ struct thread *td = curthread;
+ struct iovec *iov;
+ void *cp, *vaddr;
+ vm_offset_t page_offset;
+ size_t cnt;
+ int error = 0;
+ int save = 0;
+ bool mapped;
+
+ KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
+ ("uiomove_fromphys: mode"));
+ KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
+ ("uiomove_fromphys proc"));
+ save = td->td_pflags & TDP_DEADLKTREAT;
+ td->td_pflags |= TDP_DEADLKTREAT;
+ mapped = false;
+ while (n > 0 && uio->uio_resid) {
+ iov = uio->uio_iov;
+ cnt = iov->iov_len;
+ if (cnt == 0) {
+ uio->uio_iov++;
+ uio->uio_iovcnt--;
+ continue;
+ }
+ if (cnt > n)
+ cnt = n;
+ page_offset = offset & PAGE_MASK;
+ cnt = min(cnt, PAGE_SIZE - page_offset);
+ if (uio->uio_segflg != UIO_NOCOPY) {
+ mapped = pmap_map_io_transient(
+ &ma[offset >> PAGE_SHIFT], &vaddr, 1, true);
+ cp = (char *)vaddr + page_offset;
+ }
+ switch (uio->uio_segflg) {
+ case UIO_USERSPACE:
+ maybe_yield();
+ if (uio->uio_rw == UIO_READ)
+ error = copyout(cp, iov->iov_base, cnt);
+ else
+ error = copyin(iov->iov_base, cp, cnt);
+ if (error)
+ goto out;
+ break;
+ case UIO_SYSSPACE:
+ if (uio->uio_rw == UIO_READ)
+ bcopy(cp, iov->iov_base, cnt);
+ else
+ bcopy(iov->iov_base, cp, cnt);
+ break;
+ case UIO_NOCOPY:
+ break;
+ }
+ if (__predict_false(mapped)) {
+ pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT],
+ &vaddr, 1, true);
+ mapped = false;
+ }
+ iov->iov_base = (char *)iov->iov_base + cnt;
+ iov->iov_len -= cnt;
+ uio->uio_resid -= cnt;
+ uio->uio_offset += cnt;
+ offset += cnt;
+ n -= cnt;
+ }
+out:
+ if (__predict_false(mapped)) {
+ panic("TODO 3");
+ pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1,
+ true);
+ }
+ if (save == 0)
+ td->td_pflags &= ~TDP_DEADLKTREAT;
+ return (error);
+}
diff --git a/sys/loongarch/loongarch/unwind.c b/sys/loongarch/loongarch/unwind.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/unwind.c
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/proc.h>
+
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+bool
+unwind_frame(struct thread *td, struct unwind_state *frame)
+{
+ uintptr_t fp;
+
+ fp = frame->fp;
+
+ if (!__is_aligned(fp, sizeof(fp)) ||
+ !kstack_contains(td, fp - sizeof(fp) * 2, sizeof(fp) * 2))
+ return (false);
+
+ frame->sp = fp;
+ frame->fp = ((uintptr_t *)fp)[-2];
+ frame->pc = ((uintptr_t *)fp)[-1] - 4;
+
+ return (true);
+}
diff --git a/sys/loongarch/loongarch/vm_machdep.c b/sys/loongarch/loongarch/vm_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/vm_machdep.c
@@ -0,0 +1,446 @@
+/*-
+ * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/limits.h>
+#include <sys/proc.h>
+#include <sys/eventhandler.h>
+#include <sys/sf_buf.h>
+#include <sys/signal.h>
+#include <sys/unistd.h>
+#include <sys/reboot.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/uma.h>
+#include <vm/uma_int.h>
+
+#include <machine/loongarchreg.h>
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#include <machine/machdep.h>
+#include "opt_acpi.h"
+#ifdef DEV_ACPI
+#include <contrib/dev/acpica/include/acpi.h>
+/* AcpiGbl_SleepTypeA is set by AcpiEnterSleepStatePrep from the \_S5 object. */
+extern UINT8 AcpiGbl_SleepTypeA;
+#endif
+#include <machine/pcb.h>
+#include <machine/frame.h>
+#include <machine/vmparam.h>
+
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/openfirm.h>
+
+#define TP_OFFSET 16 /* sizeof(struct tcb) */
+
+/* GED (Generic Event Device) registers */
+#define GED_REG_SLEEP_CTL 0x00
+#define GED_REG_RESET 0x02
+#define GED_RESET_VALUE 0x42
+#define GED_SLP_EN 0x20
+#define GED_SLP_TYP_S5 0x05
+#define GED_SLP_TYP_POS 0x02
+
+static vm_paddr_t ged_paddr;
+
+/*
+ * Look up the GED physical address from the FDT. Called once during boot.
+ */
+void
+ged_init(void)
+{
+ phandle_t ged_node, reboot_node, child;
+ phandle_t regmap;
+ pcell_t reg[4];
+ int len;
+
+ reboot_node = OF_finddevice("/reboot");
+ if (reboot_node == -1)
+ return;
+
+ len = OF_getencprop(reboot_node, "regmap", &regmap, sizeof(regmap));
+ if (len <= 0)
+ return;
+
+ /* Walk root-level children to find the node with matching phandle */
+ ged_node = -1;
+ for (child = OF_child(OF_peer(0)); child != 0; child = OF_peer(child)) {
+ pcell_t phandle;
+ len = OF_getencprop(child, "phandle", &phandle,
+ sizeof(phandle));
+ if (len > 0 && phandle == regmap) {
+ ged_node = child;
+ break;
+ }
+ }
+ if (ged_node == -1)
+ return;
+
+ len = OF_getencprop(ged_node, "reg", reg, sizeof(reg));
+ if (len <= 0)
+ return;
+
+ ged_paddr = reg[1]; /* phys address from reg tuple */
+}
+
+/*
+ * Write a single byte to a GED register via DMAP.
+ */
+static void
+ged_write_1(vm_offset_t offset, uint8_t val)
+{
+
+ __compiler_membar();
+ *(volatile uint8_t *)PHYS_TO_DMAP(ged_paddr + offset) = val;
+ __compiler_membar();
+}
+
+void
+cpu_reset(void)
+{
+ intr_disable();
+
+#ifdef DEV_ACPI
+ /*
+ * Try the ACPI FADT reset register (firmware-provided address +
+ * value).
+ */
+ if (ACPI_SUCCESS(AcpiReset()))
+ DELAY(1000000);
+
+ /*
+ * Direct write to the FADT reset register via DMW0 (uncached). This
+ * bypasses ACPICA's AcpiHwWrite, which may not reach the hardware on
+ * LoongArch. On the LS7A this is RST_CNT at 0x100d0030.
+ */
+ if (AcpiGbl_FADT.ResetRegister.Address != 0 &&
+ AcpiGbl_FADT.ResetRegister.SpaceId ==
+ ACPI_ADR_SPACE_SYSTEM_MEMORY) {
+ volatile uint8_t *r = (volatile uint8_t *)
+ (DMW0_BASE_ADDRESS | AcpiGbl_FADT.ResetRegister.Address);
+ *r = (uint8_t)AcpiGbl_FADT.ResetValue;
+ DELAY(1000000);
+ }
+#endif
+
+ /* Try GED reset (FDT / QEMU). */
+ if (ged_paddr != 0) {
+ ged_write_1(GED_REG_RESET, GED_RESET_VALUE);
+ DELAY(1000000);
+ }
+
+ /* Fallback: spin forever */
+ while (1)
+ __asm __volatile("idle 0");
+}
+
+void
+cpu_halt(void)
+{
+ intr_disable();
+
+ /* Try GED poweroff (S5 sleep) */
+ if (ged_paddr != 0) {
+ ged_write_1(GED_REG_SLEEP_CTL,
+ GED_SLP_EN | (GED_SLP_TYP_S5 << GED_SLP_TYP_POS));
+ DELAY(1000000);
+ }
+
+ for (;;)
+ __asm __volatile("idle 0");
+ /* NOTREACHED */
+}
+
+#ifdef DEV_ACPI
+/*
+ * Fallback poweroff for ACPI systems where the FADT Pm1aControlBlock is
+ * missing/zero, as observed on the 3A6000, so that ACPICA's sleep path fails
+ * with AE_NOT_EXIST before reaching the hardware. The S5 sleep type (\_S5) was
+ * already evaluated by acpi_EnterSleepStatePrep in acpi_poweroff, so
+ * AcpiGbl_SleepTypeA holds it. Write the LS7A PM1_CNT register directly.
+ */
+#define LA_LS7A_PM1_CNT 0x100d0014UL
+#define LA_PM1_SLP_TYP_SHIFT 10
+#define LA_PM1_SLP_TYP_MASK (0x7u << LA_PM1_SLP_TYP_SHIFT)
+#define LA_PM1_SLP_EN (1u << 13)
+
+static void
+la_poweroff_final(void *arg, int howto)
+{
+ volatile uint32_t *pm1;
+ uint32_t slp_typ, v;
+
+ if ((howto & RB_POWEROFF) == 0)
+ return;
+ slp_typ = AcpiGbl_SleepTypeA;
+ if (slp_typ == 0 || slp_typ > 7)
+ return;
+
+ pm1 = (volatile uint32_t *)(DMW0_BASE_ADDRESS | LA_LS7A_PM1_CNT);
+ v = *pm1;
+ v &= ~(LA_PM1_SLP_TYP_MASK | LA_PM1_SLP_EN);
+ v |= slp_typ << LA_PM1_SLP_TYP_SHIFT;
+ *pm1 = v; /* write #1: SLP_TYP */
+ v |= LA_PM1_SLP_EN;
+ *pm1 = v; /* write #2: SLP_TYP + SLP_EN -> S5 */
+ DELAY(1000000);
+}
+
+static void
+la_poweroff_register(void *arg)
+{
+
+ EVENTHANDLER_REGISTER(shutdown_final, la_poweroff_final, NULL,
+ SHUTDOWN_PRI_LAST + 160);
+}
+SYSINIT(la_poweroff, SI_SUB_DRIVERS, SI_ORDER_ANY, la_poweroff_register, NULL);
+#endif /* DEV_ACPI */
+
+/*
+ * Finish a fork operation, with process p2 nearly set up.
+ * Copy and update the pcb, set up the stack so that the child
+ * ready to run and return to user mode.
+ */
+void
+cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
+{
+ struct pcb *pcb2;
+ struct trapframe *tf;
+
+ if ((flags & RFPROC) == 0)
+ return;
+
+ pcb2 = (struct pcb *)(td2->td_kstack +
+ td2->td_kstack_pages * PAGE_SIZE) - 1;
+
+ td2->td_pcb = pcb2;
+ bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
+
+ tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1);
+ bcopy(td1->td_frame, tf, sizeof(*tf));
+
+ /* Clear syscall error flag */
+ tf->tf_t[0] = 0;
+
+ /* Arguments for child */
+ tf->tf_a[0] = 0;
+ tf->tf_a[1] = 0;
+
+ tf->tf_prmd |= CSR_CRMD_IE; /* Enable interrupts. */
+ tf->tf_prmd |= PLV_USER << CSR_CRMD_PLV_SHIFT; /* User mode. */
+
+ td2->td_frame = tf;
+
+ /* Set the return value registers for fork() */
+ td2->td_pcb->pcb_s[0] = (uintptr_t)fork_return;
+ td2->td_pcb->pcb_s[1] = (uintptr_t)td2;
+ td2->td_pcb->pcb_ra = (uintptr_t)fork_trampoline;
+ td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame;
+
+ /* Setup to release spin count in fork_exit(). */
+ td2->td_md.md_spinlock_count = 1;
+ td2->td_md.md_saved_crmd_ie = CSR_CRMD_IE;
+
+ /* Child starts with no watchpoints. */
+ memset(&td2->td_md.md_watch, 0, sizeof(td2->td_md.md_watch));
+ td2->td_md.md_ss_active = 0;
+}
+
+void
+cpu_set_syscall_retval(struct thread *td, int error)
+{
+ struct trapframe *frame;
+
+ frame = td->td_frame;
+
+ if (__predict_true(error == 0)) {
+ frame->tf_a[0] = td->td_retval[0];
+ frame->tf_a[1] = td->td_retval[1];
+ frame->tf_t[0] = 0; /* syscall succeeded */
+ return;
+ }
+
+ switch (error) {
+ case ERESTART:
+ frame->tf_era -= 4; /* prev instruction */
+ break;
+ case EJUSTRETURN:
+ break;
+ default:
+ frame->tf_a[0] = error;
+ frame->tf_t[0] = 1; /* syscall error */
+ break;
+ }
+}
+
+/*
+ * Initialize machine state, mostly pcb and trap frame for a new
+ * thread, about to return to userspace. Put enough state in the new
+ * thread's PCB to get it to go back to the fork_return(), which
+ * finalizes the thread state and handles peculiarities of the first
+ * return to userspace for the new thread.
+ */
+void
+cpu_copy_thread(struct thread *td, struct thread *td0)
+{
+
+ bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe));
+ bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb));
+
+ td->td_pcb->pcb_s[0] = (uintptr_t)fork_return;
+ td->td_pcb->pcb_s[1] = (uintptr_t)td;
+ td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline;
+ td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
+
+ /* Setup to release spin count in fork_exit(). */
+ td->td_md.md_spinlock_count = 1;
+ td->td_md.md_saved_crmd_ie = CSR_CRMD_IE;
+
+ /* New threads start with no watchpoints. */
+ memset(&td->td_md.md_watch, 0, sizeof(td->td_md.md_watch));
+ td->td_md.md_ss_active = 0;
+}
+
+/*
+ * Set that machine state for performing an upcall that starts
+ * the entry function with the given argument.
+ */
+int
+cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg,
+ stack_t *stack)
+{
+ struct trapframe *tf;
+
+ tf = td->td_frame;
+
+ tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size);
+ tf->tf_era = (register_t)entry;
+ tf->tf_a[0] = (register_t)arg;
+
+ return (0);
+}
+
+int
+cpu_set_user_tls(struct thread *td, void *tls_base, int flags __unused)
+{
+
+ if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS)
+ return (EINVAL);
+
+ /*
+ * The user TLS is set by modifying the trapframe's tp value, which
+ * will be restored when returning to userspace.
+ */
+ td->td_frame->tf_tp = (register_t)tls_base + TP_OFFSET;
+
+ return (0);
+}
+
+void
+cpu_thread_exit(struct thread *td)
+{
+}
+
+void
+cpu_thread_alloc(struct thread *td)
+{
+
+ td->td_pcb = (struct pcb *)(td->td_kstack +
+ td->td_kstack_pages * PAGE_SIZE) - 1;
+ td->td_frame = (struct trapframe *)STACKALIGN(
+ (caddr_t)td->td_pcb - 8 - sizeof(struct trapframe));
+}
+
+void
+cpu_thread_free(struct thread *td)
+{
+}
+
+void
+cpu_thread_clean(struct thread *td)
+{
+}
+
+/*
+ * Intercept the return address from a freshly forked process that has NOT
+ * been scheduled yet.
+ *
+ * This is needed to make kernel threads stay in kernel mode.
+ */
+void
+cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg)
+{
+
+ td->td_pcb->pcb_s[0] = (uintptr_t)func;
+ td->td_pcb->pcb_s[1] = (uintptr_t)arg;
+ td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline;
+ td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
+}
+
+void
+cpu_exit(struct thread *td)
+{
+}
+
+bool
+cpu_exec_vmspace_reuse(struct proc *p __unused, vm_map_t map __unused)
+{
+
+ return (true);
+}
+
+int
+cpu_procctl(struct thread *td __unused, int idtype __unused, id_t id __unused,
+ int com __unused, void *data __unused)
+{
+
+ return (EINVAL);
+}
+
+void
+cpu_update_pcb(struct thread *td)
+{
+}
+
+void
+cpu_thread_new_kstack(struct thread *td)
+{
+ td->td_pcb = (struct pcb *)(td->td_kstack +
+ td->td_kstack_pages * PAGE_SIZE) - 1;
+ td->td_frame = (struct trapframe *)STACKALIGN(
+ (caddr_t)td->td_pcb - 8 - sizeof(struct trapframe));
+}
+
+void
+cpu_sync_core(void)
+{
+ flush_icache();
+}
diff --git a/sys/loongarch/loongarch/watch.c b/sys/loongarch/loongarch/watch.c
new file mode 100644
--- /dev/null
+++ b/sys/loongarch/loongarch/watch.c
@@ -0,0 +1,613 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Xiaoqiang Zhao <zhaoxiaoqiang007@gmail.com>
+ *
+ * LoongArch hardware data-watchpoint facility.
+ *
+ * Per-thread, multi-slot driver using DB0..DB7. Each thread's desired
+ * watchpoint set lives in td->td_md.md_watch (struct la_watch_image) and is
+ * programmed into the per-CPU DB CSRs by la_watch_load(), called from
+ * pmap_activate_sw after the incoming pmap's ASID is installed, so DBnASID
+ * always matches the live ASID.
+ *
+ * Arming is PLV3-only (user mode), so kernel (PLV0) accesses never fire it and
+ * no exception.S enter/exit changes are needed: CRMD.WE is auto-saved to
+ * PRMD.PWE and cleared on trap entry, then restored on ertn. The per-thread
+ * user-mode enable comes from tf_prmd.PWE (set in exec_setregs).
+ *
+ * A data watchpoint is precise: ERA points at the faulting access and the
+ * access has not completed. Re-executing would refire, so the handler
+ * disables each firing slot (fire-once). Re-arming to catch the next access
+ * is the caller's job (ptrace PT_SETDBREGS, or the debug.la_watch sysctl).
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/proc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <machine/atomic.h>
+#include <sys/signalvar.h>
+#include <sys/kdb.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_extern.h>
+
+#include <machine/frame.h>
+#include <machine/kdb.h>
+#include <machine/pcb.h>
+#include <machine/pcpu.h>
+#include <machine/pmap.h>
+#include <machine/reg.h>
+#include <machine/loongarchreg.h>
+#include <machine/watch.h>
+
+#ifdef DDB
+#include <ddb/ddb.h>
+#endif
+
+CTASSERT(LA_WATCH_NSLOTS == LA_WATCH_SLOTS);
+
+static u_int la_watch_slots; /* DB slot count, from MWPC */
+static u_int la_ib_slots; /* IB slot count, from FWPC */
+
+/*
+ * User-facing config for the debug sysctl (written then committed via
+ * armed).
+ */
+static pid_t la_cfg_pid;
+static unsigned long la_cfg_va;
+static u_int la_cfg_size = 8;
+static int la_cfg_mode = 2; /* 1=load 2=store 3=both */
+
+static u_long
+la_size_bits(u_int size)
+{
+
+ switch (size) {
+ case 1: return (CSR_DBC_LEN_1);
+ case 2: return (CSR_DBC_LEN_2);
+ case 4: return (CSR_DBC_LEN_4);
+ default: return (CSR_DBC_LEN_8);
+ }
+}
+
+/*
+ * Program DB slot n (address/mask/control/asid). The LoongArch csrwr
+ * instruction takes an immediate CSR number, so the slot cannot be computed
+ * arithmetically; expand the eight cases.
+ */
+#define DB_SLOT_WRITE(n) \
+ case (n): \
+ csr_write64(addr, LOONGARCH_CSR_DB##n##ADDR); \
+ csr_write64(0, LOONGARCH_CSR_DB##n##MASK); \
+ csr_write64(ctrl, LOONGARCH_CSR_DB##n##CTRL); \
+ csr_write64(asid, LOONGARCH_CSR_DB##n##ASID); \
+ break
+
+static void
+la_db_program(unsigned slot, uint64_t addr, uint64_t ctrl, uint64_t asid)
+{
+
+ switch (slot) {
+ DB_SLOT_WRITE(0);
+ DB_SLOT_WRITE(1);
+ DB_SLOT_WRITE(2);
+ DB_SLOT_WRITE(3);
+ DB_SLOT_WRITE(4);
+ DB_SLOT_WRITE(5);
+ DB_SLOT_WRITE(6);
+ DB_SLOT_WRITE(7);
+ }
+}
+#undef DB_SLOT_WRITE
+
+/*
+ * Program IB slot n (instruction breakpoint / single-step). ctrl == 0
+ * disarms.
+ */
+#define IB_SLOT_WRITE(n) \
+ case (n): \
+ csr_write64(addr, LOONGARCH_CSR_IB##n##ADDR); \
+ csr_write64(0, LOONGARCH_CSR_IB##n##MASK); \
+ csr_write64(ctrl, LOONGARCH_CSR_IB##n##CTRL); \
+ csr_write64(asid, LOONGARCH_CSR_IB##n##ASID); \
+ break
+
+static void
+la_ib_program(unsigned slot, uint64_t addr, uint64_t ctrl, uint64_t asid)
+{
+
+ switch (slot) {
+ IB_SLOT_WRITE(0);
+ IB_SLOT_WRITE(1);
+ IB_SLOT_WRITE(2);
+ IB_SLOT_WRITE(3);
+ IB_SLOT_WRITE(4);
+ IB_SLOT_WRITE(5);
+ IB_SLOT_WRITE(6);
+ IB_SLOT_WRITE(7);
+ }
+}
+#undef IB_SLOT_WRITE
+
+void
+la_watch_init(void)
+{
+ uint64_t crmd;
+ unsigned i;
+
+ la_watch_slots = csr_read64(LOONGARCH_CSR_MWPC) & CSR_MWPC_SLOTS;
+ if (la_watch_slots == 0 || la_watch_slots > LA_WATCH_NSLOTS)
+ la_watch_slots = LA_WATCH_NSLOTS;
+
+ la_ib_slots = csr_read64(LOONGARCH_CSR_FWPC) & CSR_MWPC_SLOTS;
+ if (la_ib_slots == 0 || la_ib_slots > LA_WATCH_NSLOTS)
+ la_ib_slots = LA_WATCH_NSLOTS;
+
+ for (i = 0; i < la_watch_slots; i++)
+ la_db_program(i, 0, 0, 0);
+ for (i = 0; i < la_ib_slots; i++)
+ la_ib_program(i, 0, 0, 0);
+
+ /* Global watchpoint enable (user mode is gated per-thread via PWE). */
+ crmd = csr_read64(LOONGARCH_CSR_CRMD);
+ csr_write64(crmd | CSR_CRMD_WE, LOONGARCH_CSR_CRMD);
+
+ printf("la_watch: %u data watchpoint + %u instruction breakpoint "
+ "slot(s), CRMD.WE enabled\n", la_watch_slots, la_ib_slots);
+}
+
+/*
+ * Called from pmap_activate_sw on every context switch, after the incoming
+ * pmap's ASID is installed. Program all DB slots from the incoming thread's
+ * desired image. Empty slots are disarmed so a previous thread's watchpoints
+ * cannot fire for the new one.
+ */
+void
+la_watch_load(struct thread *td)
+{
+ struct la_watch_image *img = &td->td_md.md_watch;
+ struct pmap *pm;
+ uint64_t asid;
+ unsigned i, cpuid;
+
+ cpuid = PCPU_GET(cpuid);
+ pm = vmspace_pmap(td->td_proc->p_vmspace);
+ asid = pmap_get_asid(pm, cpuid);
+
+ for (i = 0; i < la_watch_slots; i++) {
+ if (img->slot[i].ctrl != 0)
+ la_db_program(i, img->slot[i].addr, img->slot[i].ctrl,
+ asid);
+ else
+ la_db_program(i, 0, 0, 0);
+ }
+
+ /*
+ * Single-step (ptrace): arm IB0 at the stepped instruction and set
+ * FWPS.SKIP so exactly one instruction executes before the
+ * fetch-watchpoint exception fires. When not stepping, keep IB0
+ * disarmed, so a previous step cannot linger.
+ */
+ if (td->td_md.md_ss_active &&
+ td->td_frame->tf_era == td->td_md.md_ss_addr) {
+ csr_write64(td->td_md.md_ss_addr, LOONGARCH_CSR_IB0ADDR);
+ csr_write64(0, LOONGARCH_CSR_IB0MASK);
+ csr_write64(CSR_DBC_PLV3, LOONGARCH_CSR_IB0CTRL);
+ csr_write64(0, LOONGARCH_CSR_IB0ASID);
+ csr_write64(CSR_FWPS_SKIP, LOONGARCH_CSR_FWPS);
+ } else {
+ csr_write64(0, LOONGARCH_CSR_IB0CTRL);
+ }
+}
+
+/*
+ * EXCCODE_WATCH handler. tf_era is the faulting access instruction;
+ * tf_badvaddr is the watched address. Disable each firing slot in hardware
+ * and in the thread image (fire-once), then deliver SIGTRAP/TRAP_TRACE with
+ * si_addr = the watched data address.
+ */
+void
+la_watch_handler(struct thread *td, struct trapframe *frame)
+{
+ uint64_t mwps, fwps;
+ ksiginfo_t ksi;
+ unsigned i;
+ bool fired;
+ void *addr;
+
+ mwps = csr_read64(LOONGARCH_CSR_MWPS);
+ if (mwps != 0)
+ csr_write64(mwps, LOONGARCH_CSR_MWPS); /* W1C */
+
+ addr = NULL;
+ fired = false;
+
+ /* Single-step via IB0 + FWPS.SKIP (instruction fetch watchpoint). */
+ fwps = csr_read64(LOONGARCH_CSR_FWPS);
+ if ((fwps & 0x1ul) != 0) {
+ csr_write64(fwps & 0xfful, LOONGARCH_CSR_FWPS); /* W1C */
+ csr_write64(0, LOONGARCH_CSR_IB0CTRL);
+ td->td_md.md_ss_active = 0;
+ fired = true;
+ addr = (void *)frame->tf_era;
+ }
+
+ /* Data watchpoint(s). */
+ for (i = 0; i < la_watch_slots; i++) {
+ if ((mwps & (1ul << i)) == 0)
+ continue;
+ la_db_program(i, 0, 0, 0);
+ td->td_md.md_watch.slot[i].addr = 0;
+ td->td_md.md_watch.slot[i].ctrl = 0;
+ fired = true;
+ addr = (void *)frame->tf_badvaddr;
+ }
+
+ if (!fired)
+ return;
+
+ printf("WATCHPOINT HIT: pid=%d (%s) badvaddr=%#lx era=%#lx "
+ "MWPS=%#lx FWPS=%#lx\n", td->td_proc->p_pid, td->td_proc->p_comm,
+ (unsigned long)frame->tf_badvaddr, (unsigned long)frame->tf_era,
+ (unsigned long)mwps, (unsigned long)fwps);
+
+ ksiginfo_init_trap(&ksi);
+ ksi.ksi_signo = SIGTRAP;
+ ksi.ksi_code = TRAP_TRACE;
+ ksi.ksi_addr = addr;
+ ksi.ksi_trapno = EXCCODE_WATCH;
+ trapsignal(td, &ksi);
+}
+
+/*
+ * Sysctl debug knob: arm/clear slot 0 on a target process's first thread.
+ * Best-effort (no synchronization with the target running on another CPU);
+ * for precise control use ptrace PT_SETDBREGS.
+ */
+SYSCTL_NODE(_debug, OID_AUTO, la_watch, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+ "LoongArch data watchpoint (debug)");
+
+SYSCTL_UINT(_debug_la_watch, OID_AUTO, slots, CTLFLAG_RD, &la_watch_slots, 0,
+ "number of data watchpoint slots");
+SYSCTL_INT(_debug_la_watch, OID_AUTO, target_pid, CTLFLAG_RW, &la_cfg_pid, 0,
+ "target process pid");
+SYSCTL_ULONG(_debug_la_watch, OID_AUTO, va, CTLFLAG_RW, &la_cfg_va, 0,
+ "watched virtual address");
+SYSCTL_UINT(_debug_la_watch, OID_AUTO, size, CTLFLAG_RW, &la_cfg_size, 8,
+ "watch size (1/2/4/8)");
+SYSCTL_INT(_debug_la_watch, OID_AUTO, mode, CTLFLAG_RW, &la_cfg_mode, 0,
+ "access mode (1=load 2=store 3=both)");
+
+static u_int la_cfg_armed;
+
+static int
+sysctl_la_watch_arm(SYSCTL_HANDLER_ARGS)
+{
+ struct proc *p;
+ struct thread *td2;
+ u_int mode_bits, sizeb;
+ int error, val;
+
+ val = la_cfg_armed;
+ error = sysctl_handle_int(oidp, &val, 0, req);
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+
+ mode_bits = (la_cfg_mode & 1 ? CSR_DBC_LOADEN : 0) |
+ (la_cfg_mode & 2 ? CSR_DBC_STOREEN : 0);
+ if (mode_bits == 0)
+ return (EINVAL);
+ sizeb = la_size_bits(la_cfg_size);
+
+ p = pfind(la_cfg_pid);
+ if (p == NULL) {
+ la_cfg_armed = 0;
+ return (ESRCH);
+ }
+ td2 = FIRST_THREAD_IN_PROC(p);
+ if (td2 != NULL) {
+ if (val) {
+ td2->td_md.md_watch.slot[0].addr = la_cfg_va;
+ td2->td_md.md_watch.slot[0].ctrl =
+ CSR_DBC_PLV3 | mode_bits | sizeb;
+ } else {
+ td2->td_md.md_watch.slot[0].addr = 0;
+ td2->td_md.md_watch.slot[0].ctrl = 0;
+ }
+ }
+ PROC_UNLOCK(p);
+ la_cfg_armed = (val != 0);
+ return (0);
+}
+SYSCTL_PROC(_debug_la_watch, OID_AUTO, armed, CTLTYPE_INT | CTLFLAG_RW |
+ CTLFLAG_MPSAFE, NULL, 0, sysctl_la_watch_arm, "I",
+ "arm/disarm slot 0 on target_pid");
+
+/*
+ * DDB kernel watchpoints (hwatch/dhwatch). Kernel (PLV0) watchpoints armed
+ * by the debugger on the current CPU; ASID is don't-care (0). Because trap
+ * entry clears CRMD.WE, tf_prmd.PWE is toggled so WE re-enables in kernel mode
+ * on ertn after `continue'.
+ *
+ * These share the DB CSRs with per-thread user watchpoints; a kernel
+ * watchpoint survives until the next context switch on this CPU (la_watch_load
+ * reprograms the slots from the thread image) -- sufficient for "set,
+ * continue, catch the write" debugging.
+ */
+#ifdef DDB
+struct la_kern_watch_entry {
+ uint64_t addr;
+ vm_size_t size;
+ uint32_t ctrl; /* DBnCTRL bits; 0 = unused */
+};
+static struct la_kern_watch_entry la_kern_watch[LA_WATCH_NSLOTS];
+static unsigned la_kern_narmed;
+
+/* DDB instruction breakpoints (hbreak); IB0 is reserved for single-step. */
+static struct la_kern_watch_entry la_kern_hbreak[LA_WATCH_NSLOTS];
+static unsigned la_kern_hbreak_count;
+
+static u_int
+la_access_bits(int access)
+{
+
+ switch (access) {
+ case KDB_DBG_ACCESS_R: return (CSR_DBC_LOADEN);
+ case KDB_DBG_ACCESS_W: return (CSR_DBC_STOREEN);
+ case KDB_DBG_ACCESS_RW: return (CSR_DBC_LOADEN |
+ CSR_DBC_STOREEN);
+ default: return (0);
+ }
+}
+
+int
+kdb_cpu_set_watchpoint(vm_offset_t addr, vm_size_t size, int access)
+{
+ uint32_t ctrl, mode;
+ unsigned i;
+
+ mode = la_access_bits(access);
+ if (mode == 0)
+ return (EINVAL);
+
+ for (i = 0; i < LA_WATCH_NSLOTS; i++)
+ if (la_kern_watch[i].ctrl == 0)
+ break;
+ if (i == LA_WATCH_NSLOTS)
+ return (EBUSY);
+
+ ctrl = CSR_DBC_PLV0 | mode | la_size_bits(size);
+ la_kern_watch[i].addr = addr;
+ la_kern_watch[i].size = size;
+ la_kern_watch[i].ctrl = ctrl;
+ la_db_program(i, addr, ctrl, 0); /* ASID 0 = don't-care */
+
+ if (la_kern_narmed++ == 0 && kdb_frame != NULL)
+ kdb_frame->tf_prmd |= CSR_PRMD_PWE;
+ return (0);
+}
+
+int
+kdb_cpu_clr_watchpoint(vm_offset_t addr, vm_size_t size)
+{
+ unsigned i;
+
+ for (i = 0; i < LA_WATCH_NSLOTS; i++) {
+ if (la_kern_watch[i].ctrl == 0 ||
+ la_kern_watch[i].addr != addr ||
+ la_kern_watch[i].size != size)
+ continue;
+ la_kern_watch[i].ctrl = 0;
+ la_db_program(i, 0, 0, 0);
+ if (--la_kern_narmed == 0 && kdb_frame != NULL)
+ kdb_frame->tf_prmd &= ~CSR_PRMD_PWE;
+ return (0);
+ }
+ return (EINVAL);
+}
+
+/*
+ * DDB instruction breakpoint (hbreak). Uses IB slots >= 1 (IB0 is reserved
+ * for single-step); armed PLV0 so kernel-mode execution traps. On a hit DDB
+ * steps over via single-step then re-arms (db_restart_at_pc).
+ */
+int
+kdb_cpu_set_breakpoint(vm_offset_t addr)
+{
+ unsigned i;
+
+ for (i = 1; i < la_ib_slots; i++)
+ if (la_kern_hbreak[i].ctrl == 0)
+ break;
+ if (i == la_ib_slots)
+ return (EBUSY);
+
+ la_kern_hbreak[i].addr = addr;
+ la_kern_hbreak[i].ctrl = CSR_DBC_PLV0;
+ la_ib_program(i, addr, CSR_DBC_PLV0, 0);
+ la_kern_hbreak_count++;
+
+ if (kdb_frame != NULL)
+ kdb_frame->tf_prmd |= CSR_PRMD_PWE;
+ return (0);
+}
+
+int
+kdb_cpu_clr_breakpoint(vm_offset_t addr)
+{
+ unsigned i;
+
+ for (i = 1; i < la_ib_slots; i++) {
+ if (la_kern_hbreak[i].ctrl == 0 ||
+ la_kern_hbreak[i].addr != addr)
+ continue;
+ la_kern_hbreak[i].ctrl = 0;
+ la_ib_program(i, 0, 0, 0);
+ if (la_kern_hbreak_count > 0)
+ la_kern_hbreak_count--;
+ return (0);
+ }
+ return (EINVAL);
+}
+
+/*
+ * Fire-once: disarm the firing kernel slot(s) before dropping into DDB so
+ * `continue' does not immediately refire on the same instruction (there is no
+ * per-instruction RF for data watchpoints).
+ */
+void
+la_watch_kern_handler(struct trapframe *frame)
+{
+ uint64_t mwps, fwps;
+ unsigned i;
+
+ mwps = csr_read64(LOONGARCH_CSR_MWPS);
+ if (mwps != 0)
+ csr_write64(mwps, LOONGARCH_CSR_MWPS); /* W1C */
+
+ /*
+ * Fetch watchpoints (instruction breakpoints / single-step). Clear the
+ * fired slot status; only the single-step slot IB0 is one-shot -- the
+ * instruction-breakpoint slots are managed by DDB, which clears and
+ * re-arms them around the step-over (db_restart_at_pc).
+ */
+ fwps = csr_read64(LOONGARCH_CSR_FWPS);
+ if (fwps != 0) {
+ csr_write64(fwps & 0xfful, LOONGARCH_CSR_FWPS); /* W1C */
+ if ((fwps & 0x1ul) != 0)
+ csr_write64(0, LOONGARCH_CSR_IB0CTRL);
+ }
+
+ for (i = 0; i < la_watch_slots; i++) {
+ if ((mwps & (1ul << i)) == 0)
+ continue;
+ la_db_program(i, 0, 0, 0);
+ if (la_kern_watch[i].ctrl != 0) {
+ la_kern_watch[i].ctrl = 0;
+ if (la_kern_narmed > 0)
+ la_kern_narmed--;
+ }
+ }
+ if (kdb_frame != NULL) {
+ if (la_kern_narmed > 0 || la_kern_hbreak_count > 0)
+ kdb_frame->tf_prmd |= CSR_PRMD_PWE;
+ else
+ kdb_frame->tf_prmd &= ~CSR_PRMD_PWE;
+ }
+
+ printf("KWATCH HIT: badvaddr=%#lx era=%#lx MWPS=%#lx FWPS=%#lx\n",
+ (unsigned long)frame->tf_badvaddr, (unsigned long)frame->tf_era,
+ (unsigned long)mwps, (unsigned long)fwps);
+}
+
+/*
+ * DDB single-step: arm IB0 (PLV0) at the current PC and set FWPS.SKIP so one
+ * kernel instruction executes before EXCCODE_WATCH re-enters the debugger.
+ * tf_prmd.PWE is set so CRMD.WE re-enables in kernel mode on ertn.
+ */
+void
+kdb_cpu_set_singlestep(void)
+{
+
+ if (kdb_frame == NULL)
+ return;
+ csr_write64(kdb_frame->tf_era, LOONGARCH_CSR_IB0ADDR);
+ csr_write64(0, LOONGARCH_CSR_IB0MASK);
+ csr_write64(CSR_DBC_PLV0, LOONGARCH_CSR_IB0CTRL);
+ csr_write64(0, LOONGARCH_CSR_IB0ASID);
+ csr_write64(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS);
+ kdb_frame->tf_prmd |= CSR_PRMD_PWE;
+}
+
+void
+kdb_cpu_clear_singlestep(void)
+{
+
+ csr_write64(0, LOONGARCH_CSR_IB0CTRL);
+}
+
+void
+db_md_list_watchpoints(void)
+{
+ unsigned i;
+
+ for (i = 0; i < LA_WATCH_NSLOTS; i++) {
+ if (la_kern_watch[i].ctrl == 0)
+ continue;
+ db_printf(" slot %u: addr=%#lx size=%lu ctrl=%#x (PLV0)\n",
+ i, (unsigned long)la_kern_watch[i].addr,
+ (unsigned long)la_kern_watch[i].size,
+ la_kern_watch[i].ctrl);
+ }
+}
+
+#ifdef HAS_HW_BREAKPOINT
+void
+db_md_list_breakpoints(void)
+{
+ unsigned i;
+
+ for (i = 1; i < la_ib_slots; i++) {
+ if (la_kern_hbreak[i].ctrl == 0)
+ continue;
+ db_printf(" slot %u: addr=%#lx ctrl=%#x (PLV0 hbreak)\n",
+ i, (unsigned long)la_kern_hbreak[i].addr,
+ la_kern_hbreak[i].ctrl);
+ }
+}
+#endif /* HAS_HW_BREAKPOINT */
+
+#else /* !DDB */
+
+int
+kdb_cpu_set_watchpoint(vm_offset_t addr, vm_size_t size, int access)
+{
+
+ return (ENXIO);
+}
+
+int
+kdb_cpu_clr_watchpoint(vm_offset_t addr, vm_size_t size)
+{
+
+ return (0);
+}
+
+void
+la_watch_kern_handler(struct trapframe *frame)
+{
+}
+
+void
+kdb_cpu_set_singlestep(void)
+{
+}
+
+void
+kdb_cpu_clear_singlestep(void)
+{
+}
+
+int
+kdb_cpu_set_breakpoint(vm_offset_t addr)
+{
+
+ return (ENXIO);
+}
+
+int
+kdb_cpu_clr_breakpoint(vm_offset_t addr)
+{
+
+ return (0);
+}
+
+#endif /* DDB */
diff --git a/sys/sys/kerneldump.h b/sys/sys/kerneldump.h
--- a/sys/sys/kerneldump.h
+++ b/sys/sys/kerneldump.h
@@ -87,6 +87,7 @@
#define KERNELDUMP_AMD64_VERSION 2
#define KERNELDUMP_ARM_VERSION 1
#define KERNELDUMP_I386_VERSION 2
+#define KERNELDUMP_LOONGARCH_VERSION 1
#define KERNELDUMP_MIPS_VERSION 1
#define KERNELDUMP_POWERPC_VERSION 1
#define KERNELDUMP_RISCV_VERSION 1
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -847,6 +847,9 @@
("kva_import: Size %jd is not a multiple of %d",
(intmax_t)size, (int)KVA_QUANTUM));
addr = vm_map_min(kernel_map);
+#ifdef KMOD_MAX_ADDRESS
+ addr = MAX(addr, KMOD_MAX_ADDRESS);
+#endif
result = vm_map_find(kernel_map, NULL, 0, &addr, size, 0,
VMFS_SUPER_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
if (result != KERN_SUCCESS) {

File Metadata

Mime Type
text/plain
Expires
Sun, Oct 18, 5:49 AM (5 h, 1 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
40911836
Default Alt Text
D59890.id.diff (813 KB)

Event Timeline