diff --git a/.gitattributes b/.gitattributes index 11fec084166f..ab5a1eee005d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,7 +1,8 @@ *.c diff=cpp *.h diff=cpp *.cpp diff=cpp *.hpp diff=cpp *.py diff=python . svn-properties=svn:keywords=tools/build/options/WITHOUT_LOADER_ZFS .clang-format svn-properties=svn:keywords=FreeBSD=%H +contrib/one-true-awk/bugs-fixed/unicode-null-match.bad binary diff --git a/UPDATING b/UPDATING index 763f8f29a4f0..17117a193394 100644 --- a/UPDATING +++ b/UPDATING @@ -1,2101 +1,2106 @@ Updating Information for users of FreeBSD-CURRENT. This file is maintained and copyrighted by M. Warner Losh . See end of file for further details. For commonly done items, please see the COMMON ITEMS: section later in the file. These instructions assume that you basically know what you are doing. If not, then please consult the FreeBSD handbook: https://docs.freebsd.org/en/books/handbook/cutting-edge/#makeworld Items affecting the ports and packages system can be found in /usr/ports/UPDATING. Please read that file before updating system packages and/or ports. NOTE TO PEOPLE WHO THINK THAT FreeBSD 15.x IS SLOW: FreeBSD 15.x has many debugging features turned on, in both the kernel and userland. These features attempt to detect incorrect use of system primitives, and encourage loud failure through extra sanity checking and fail stop semantics. They also substantially impact system performance. If you want to do performance measurement, benchmarking, and optimization, you'll want to turn them off. This includes various WITNESS- related kernel options, INVARIANTS, malloc debugging flags in userland, and various verbose features in the kernel. Many developers choose to disable these features on build machines to maximize performance. (To completely disable malloc debugging, define WITH_MALLOC_PRODUCTION in /etc/src.conf and rebuild world, or to merely disable the most expensive debugging functionality at runtime, run "ln -s 'abort:false,junk:false' /etc/malloc.conf".) +20231114: + One True Awk updated to the Second Edition as of 20231102 (254b979f32df) + Notable features include UTF-8 support and --csv to support comma + separated data. + 20231113: The WITHOUT_LLD_IS_LD option has been removed. When LLD is enabled it is always installed as /usr/bin/ld. 20231027: Forward compatibility (running the new code on old kernels) for the "ino64" project have been removed. The need for it has passed long ago. 20231018: Commit 57ce37f9dcd0 changed the internal KAPI between the nfscommon and nfscl modules. Both must be rebuilt from sources. 20231010: dialog(1) has been replaced in base by bsddialog(1), while most of the time replacing a dialog(1) call by a bsddialog(1) call works out of the box, bsddialog(1) is not considered as a drop-in replacement for dialog(1). If you do depend on dialog(1) functionality, please install cdialog from ports: pkg install cdialog 20230927: The EARLY_AP_STARTUP kernel option is mandatory on x86. The option has been added to DEFAULTS, so it should automatically be included in custom kernel configurations without any additional change. 20230922: A new loader tunable net.pf.default_to_drop allows pf(4)’s default behaviour to be changed from pass to drop. Previously this required recompiling the kernel with the option PF_DEFAULT_TO_DROP. 20230914: Enable splitting out pkgbase manpages into separate packages by default. To disable this, set WITHOUT_MANSPLITPKG=yes in src.conf. 20230911: Move standard include files to the clibs-dev package and move clang internal libraries and headers to clang and clang-dev. Upgrading systems installed using pkgbase past this change involves extra steps to allow for these file moves: pkg upgrade -y FreeBSD-utilities pkg upgrade -y FreeBSD-utilities-dev pkg upgrade -y 20230909: Enable vnet sysctl variables to be loader tunable. SYSCTLs which belongs to VNETs can be initialized during early boot or module loading if they are marked with CTLFLAG_TUN and there are corresponding kernel environment variables. 20230901: The WITH_INIT_ALL_PATTERN and WITH_INIT_ALL_ZERO build options have been replaced by INIT_ALL=pattern and INIT_ALL=zero respectively. 20230824: FreeBSD 15.0-CURRENT. 20230817: Serial communication (in boot loaders, kernel, and userland) has been changed to default to 115200 bps, in line with common industry practice and typcial firmware serial console redirection configuration. Note that the early x86 BIOS bootloader (i.e., boot0sio) does not support rates above 9600 bps and is not changed. boot0sio users may set BOOT_COMCONSOLE_SPEED=9600 to use 9600 for all of the boot components, or use the standard boot0 and have the boot2 stage start with the serial port at 115200. 20230807: Following the general removal of MIPS support, the ath(4) AHB bus- frontend has been removed, too, and building of the PCI support is integrated with the ath(4) main module again. As a result, there's no longer a need for if_ath_pci_load="YES" in /boot/loader.conf or "device ath_pci" in the kernel configuration. 20230803: MAXCPU has been increased to 1024 in the amd64 GENERIC kernel config. Out-of-tree kernel modules will need to be rebuilt. 20230724: CAM has been mechanically updated s/u_int(64|32|16|8)_t/uint\1_t/g to move to the standard uintXX_t types from the old, traditional BSD u_intXX_t types. This should be a NOP, but may cause problems for out of tree changes. The SIMs were not updated since most of the old u_intXX_t uses weren't due to CAM interfaces. 20230713: stable/14 branch created. 20230629: The heuristic for detecting old chromebooks with an EC bug that requires atkbdc driver workarounds has changed. There should be no functional change, but if your old chromebook's keyboard stops working, please file a PR and assign it to imp. 20230623: OpenSSL has been updated to version 3.0, including changes throughout the base system. It is important to rebuild third-party software after upgrading. 20230619: To enable pf rdr rules for connections initiated from the host, pf filter rules can be optionally enabled for packets delivered locally. This can change the behavior of rules which match packets delivered to lo0. To enable this feature: sysctl net.pf.filter_local=1 service pf restart When enabled, its best to ensure that packets delivered locally are not filtered, e.g. by adding a 'skip on lo' rule. 20230613: Improvements to libtacplus(8) mean that tacplus.conf(5) now follows POSIX shell syntax rules. This may cause TACACS+ authentication to fail if the shared secret contains a single quote, double quote, or backslash character which isn't already properly quoted or escaped. 20230612: Belatedly switch the default nvme block device on x86 from nvd to nda. nda created nvd compatibility links by default, so this should be a nop. If this causes problems for your application, set hw.nvme.use_nvd=1 in your loader.conf or add `options NVME_USE_NVD=1` to your kernel config. To disable the nvd compatibility aliases, add kern.cam.nda.nvd_compat=0 to loader.conf. The default has been nda on all non-x86 platforms for some time now. If you need to fall back, please email imp@freebsd.org about why. Encrypted swap partitions need to be changed from nvd to nda if you migrate, or you need to use the above to switch back to nvd. 20230422: Remove portsnap(8). Users are encouraged to obtain the ports tree using git instead. 20230420: Add jobs.mk to save typing. Enables -j${JOB_MAX} and logging eg. make buildworld-jobs runs make -j${JOB_MAX} buildworld > ../buildworld.log 2>&1 where JOB_MAX is derrived from ncpus in local.sys.mk if not set in env. 20230316: Video related devices for some arm devices have been renamed. If you have a custom kernel config and want to use hdmi output on IMX6 board you need to add "device dwc_hdmi" "device imx6_hdmi" and "device imx6_ipu" to it. If you have a custom kernel config and want to use hdmi output on TI AM335X board you need to add "device tda19988" to it. If you add "device hdmi" in it you need to remove it as it doesn't exist anymore. 20230221: Introduce new kernel options KBD_DELAY1 and KBD_DELAY2. See atkbdc(4) for details. 20230206: sshd now defaults to having X11Forwarding disabled, following upstream. Administrators who wish to enable X11Forwarding should add `X11Forwarding yes` to /etc/ssh/sshd_config. 20230204: Since commit 75d41cb6967 Huawei 3G/4G LTE Mobile Devices do not default to ECM, but NCM mode and need u3g and ucom modules loaded. See cdce(4). 20230130: As of commit 7c40e2d5f685, the dependency on netlink(4) has been added to the linux_common(4) module. Users relying on linux_common may need to complile netlink(4) module if it is not present in their kernel. 20230126: The WITHOUT_CXX option has been removed. C++ components in the base system are now built unconditionally. 20230113: LinuxKPI pci.h changes may require out-of-tree drivers to be recompiled. Bump _FreeBSD_version to 1400078 to be able to detect this change. 20221212: llvm-objump is now always installed as objdump. Previously there was no /usr/bin/objdump unless the WITH_LLVM_BINUTILS knob was used. Some LLVM objdump options have a different output format compared to GNU objdump; readelf is available for inspecting ELF files, and GNU objdump is available from the devel/binutils port or package. 20221205: dma(8) has replaced sendmail(8) as the default mta. For people willing to reenable sendmail(8): $ cp /usr/share/examples/sendmail/mailer.conf /etc/mail/mailer.conf and add sendmail_enable="YES" to rc.conf. 20221204: hw.bus.disable_failed_devices has changed from 'false' to 'true' by default. Now if newbus succeeds in probing a device, but fails to attach the device, we'll disable the device. In the past, we'd keep retrying the device on each new driver loaded. To get that behavior now, one needs to use devctl to re-enable the device, and reprobe it (or set the sysctl/tunable hw.bus.disable_failed_devices=false). NOTE: This was reverted 20221205 due to unexpected compatibility issues 20221122: pf no longer accepts 'scrub fragment crop' or 'scrub fragment drop-ovl'. These configurations are no longer automatically reinterpreted as 'scrub fragment reassemble'. 20221121: The WITHOUT_CLANG_IS_CC option has been removed. When Clang is enabled it is always installed as /usr/bin/cc (and c++, cpp). 20221026: Some programs have been moved into separate packages. It is recommended for pkgbase users to do: pkg install FreeBSD-dhclient FreeBSD-geom FreeBSD-resolvconf \ FreeBSD-devd FreeBSD-devmatch after upgrading to restore all the component that were previously installed. 20221002: OPIE has been removed from the base system. If needed, it can be installed from ports (security/opie) or packages (opie). Otherwise, make sure that your PAM policies do not reference pam_opie or pam_opieaccess. 20220610: LinuxKPI pm.h changes require an update to the latest drm-kmod version before re-compiling to avoid errors. 20211230: The macros provided for the manipulation of CPU sets (e.g. CPU_AND) have been modified to take 2 source arguments instead of only 1. Externally maintained sources that use these macros will have to be adapted. The FreeBSD version has been bumped to 1400046 to reflect this change. 20211214: A number of the kernel include files are able to be included by themselves. A test has been added to buildworld to enforce this. 20211209: Remove mips as a recognized target. This starts the decommissioning of mips support in FreeBSD. mips related items will be removed wholesale in the coming days and weeks. This broke the NO_CLEAN build for some people. Either do a clean build or touch lib/clang/include/llvm/Config/Targets.def lib/clang/include/llvm/Config/AsmParsers.def lib/clang/include/llvm/Config/Disassemblers.def lib/clang/include/llvm/Config/AsmPrinters.def before the build to force everything to rebuild that needs to. 20211202: Unbound support for RFC8375: The special-use domain 'home.arpa' is by default blocked. To unblock it use a local-zone nodefault statement in unbound.conf: local-zone: "home.arpa." nodefault Or use another type of local-zone to override with your choice. The reason for this is discussed in Section 6.1 of RFC8375: Because 'home.arpa.' is not globally scoped and cannot be secured using DNSSEC based on the root domain's trust anchor, there is no way to tell, using a standard DNS query, in which homenet scope an answer belongs. Consequently, users may experience surprising results with such names when roaming to different homenets. 20211110: Commit b8d60729deef changed the TCP congestion control framework so that any of the included congestion control modules could be the single module built into the kernel. Previously newreno was automatically built in through direct reference. As of this commit you are required to declare at least one congestion control module (e.g. 'options CC_NEWRENO') and to also declare a default using the CC_DEFAULT option (e.g. options CC_DEFAULT="newreno\"). The GENERIC configuration includes CC_NEWRENO and defines newreno as the default. If no congestion control option is built into the kernel and you are including networking, the kernel compile will fail. Also if no default is declared the kernel compile will fail. 20211118: Mips has been removed from universe builds. It will be removed from the tree shortly. 20211106: Commit f0c9847a6c47 changed the arguments for VOP_ALLOCATE. The NFS modules must be rebuilt from sources and any out of tree file systems that implement their own VOP_ALLOCATE may need to be modified. 20211022: The synchronous PPP kernel driver sppp(4) has been removed. The cp(4) and ce(4) drivers are now always compiled with netgraph(4) support, formerly enabled by NETGRAPH_CRONYX option. 20211020: sh(1) is now the default shell for the root user. To force root to use the csh shell, please run the following command as root: # chsh -s csh 20211004: Ncurses distribution has been split between libtinfow and libncurses with libncurses.so becoming a linker (ld) script to seamlessly link to libtinfow as needed. Bump _FreeBSD_version to 1400035 to reflect this change. 20210923: As of commit 8160a0f62be6, the dummynet module no longer depends on the ipfw module. Dummynet can now be used by pf as well as ipfw. As such users who relied on this dependency may need to include ipfw in the list of modules to load on their systems. 20210922: As of commit 903873ce1560, the mixer(8) utility has got a slightly new syntax. Please refer to the mixer(8) manual page for more information. The old mixer utility can be installed from ports: audio/freebsd-13-mixer 20210911: As of commit 55089ef4f8bb, the global variable nfs_maxcopyrange has been deleted from the nfscommon.ko. As such, nfsd.ko must be built from up to date sources to avoid an undefined reference when being loaded. 20210817: As of commit 62ca9fc1ad56 OpenSSL no longer enables kernel TLS by default. Users can enable kernel TLS via the "KTLS" SSL option. This can be enabled globally by using a custom OpenSSL config file via OPENSSL_CONF or via an application-specific configuration option for applications which permit setting SSL options via SSL_CONF_cmd(3). 20210811: Commit 3ad1e1c1ce20 changed the internal KAPI between the NFS modules. Therefore, all need to be rebuilt from sources. 20210730: Commit b69019c14cd8 removes pf's DIOCGETSTATESNV ioctl. As of be70c7a50d32 it is no longer used by userspace, but it does mean users may not be able to enumerate pf states if they update the kernel past b69019c14cd8 without first updating userspace past be70c7a50d32. 20210729: As of commit 01ad0c007964 if_bridge member interfaces can no longer change their MTU. Changing the MTU of the bridge itself will change the MTU on all member interfaces instead. 20210716: Commit ee29e6f31111 changed the internal KAPI between the nfscommon and nfsd modules. Therefore, both need to be rebuilt from sources. Bump __FreeBSD_version to 1400026 for this KAPI change. 20210715: The 20210707 awk update brought in a change in behavior. This has been corrected as of d4d252c49976. Between these dates, if you installed a new awk binary, you may not be able to build a new kernel because the change in behavior affected the genoffset script used to build the kernel. If you did update, the fix is to update your sources past the above hash and do % cd usr.bin/awk % make clean all % sudo -E make install to enable building kernels again. 20210708: Commit 1e0a518d6548 changed the internal KAPI between the NFS modules. They all need to be rebuilt from sources. I did not bump __FreeBSD_version, since it was bumped recently. 20210707: awk has been updated to the latest one-true-awk version 20210215. This contains a number of minor bug fixes. 20210624: The NFSv4 client now uses the highest minor version of NFSv4 supported by the NFSv4 server by default instead of minor version 0, for NFSv4 mounts. The "minorversion" mount option may be used to override this default. 20210618: Bump __FreeBSD_version to 1400024 for LinuxKPI changes. Most notably netdev.h can change now as the (last) dependencies (mlx4/ofed) are now using struct ifnet directly, but also for PCI additions and others. 20210618: The directory "blacklisted" under /usr/share/certs/ has been renamed to "untrusted". 20210611: svnlite has been removed from base. Should you need svn for any reason please install the svn package or port. 20210611: Commit e1a907a25cfa changed the internal KAPI between the krpc and nfsserver. As such, both modules must be rebuilt from sources. Bump __FreeBSD_version to 1400022. 20210610: The an(4) driver has been removed from FreeBSD. 20210608: The vendor/openzfs branch was renamed to vendor/openzfs/legacy to start tracking OpenZFS upstream more closely. Please see https://lists.freebsd.org/archives/freebsd-current/2021-June/000153.html for details on how to correct any errors that might result. The short version is that you need to remove the old branch locally: git update-ref -d refs/remotes/freebsd/vendor/openzfs (assuming your upstream origin is named 'freebsd'). 20210525: Commits 17accc08ae15 and de102f870501 add new files to LinuxKPI which break drm-kmod. In addition various other additions where committed. Bump __FreeBSD_version to 1400015 to be able to detect this. 20210513: Commit ca179c4d74f2 changed the package in which the OpenSSL libraries and utilities are packaged. It is recommended for pkgbase user to do: pkg install -f FreeBSD-openssl before pkg upgrade otherwise some dependencies might not be met and pkg will stop working as libssl will not be present anymore on the system. 20210426: Commit 875977314881 changed the internal KAPI between the nfsd and nfscommon modules. As such these modules need to be rebuilt from sources. Without this patch in your NFSv4.1/4.2 server, enabling delegations by setting vfs.nfsd.issue_delegations non-zero is not recommended. 20210411: Commit 7763814fc9c2 changed the internal KAPI between the krpc and NFS. As such, the krpc, nfscommon and nfscl modules must all be rebuilt from sources. Without this patch, NFSv4.1/4.2 mounts should not be done with the nfscbd(8) daemon running, to avoid needing a working back channel for server->client RPCs. 20210330: Commit 01ae8969a9ee fixed the NFSv4.1/4.2 server so that it handles binding of the back channel as required by RFC5661. Until this patch is in your server, avoid use of the "nconnects" mount option for Linux NFSv4.1/4.2 mounts. 20210225: For 64-bit architectures the base system is now built with Position Independent Executable (PIE) support enabled by default. It may be disabled using the WITHOUT_PIE knob. A clean build is required. 20210128: Various LinuxKPI functionality was added which conflicts with DRM. Please update your drm-kmod port to after the __FreeBSD_version 1400003 update. 20210121: stable/13 branch created. 20210108: PC Card attachments for all devices have been removed. In the case of wi and cmx, the entire drivers were removed because they were only PC Card devices. FreeBSD_version 1300134 should be used for this since it was bumped so recently. 20210107: Transport-independent parts of HID support have been split off the USB code in to separate subsystem. Kernel configs which include one of ums, ukbd, uhid, atp, wsp, wmt, uaudio, ugold or ucycom drivers should be updated with adding of "device hid" line. 20210105: ncurses installation has been modified to only keep the widechar enabled version. Incremental build is broken for that change, so it requires a clean build. 20201223: The FreeBSD project has migrated from Subversion to Git. Temporary instructions can be found at https://github.com/bsdimp/freebsd-git-docs/blob/main/src-cvt.md and other documents in that repo. 20201216: The services database has been updated to cover more of the basic services expected in a modern system. The database is big enough that it will cause issues in mergemaster in Releases previous to 12.2 and 11.3, or in very old current systems from before r358154. 20201215: Obsolete in-tree GDB 6.1.1 has been removed. GDB (including kgdb) may be installed from ports or packages. 20201124: ping6 has been merged into ping. It can now be called as "ping -6". See ping(8) for details. 20201108: Default value of net.add_addr_allfibs has been changed to 0. If you have multi-fib configuration and rely on existence of all interface routes in every fib, you need to set the above sysctl to 1. 20201030: The internal pre-processor in the calendar(1) program has been extended to support more C pre-processor commands (e.g. #ifdef, #else, and #undef) and to detect unbalanced conditional statements. Error messages have been extended to include the filename and line number if processing stops to help fixing malformed data files. 20201026: All the data files for the calendar(1) program, except calendar.freebsd, have been moved to the deskutils/calendar-data port, much like the jewish calendar entries were moved to deskutils/hebcal years ago. After make delete-old-files, you need to install it to retain full functionality. calendar(1) will issue a reminder for files it can't find. 20200923: LINT files are no longer generated. We now include the relevant NOTES files. Note: This may cause conflicts with updating in some cases. find sys -name LINT\* -delete is suggested across this commit to remove the generated LINT files. If you have tried to update with generated files there, the svn command you want to un-auger the tree is cd sys/amd64/conf svn revert -R . and then do the above find from the top level. Substitute 'amd64' above with where the error message indicates a conflict. 20200824: OpenZFS support has been integrated. Do not upgrade root pools until the loader is updated to support zstd. Furthermore, we caution against 'zpool upgrade' for the next few weeks. The change should be transparent unless you want to use new features. Not all "NO_CLEAN" build scenarios work across these changes. Many scenarios have been tested and fixed, but rebuilding kernels without rebuilding world may fail. The ZFS cache file has moved from /boot to /etc to match the OpenZFS upstream default. A fallback to /boot has been added for mountroot. Pool auto import behavior at boot has been moved from the kernel module to an explicit "zpool import -a" in one of the rc scripts enabled by zfs_enable=YES. This means your non-root zpools won't auto import until you upgrade your /etc/rc.d files. 20200824: The resume code now notifies devd with the 'kernel' system rather than the old 'kern' subsystem to be consistent with other use. The old notification will be created as well, but will be removed prior to FreeBSD 14.0. 20200821: r362275 changed the internal API between the kernel RPC and the NFS modules. As such, all the modules must be recompiled from sources. 20200817: r364330 modified the internal API used between the NFS modules. As such, all the NFS modules must be re-compiled from sources. 20200816: Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have been upgraded to 11.0.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20200810: r364092 modified the internal ABI used between the kernel NFS modules. As such, all of these modules need to be rebuilt from sources, so a version bump was done. 20200807: Makefile.inc has been updated to work around the issue documented in 20200729. It was a case where the optimization of using symbolic links to point to binaries created a situation where we'd run new binaries with old libraries starting midway through the installworld process. 20200729: r363679 has redefined some undefined behavior in regcomp(3); notably, extraneous escapes of most ordinary characters will no longer be accepted. An exp-run has identified all of the problems with this in ports, but other non-ports software may need extra escapes removed to continue to function. Because of this change, installworld may encounter the following error from rtld: Undefined symbol "regcomp@FBSD_1.6" -- It is imperative that you do not halt installworld. Instead, let it run to completion (whether successful or not) and run installworld once more. 20200627: A new implementation of bc and dc has been imported in r362681. This implementation corrects non-conformant behavior of the previous bc and adds GNU bc compatible options. It offers a number of extensions, is much faster on large values, and has support for message catalogs (a number of languages are already supported, contributions of further languages welcome). The option WITHOUT_GH_BC can be used to build the world with the previous versions of bc and dc. 20200625: r362639 changed the internal API used between the NFS kernel modules. As such, they all need to be rebuilt from sources. 20200613: r362158 changed the arguments for VFS_CHECKEXP(). As such, any out of tree file systems need to be modified and rebuilt. Also, any file systems that are modules must be rebuilt. 20200604: read(2) of a directory fd is now rejected by default. root may re-enable it for system root only on non-ZFS filesystems with the security.bsd.allow_read_dir sysctl(8) MIB if security.bsd.suser_enabled=1. It may be advised to setup aliases for grep to default to `-d skip` if commonly non-recursively grepping a list that includes directories and the potential for the resulting stderr output is not tolerable. Example aliases are now installed, commented out, in /root/.cshrc and /root/.shrc. 20200523: Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have been upgraded to 10.0.1. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20200512: Support for obsolete compilers has been removed from the build system. Clang 6 and GCC 6.4 are the minimum supported versions. 20200424: closefrom(2) has been moved under COMPAT12, and replaced in libc with a stub that calls close_range(2). If using a custom kernel configuration, you may want to ensure that the COMPAT_FREEBSD12 option is included, as a slightly older -CURRENT userland and older FreeBSD userlands may not be functional without closefrom(2). 20200414: Upstream DTS from Linux 5.6 was merged and they now have the SID and THS (Secure ID controller and THermal Sensor) node present. The DTB overlays have now been removed from the tree for the H3/H5 and A64 SoCs and the aw_sid and aw_thermal driver have been updated to deal with upstream DTS. If you are using those overlays you need to remove them from loader.conf and update the DTBs on the FAT partition. 20200310: Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have been upgraded to 10.0.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20200309: The amd(8) automount daemon has been removed from the source tree. As of FreeBSD 10.1 autofs(5) is the preferred tool for automounting. amd is still available in the sysutils/am-utils port. 20200301: Removed brooktree driver (bktr.4) from the tree. 20200229: The WITH_GPL_DTC option has been removed. The BSD-licenced device tree compiler in usr.bin/dtc is used on all architectures which use dtc, and the GPL dtc is available (if needed) from the sysutils/dtc port. 20200229: The WITHOUT_LLVM_LIBUNWIND option has been removed. LLVM's libunwind is used by all supported CPU architectures. 20200229: GCC 4.2.1 has been removed from the tree. The WITH_GCC, WITH_GCC_BOOTSTRAP, and WITH_GNUCXX options are no longer available. Users who wish to build FreeBSD with GCC must use the external toolchain ports or packages. 20200220: ncurses has been updated to a newer version (6.2-20200215). Given the ABI has changed, users will have to rebuild all the ports that are linked to ncurses. 20200217: The size of struct vnet and the magic cookie have changed. Users need to recompile libkvm and all modules using VIMAGE together with their new kernel. 20200212: Defining the long deprecated NO_CTF, NO_DEBUG_FILES, NO_INSTALLLIB, NO_MAN, NO_PROFILE, and NO_WARNS variables is now an error. Update your Makefiles and scripts to define MK_=no instead as required. One exception to this is that program or library Makefiles should define MAN to empty rather than setting MK_MAN=no. 20200108: Clang/LLVM is now the default compiler and LLD the default linker for riscv64. 20200107: make universe no longer uses GCC 4.2.1 on any architectures. Architectures not supported by in-tree Clang/LLVM require an external toolchain package. 20200104: GCC 4.2.1 is now not built by default, as part of the GCC 4.2.1 retirement plan. Specifically, the GCC, GCC_BOOTSTRAP, and GNUCXX options default to off for all supported CPU architectures. As a short-term transition aid they may be enabled via WITH_* options. GCC 4.2.1 is expected to be removed from the tree on 2020-03-31. 20200102: Support for armv5 has been disconnected and is being removed. The machine combination MACHINE=arm MACHINE_ARCH=arm is no longer valid. You must now use a MACHINE_ARCH of armv6 or armv7. The default MACHINE_ARCH for MACHINE=arm is now armv7. 20191226: Clang/LLVM is now the default compiler for all powerpc architectures. LLD is now the default linker for powerpc64. The change for powerpc64 also includes a change to the ELFv2 ABI, incompatible with the existing ABI. 20191226: Kernel-loadable random(4) modules are no longer unloadable. 20191222: Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have been upgraded to 9.0.1. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20191212: r355677 has modified the internal interface used between the NFS modules in the kernel. As such, they must all be upgraded simultaneously. I will do a version bump for this. 20191205: The root certificates of the Mozilla CA Certificate Store have been imported into the base system and can be managed with the certctl(8) utility. If you have installed the security/ca_root_nss port or package with the ETCSYMLINK option (the default), be advised that there may be differences between those included in the port and those included in base due to differences in nss branch used as well as general update frequency. Note also that certctl(8) cannot manage certs in the format used by the security/ca_root_nss port. 20191120: The amd(8) automount daemon has been disabled by default, and will be removed in the future. As of FreeBSD 10.1 the autofs(5) is available for automounting. 20191107: The nctgpio and wbwd drivers have been moved to the superio bus. If you have one of these drivers in a kernel configuration, then you should add device superio to it. If you use one of these drivers as a module and you compile a custom set of modules, then you should add superio to the set. 20191021: KPIs for network drivers to access interface addresses have changed. Users need to recompile NIC driver modules together with kernel. 20191021: The net.link.tap.user_open sysctl no longer prevents user opening of already created /dev/tapNN devices. Access is still controlled by node permissions, just like tun devices. The net.link.tap.user_open sysctl is now used only to allow users to perform devfs cloning of tap devices, and the subsequent open may not succeed if the user is not in the appropriate group. This sysctl may be deprecated/removed completely in the future. 20191009: mips, powerpc, and sparc64 are no longer built as part of universe / tinderbox unless MAKE_OBSOLETE_GCC is defined. If not defined, mips, powerpc, and sparc64 builds will look for the xtoolchain binaries and if installed use them for universe builds. As llvm 9.0 becomes vetted for these architectures, they will be removed from the list. 20191009: Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have been upgraded to 9.0.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20191003: The hpt27xx, hptmv, hptnr, and hptrr drivers have been removed from GENERIC. They are available as modules and can be loaded by adding to /boot/loader.conf hpt27xx_load="YES", hptmv_load="YES", hptnr_load="YES", or hptrr_load="YES", respectively. 20190913: ntpd no longer by default locks its pages in memory, allowing them to be paged out by the kernel. Use rlimit memlock to restore historic BSD behaviour. For example, add "rlimit memlock 32" to ntp.conf to lock up to 32 MB of ntpd address space in memory. 20190823: Several of ping6's options have been renamed for better consistency with ping. If you use any of -ARWXaghmrtwx, you must update your scripts. See ping6(8) for details. 20190727: The vfs.fusefs.sync_unmount and vfs.fusefs.init_backgrounded sysctls and the "-o sync_unmount" and "-o init_backgrounded" mount options have been removed from mount_fusefs(8). You can safely remove them from your scripts, because they had no effect. The vfs.fusefs.fix_broken_io, vfs.fusefs.sync_resize, vfs.fusefs.refresh_size, vfs.fusefs.mmap_enable, vfs.fusefs.reclaim_revoked, and vfs.fusefs.data_cache_invalidate sysctls have been removed. If you felt the need to set any of them to a non-default value, please tell asomers@FreeBSD.org why. 20190713: Default permissions on the /var/account/acct file (and copies of it rotated by periodic daily scripts) are changed from 0644 to 0640 because the file contains sensitive information that should not be world-readable. If the /var/account directory must be created by rc.d/accounting, the mode used is now 0750. Admins who use the accounting feature are encouraged to change the mode of an existing /var/account directory to 0750 or 0700. 20190620: Entropy collection and the /dev/random device are no longer optional components. The "device random" option has been removed. Implementations of distilling algorithms can still be made loadable with "options RANDOM_LOADABLE" (e.g., random_fortuna.ko). 20190612: Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have been upgraded to 8.0.1. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20190608: A fix was applied to i386 kernel modules to avoid panics with dpcpu or vnet. Users need to recompile i386 kernel modules having pcpu or vnet sections or they will refuse to load. 20190513: User-wired pages now have their own counter, vm.stats.vm.v_user_wire_count. The vm.max_wired sysctl was renamed to vm.max_user_wired and changed from an unsigned int to an unsigned long. bhyve VMs wired with the -S are now subject to the user wiring limit; the vm.max_user_wired sysctl may need to be tuned to avoid running into the limit. 20190507: The IPSEC option has been removed from GENERIC. Users requiring ipsec(4) must now load the ipsec(4) kernel module. 20190507: The tap(4) driver has been folded into tun(4), and the module has been renamed to tuntap. You should update any kld_list="if_tap" or kld_list="if_tun" entries in /etc/rc.conf, if_tap_load="YES" or if_tun_load="YES" entries in /boot/loader.conf to load the if_tuntap module instead, and "device tap" or "device tun" entries in kernel config files to select the tuntap device instead. 20190418: The following knobs have been added related to tradeoffs between safe use of the random device and availability in the absence of entropy: kern.random.initial_seeding.bypass_before_seeding: tunable; set non-zero to bypass the random device prior to seeding, or zero to block random requests until the random device is initially seeded. For now, set to 1 (unsafe) by default to restore pre-r346250 boot availability properties. kern.random.initial_seeding.read_random_bypassed_before_seeding: read-only diagnostic sysctl that is set when bypass is enabled and read_random(9) is bypassed, to enable programmatic handling of this initial condition, if desired. kern.random.initial_seeding.arc4random_bypassed_before_seeding: Similar to the above, but for arc4random(9) initial seeding. kern.random.initial_seeding.disable_bypass_warnings: tunable; set non-zero to disable warnings in dmesg when the same conditions are met as for the diagnostic sysctls above. Defaults to zero, i.e., produce warnings in dmesg when the conditions are met. 20190416: The loadable random module KPI has changed; the random_infra_init() routine now requires a 3rd function pointer for a bool (*)(void) method that returns true if the random device is seeded (and therefore unblocked). 20190404: r345895 reverts r320698. This implies that an nfsuserd(8) daemon built from head sources between r320757 (July 6, 2017) and r338192 (Aug. 22, 2018) will not work unless the "-use-udpsock" is added to the command line. nfsuserd daemons built from head sources that are post-r338192 are not affected and should continue to work. 20190320: The fuse(4) module has been renamed to fusefs(4) for consistency with other filesystems. You should update any kld_load="fuse" entries in /etc/rc.conf, fuse_load="YES" entries in /boot/loader.conf, and "options FUSE" entries in kernel config files. 20190304: Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to 8.0.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20190226: geom_uzip(4) depends on the new module xz. If geom_uzip is statically compiled into your custom kernel, add 'device xz' statement to the kernel config. 20190219: drm and drm2 have been removed from the tree. Please see https://wiki.freebsd.org/Graphics for the latest information on migrating to the drm ports. 20190131: Iflib is no longer unconditionally compiled into the kernel. Drivers using iflib and statically compiled into the kernel, now require the 'device iflib' config option. For the same drivers loaded as modules on kernels not having 'device iflib', the iflib.ko module is loaded automatically. 20190125: The IEEE80211_AMPDU_AGE and AH_SUPPORT_AR5416 kernel configuration options no longer exist since r343219 and r343427 respectively; nothing uses them, so they should be just removed from custom kernel config files. 20181230: r342635 changes the way efibootmgr(8) works by requiring users to add the -b (bootnum) parameter for commands where the bootnum was previously specified with each option. For example 'efibootmgr -B 0001' is now 'efibootmgr -B -b 0001'. 20181220: r342286 modifies the NFSv4 server so that it obeys vfs.nfsd.nfs_privport in the same as it is applied to NFSv2 and 3. This implies that NFSv4 servers that have vfs.nfsd.nfs_privport set will only allow mounts from clients using a reserved port. Since both the FreeBSD and Linux NFSv4 clients use reserved ports by default, this should not affect most NFSv4 mounts. 20181219: The XLP config has been removed. We can't support 64-bit atomics in this kernel because it is running in 32-bit mode. XLP users must transition to running a 64-bit kernel (XLP64 or XLPN32). The mips GXEMUL support has been removed from FreeBSD. MALTA* + qemu is the preferred emulator today and we don't need two different ones. The old sibyte / swarm / Broadcom BCM1250 support has been removed from the mips port. 20181211: Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to 7.0.1. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20181211: Remove the timed and netdate programs from the base tree. Setting the time with these daemons has been obsolete for over a decade. 20181126: On amd64, arm64 and armv7 (architectures that install LLVM's ld.lld linker as /usr/bin/ld) GNU ld is no longer installed as ld.bfd, as it produces broken binaries when ifuncs are in use. Users needing GNU ld should install the binutils port or package. 20181123: The BSD crtbegin and crtend code has been enabled by default. It has had extensive testing on amd64, arm64, and i386. It can be disabled by building a world with -DWITHOUT_BSD_CRTBEGIN. 20181115: The set of CTM commands (ctm, ctm_smail, ctm_rmail, ctm_dequeue) has been converted to a port (misc/ctm) and will be removed from FreeBSD-13. It is available as a package (ctm) for all supported FreeBSD versions. 20181110: The default newsyslog.conf(5) file has been changed to only include files in /etc/newsyslog.conf.d/ and /usr/local/etc/newsyslog.conf.d/ if the filenames end in '.conf' and do not begin with a '.'. You should check the configuration files in these two directories match this naming convention. You can verify which configuration files are being included using the command: $ newsyslog -Nrv 20181019: Stable/12 was branched created. 20181015: Ports for the DRM modules have been simplified. Now, amd64 users should just install the drm-kmod port. All others should install drm-legacy-kmod. Graphics hardware that's newer than about 2010 usually works with drm-kmod. For hardware older than 2013, however, some users will need to use drm-legacy-kmod if drm-kmod doesn't work for them. Hardware older than 2008 usually only works in drm-legacy-kmod. The graphics team can only commit to hardware made since 2013 due to the complexity of the market and difficulty to test all the older cards effectively. If you have hardware supported by drm-kmod, you are strongly encouraged to use that as you will get better support. Other than KPI chasing, drm-legacy-kmod will not be updated. As outlined elsewhere, the drm and drm2 modules will be eliminated from the src base soon (with a limited exception for arm). Please update to the package asap and report any issues to x11@freebsd.org. Generally, anybody using the drm*-kmod packages should add WITHOUT_DRM_MODULE=t and WITHOUT_DRM2_MODULE=t to avoid nasty cross-threading surprises, especially with automatic driver loading from X11 startup. These will become the defaults in 13-current shortly. 20181012: The ixlv(4) driver has been renamed to iavf(4). As a consequence, custom kernel and module loading configuration files must be updated accordingly. Moreover, interfaces previous presented as ixlvN to the system are now exposed as iavfN and network configuration files must be adjusted as necessary. 20181009: OpenSSL has been updated to version 1.1.1. This update included additional various API changes throughout the base system. It is important to rebuild third-party software after upgrading. The value of __FreeBSD_version has been bumped accordingly. 20181006: The legacy DRM modules and drivers have now been added to the loader's module blacklist, in favor of loading them with kld_list in rc.conf(5). The module blacklist may be overridden with the loader.conf(5) 'module_blacklist' variable, but loading them via rc.conf(5) is strongly encouraged. 20181002: The cam(4) based nda(4) driver will be used over nvd(4) by default on powerpc64. You may set 'options NVME_USE_NVD=1' in your kernel conf or loader tunable 'hw.nvme.use_nvd=1' if you wish to use the existing driver. Make sure to edit /boot/etc/kboot.conf and fstab to use the nda device name. 20180913: Reproducible build mode is now on by default, in preparation for FreeBSD 12.0. This eliminates build metadata such as the user, host, and time from the kernel (and uname), unless the working tree corresponds to a modified checkout from a version control system. The previous behavior can be obtained by setting the /etc/src.conf knob WITHOUT_REPRODUCIBLE_BUILD. 20180826: The Yarrow CSPRNG has been removed from the kernel as it has not been supported by its designers since at least 2003. Fortuna has been the default since FreeBSD-11. 20180822: devctl freeze/thaw have gone into the tree, the rc scripts have been updated to use them and devmatch has been changed. You should update kernel, userland and rc scripts all at the same time. 20180818: The default interpreter has been switched from 4th to Lua. LOADER_DEFAULT_INTERP, documented in build(7), will override the default interpreter. If you have custom FORTH code you will need to set LOADER_DEFAULT_INTERP=4th (valid values are 4th, lua or simp) in src.conf for the build. This will create default hard links between loader and loader_4th instead of loader and loader_lua, the new default. If you are using UEFI it will create the proper hard link to loader.efi. bhyve uses userboot.so. It remains 4th-only until some issues are solved regarding coexisting with multiple versions of FreeBSD are resolved. 20180815: ls(1) now respects the COLORTERM environment variable used in other systems and software to indicate that a colored terminal is both supported and desired. If ls(1) is suddenly emitting colors, they may be disabled again by either removing the unwanted COLORTERM from your environment, or using `ls --color=never`. The ls(1) specific CLICOLOR may not be observed in a future release. 20180808: The default pager for most commands has been changed to "less". To restore the old behavior, set PAGER="more" and MANPAGER="more -s" in your environment. 20180731: The jedec_ts(4) driver has been removed. A superset of its functionality is available in the jedec_dimm(4) driver, and the manpage for that driver includes migration instructions. If you have "device jedec_ts" in your kernel configuration file, it must be removed. 20180730: amd64/GENERIC now has EFI runtime services, EFIRT, enabled by default. This should have no effect if the kernel is booted via BIOS/legacy boot. EFIRT may be disabled via a loader tunable, efi.rt.disabled, if a system has a buggy firmware that prevents a successful boot due to use of runtime services. 20180727: Atmel AT91RM9200 and AT91SAM9, Cavium CNS 11xx and XScale support has been removed from the tree. These ports were obsolete and/or known to be broken for many years. 20180723: loader.efi has been augmented to participate more fully in the UEFI boot manager protocol. loader.efi will now look at the BootXXXX environment variable to determine if a specific kernel or root partition was specified. XXXX is derived from BootCurrent. efibootmgr(8) manages these standard UEFI variables. 20180720: zfsloader's functionality has now been folded into loader. zfsloader is no longer necessary once you've updated your boot blocks. For a transition period, we will install a hardlink for zfsloader to loader to allow a smooth transition until the boot blocks can be updated (hard link because old zfs boot blocks don't understand symlinks). 20180719: ARM64 now have efifb support, if you want to have serial console on your arm64 board when an screen is connected and the bootloader setup a frame buffer for us to use, just add : boot_serial=YES boot_multicons=YES in /boot/loader.conf For Raspberry Pi 3 (RPI) users, this is needed even if you don't have an screen connected as the firmware will setup a frame buffer are that u-boot will expose as an EFI frame buffer. 20180719: New uid:gid added, ntpd:ntpd (123:123). Be sure to run mergemaster or take steps to update /etc/passwd before doing installworld on existing systems. Do not skip the "mergemaster -Fp" step before installworld, as described in the update procedures near the bottom of this document. Also, rc.d/ntpd now starts ntpd(8) as user ntpd if the new mac_ntpd(4) policy is available, unless ntpd_flags or the ntp config file contain options that change file/dir locations. When such options (e.g., "statsdir" or "crypto") are used, ntpd can still be run as non-root by setting ntpd_user=ntpd in rc.conf, after taking steps to ensure that all required files/dirs are accessible by the ntpd user. 20180717: Big endian arm support has been removed. 20180711: The static environment setup in kernel configs is no longer mutually exclusive with the loader(8) environment by default. In order to restore the previous default behavior of disabling the loader(8) environment if a static environment is present, you must specify loader_env.disabled=1 in the static environment. 20180705: The ABI of syscalls used by management tools like sockstat and netstat has been broken to allow 32-bit binaries to work on 64-bit kernels without modification. These programs will need to match the kernel in order to function. External programs may require minor modifications to accommodate a change of type in structures from pointers to 64-bit virtual addresses. 20180702: On i386 and amd64 atomics are now inlined. Out of tree modules using atomics will need to be rebuilt. 20180701: The '%I' format in the kern.corefile sysctl limits the number of core files that a process can generate to the number stored in the debug.ncores sysctl. The '%I' format is replaced by the single digit index. Previously, if all indexes were taken the kernel would overwrite only a core file with the highest index in a filename. Currently the system will create a new core file if there is a free index or if all slots are taken it will overwrite the oldest one. 20180630: Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to 6.0.1. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20180628: r335753 introduced a new quoting method. However, etc/devd/devmatch.conf needed to be changed to work with it. This change was made with r335763 and requires a mergemaster / etcupdate / etc to update the installed file. 20180612: r334930 changed the interface between the NFS modules, so they all need to be rebuilt. r335018 did a __FreeBSD_version bump for this. 20180530: As of r334391 lld is the default amd64 system linker; it is installed as /usr/bin/ld. Kernel build workarounds (see 20180510 entry) are no longer necessary. 20180530: The kernel / userland interface for devinfo changed, so you'll need a new kernel and userland as a pair for it to work (rebuilding lib/libdevinfo is all that's required). devinfo and devmatch will not work, but everything else will when there's a mismatch. 20180523: The on-disk format for hwpmc callchain records has changed to include threadid corresponding to a given record. This changes the field offsets and thus requires that libpmcstat be rebuilt before using a kernel later than r334108. 20180517: The vxge(4) driver has been removed. This driver was introduced into HEAD one week before the Exar left the Ethernet market and is not known to be used. If you have device vxge in your kernel config file it must be removed. 20180510: The amd64 kernel now requires a ld that supports ifunc to produce a working kernel, either lld or a newer binutils. lld is built by default on amd64, and the 'buildkernel' target uses it automatically. However, it is not the default linker, so building the kernel the traditional way requires LD=ld.lld on the command line (or LD=/usr/local/bin/ld for binutils port/package). lld will soon be default, and this requirement will go away. NOTE: As of r334391 lld is the default system linker on amd64, and no workaround is necessary. 20180508: The nxge(4) driver has been removed. This driver was for PCI-X 10g cards made by s2io/Neterion. The company was acquired by Exar and no longer sells or supports Ethernet products. If you have device nxge in your kernel config file it must be removed. 20180504: The tz database (tzdb) has been updated to 2018e. This version more correctly models time stamps in time zones with negative DST such as Europe/Dublin (from 1971 on), Europe/Prague (1946/7), and Africa/Windhoek (1994/2017). This does not affect the UT offsets, only time zone abbreviations and the tm_isdst flag. 20180502: The ixgb(4) driver has been removed. This driver was for an early and uncommon legacy PCI 10GbE for a single ASIC, Intel 82597EX. Intel quickly shifted to the long lived ixgbe family. If you have device ixgb in your kernel config file it must be removed. 20180501: The lmc(4) driver has been removed. This was a WAN interface card that was already reportedly rare in 2003, and had an ambiguous license. If you have device lmc in your kernel config file it must be removed. 20180413: Support for Arcnet networks has been removed. If you have device arcnet or device cm in your kernel config file they must be removed. 20180411: Support for FDDI networks has been removed. If you have device fddi or device fpa in your kernel config file they must be removed. 20180406: In addition to supporting RFC 3164 formatted messages, the syslogd(8) service is now capable of parsing RFC 5424 formatted log messages. The main benefit of using RFC 5424 is that clients may now send log messages with timestamps containing year numbers, microseconds and time zone offsets. Similarly, the syslog(3) C library function has been altered to send RFC 5424 formatted messages to the local system logging daemon. On systems using syslogd(8), this change should have no negative impact, as long as syslogd(8) and the C library are updated at the same time. On systems using a different system logging daemon, it may be necessary to make configuration adjustments, depending on the software used. When using syslog-ng, add the 'syslog-protocol' flag to local input sources to enable parsing of RFC 5424 formatted messages: source src { unix-dgram("/var/run/log" flags(syslog-protocol)); } When using rsyslog, disable the 'SysSock.UseSpecialParser' option of the 'imuxsock' module to let messages be processed by the regular RFC 3164/5424 parsing pipeline: module(load="imuxsock" SysSock.UseSpecialParser="off") Do note that these changes only affect communication between local applications and syslogd(8). The format that syslogd(8) uses to store messages on disk or forward messages to other systems remains unchanged. syslogd(8) still uses RFC 3164 for these purposes. Options to customize this behaviour will be added in the future. Utilities that process log files stored in /var/log are thus expected to continue to function as before. __FreeBSD_version has been incremented to 1200061 to denote this change. 20180328: Support for token ring networks has been removed. If you have "device token" in your kernel config you should remove it. No device drivers supported token ring. 20180323: makefs was modified to be able to tag ISO9660 El Torito boot catalog entries as EFI instead of overloading the i386 tag as done previously. The amd64 mkisoimages.sh script used to build amd64 ISO images for release was updated to use this. This may mean that makefs must be updated before "make cdrom" can be run in the release directory. This should be as simple as: $ cd $SRCDIR/usr.sbin/makefs $ make depend all install 20180212: FreeBSD boot loader enhanced with Lua scripting. It's purely opt-in for now by building WITH_LOADER_LUA and WITHOUT_FORTH in /etc/src.conf. Co-existence for the transition period will come shortly. Booting is a complex environment and test coverage for Lua-enabled loaders has been thin, so it would be prudent to assume it might not work and make provisions for backup boot methods. 20180211: devmatch functionality has been turned on in devd. It will automatically load drivers for unattached devices. This may cause unexpected drivers to be loaded. Please report any problems to current@ and imp@freebsd.org. 20180114: Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to 6.0.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20180110: LLVM's lld linker is now used as the FreeBSD/amd64 bootstrap linker. This means it is used to link the kernel and userland libraries and executables, but is not yet installed as /usr/bin/ld by default. To revert to ld.bfd as the bootstrap linker, in /etc/src.conf set WITHOUT_LLD_BOOTSTRAP=yes 20180110: On i386, pmtimer has been removed. Its functionality has been folded into apm. It was a no-op on ACPI in current for a while now (but was still needed on i386 in FreeBSD 11 and earlier). Users may need to remove it from kernel config files. 20180104: The use of RSS hash from the network card aka flowid has been disabled by default for lagg(4) as it's currently incompatible with the lacp and loadbalance protocols. This can be re-enabled by setting the following in loader.conf: net.link.lagg.default_use_flowid="1" 20180102: The SW_WATCHDOG option is no longer necessary to enable the hardclock-based software watchdog if no hardware watchdog is configured. As before, SW_WATCHDOG will cause the software watchdog to be enabled even if a hardware watchdog is configured. 20171215: r326887 fixes the issue described in the 20171214 UPDATING entry. r326888 flips the switch back to building GELI support always. 20171214: r362593 broke ZFS + GELI support for reasons unknown. However, it also broke ZFS support generally, so GELI has been turned off by default as the lesser evil in r326857. If you boot off ZFS and/or GELI, it might not be a good time to update. 20171125: PowerPC users must update loader(8) by rebuilding world before installing a new kernel, as the protocol connecting them has changed. Without the update, loader metadata will not be passed successfully to the kernel and users will have to enter their root partition at the kernel mountroot prompt to continue booting. Newer versions of loader can boot old kernels without issue. 20171110: The LOADER_FIREWIRE_SUPPORT build variable has been renamed to WITH/OUT_LOADER_FIREWIRE. LOADER_{NO_,}GELI_SUPPORT has been renamed to WITH/OUT_LOADER_GELI. 20171106: The naive and non-compliant support of posix_fallocate(2) in ZFS has been removed as of r325320. The system call now returns EINVAL when used on a ZFS file. Although the new behavior complies with the standard, some consumers are not prepared to cope with it. One known victim is lld prior to r325420. 20171102: Building in a FreeBSD src checkout will automatically create object directories now rather than store files in the current directory if 'make obj' was not ran. Calling 'make obj' is no longer necessary. This feature can be disabled by setting WITHOUT_AUTO_OBJ=yes in /etc/src-env.conf (not /etc/src.conf), or passing the option in the environment. 20171101: The default MAKEOBJDIR has changed from /usr/obj/ for native builds, and /usr/obj// for cross-builds, to a unified /usr/obj//. This behavior can be changed to the old format by setting WITHOUT_UNIFIED_OBJDIR=yes in /etc/src-env.conf, the environment, or with -DWITHOUT_UNIFIED_OBJDIR when building. The UNIFIED_OBJDIR option is a transitional feature that will be removed for 12.0 release; please migrate to the new format for any tools by looking up the OBJDIR used by 'make -V .OBJDIR' means rather than hardcoding paths. 20171028: The native-xtools target no longer installs the files by default to the OBJDIR. Use the native-xtools-install target with a DESTDIR to install to ${DESTDIR}/${NXTP} where NXTP defaults to /nxb-bin. 20171021: As part of the boot loader infrastructure cleanup, LOADER_*_SUPPORT options are changing from controlling the build if defined / undefined to controlling the build with explicit 'yes' or 'no' values. They will shift to WITH/WITHOUT options to match other options in the system. 20171010: libstand has turned into a private library for sys/boot use only. It is no longer supported as a public interface outside of sys/boot. 20171005: The arm port has split armv6 into armv6 and armv7. armv7 is now a valid TARGET_ARCH/MACHINE_ARCH setting. If you have an armv7 system and are running a kernel from before r324363, you will need to add MACHINE_ARCH=armv7 to 'make buildworld' to do a native build. 20171003: When building multiple kernels using KERNCONF, non-existent KERNCONF files will produce an error and buildkernel will fail. Previously missing KERNCONF files silently failed giving no indication as to why, only to subsequently discover during installkernel that the desired kernel was never built in the first place. 20170912: The default serial number format for CTL LUNs has changed. This will affect users who use /dev/diskid/* device nodes, or whose FibreChannel or iSCSI clients care about their LUNs' serial numbers. Users who require serial number stability should hardcode serial numbers in /etc/ctl.conf . 20170912: For 32-bit arm compiled for hard-float support, soft-floating point binaries now always get their shared libraries from LD_SOFT_LIBRARY_PATH (in the past, this was only used if /usr/libsoft also existed). Only users with a hard-float ld.so, but soft-float everything else should be affected. 20170826: The geli password typed at boot is now hidden. To restore the previous behavior, see geli(8) for configuration options. 20170825: Move PMTUD blackhole counters to TCPSTATS and remove them from bare sysctl values. Minor nit, but requires a rebuild of both world/kernel to complete. 20170814: "make check" behavior (made in ^/head@r295380) has been changed to execute from a limited sandbox, as opposed to executing from ${TESTSDIR}. Behavioral changes: - The "beforecheck" and "aftercheck" targets are now specified. - ${CHECKDIR} (added in commit noted above) has been removed. - Legacy behavior can be enabled by setting WITHOUT_MAKE_CHECK_USE_SANDBOX in src.conf(5) or the environment. If the limited sandbox mode is enabled, "make check" will execute "make distribution", then install, execute the tests, and clean up the sandbox if successful. The "make distribution" and "make install" targets are typically run as root to set appropriate permissions and ownership at installation time. The end-user should set "WITH_INSTALL_AS_USER" in src.conf(5) or the environment if executing "make check" with limited sandbox mode using an unprivileged user. 20170808: Since the switch to GPT disk labels, fsck for UFS/FFS has been unable to automatically find alternate superblocks. As of r322297, the information needed to find alternate superblocks has been moved to the end of the area reserved for the boot block. Filesystems created with a newfs of this vintage or later will create the recovery information. If you have a filesystem created prior to this change and wish to have a recovery block created for your filesystem, you can do so by running fsck in foreground mode (i.e., do not use the -p or -y options). As it starts, fsck will ask ``SAVE DATA TO FIND ALTERNATE SUPERBLOCKS'' to which you should answer yes. 20170728: As of r321665, an NFSv4 server configuration that services Kerberos mounts or clients that do not support the uid/gid in owner/owner_group string capability, must explicitly enable the nfsuserd daemon by adding nfsuserd_enable="YES" to the machine's /etc/rc.conf file. 20170722: Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 5.0.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20170701: WITHOUT_RCMDS is now the default. Set WITH_RCMDS if you need the r-commands (rlogin, rsh, etc.) to be built with the base system. 20170625: The FreeBSD/powerpc platform now uses a 64-bit type for time_t. This is a very major ABI incompatible change, so users of FreeBSD/powerpc must be careful when performing source upgrades. It is best to run 'make installworld' from an alternate root system, either a live CD/memory stick, or a temporary root partition. Additionally, all ports must be recompiled. powerpc64 is largely unaffected, except in the case of 32-bit compatibility. All 32-bit binaries will be affected. 20170623: Forward compatibility for the "ino64" project have been committed. This will allow most new binaries to run on older kernels in a limited fashion. This prevents many of the common foot-shooting actions in the upgrade as well as the limited ability to roll back the kernel across the ino64 upgrade. Complicated use cases may not work properly, though enough simpler ones work to allow recovery in most situations. 20170620: Switch back to the BSDL dtc (Device Tree Compiler). Set WITH_GPL_DTC if you require the GPL compiler. 20170619: Forward compatibility for the "ino64" project have been committed. This will allow most new binaries to run on older kernels in a limited fashion. This prevents many of the common foot-shooting actions in the upgrade as well as the limited ability to roll back the kernel across the ino64 upgrade. Complicated use cases may not work properly, though enough simpler ones work to allow recovery in most situations. 20170618: The internal ABI used for communication between the NFS kernel modules was changed by r320085, so __FreeBSD_version was bumped to ensure all the NFS related modules are updated together. 20170617: The ABI of struct event was changed by extending the data member to 64bit and adding ext fields. For upgrade, same precautions as for the entry 20170523 "ino64" must be followed. 20170531: The GNU roff toolchain has been removed from base. To render manpages which are not supported by mandoc(1), man(1) can fallback on GNU roff from ports (and recommends to install it). To render roff(7) documents, consider using GNU roff from ports or the heirloom doctools roff toolchain from ports via pkg install groff or via pkg install heirloom-doctools. 20170524: The ath(4) and ath_hal(4) modules now build piecemeal to allow for smaller runtime footprint builds. This is useful for embedded systems which only require one chipset support. If you load it as a module, make sure this is in /boot/loader.conf: if_ath_load="YES" This will load the HAL, all chip/RF backends and if_ath_pci. If you have if_ath_pci in /boot/loader.conf, ensure it is after if_ath or it will not load any HAL chipset support. If you want to selectively load things (eg on cheaper ARM/MIPS platforms where RAM is at a premium) you should: * load ath_hal * load the chip modules in question * load ath_rate, ath_dfs * load ath_main * load if_ath_pci and/or if_ath_ahb depending upon your particular bus bind type - this is where probe/attach is done. For further comments/feedback, poke adrian@ . 20170523: The "ino64" 64-bit inode project has been committed, which extends a number of types to 64 bits. Upgrading in place requires care and adherence to the documented upgrade procedure. If using a custom kernel configuration ensure that the COMPAT_FREEBSD11 option is included (as during the upgrade the system will be running the ino64 kernel with the existing world). For the safest in-place upgrade begin by removing previous build artifacts via "rm -rf /usr/obj/*". Then, carefully follow the full procedure documented below under the heading "To rebuild everything and install it on the current system." Specifically, a reboot is required after installing the new kernel before installing world. While an installworld normally works by accident from multiuser after rebooting the proper kernel, there are many cases where this will fail across this upgrade and installworld from single user is required. 20170424: The NATM framework including the en(4), fatm(4), hatm(4), and patm(4) devices has been removed. Consumers should plan a migration before the end-of-life date for FreeBSD 11. 20170420: GNU diff has been replaced by a BSD licensed diff. Some features of GNU diff has not been implemented, if those are needed a newer version of GNU diff is available via the diffutils package under the gdiff name. 20170413: As of r316810 for ipfilter, keep frags is no longer assumed when keep state is specified in a rule. r316810 aligns ipfilter with documentation in man pages separating keep frags from keep state. This allows keep state to be specified without forcing keep frags and allows keep frags to be specified independently of keep state. To maintain previous behaviour, also specify keep frags with keep state (as documented in ipf.conf.5). 20170407: arm64 builds now use the base system LLD 4.0.0 linker by default, instead of requiring that the aarch64-binutils port or package be installed. To continue using aarch64-binutils, set CROSS_BINUTILS_PREFIX=/usr/local/aarch64-freebsd/bin . 20170405: The UDP optimization in entry 20160818 that added the sysctl net.inet.udp.require_l2_bcast has been reverted. L2 broadcast packets will no longer be treated as L3 broadcast packets. 20170331: Binds and sends to the loopback addresses, IPv6 and IPv4, will now use any explicitly assigned loopback address available in the jail instead of using the first assigned address of the jail. 20170329: The ctl.ko module no longer implements the iSCSI target frontend: cfiscsi.ko does instead. If building cfiscsi.ko as a kernel module, the module can be loaded via one of the following methods: - `cfiscsi_load="YES"` in loader.conf(5). - Add `cfiscsi` to `$kld_list` in rc.conf(5). - ctladm(8)/ctld(8), when compiled with iSCSI support (`WITH_ISCSI=yes` in src.conf(5)) Please see cfiscsi(4) for more details. 20170316: The mmcsd.ko module now additionally depends on geom_flashmap.ko. Also, mmc.ko and mmcsd.ko need to be a matching pair built from the same source (previously, the dependency of mmcsd.ko on mmc.ko was missing, but mmcsd.ko now will refuse to load if it is incompatible with mmc.ko). 20170315: The syntax of ipfw(8) named states was changed to avoid ambiguity. If you have used named states in the firewall rules, you need to modify them after installworld and before rebooting. Now named states must be prefixed with colon. 20170311: The old drm (sys/dev/drm/) drivers for i915 and radeon have been removed as the userland we provide cannot use them. The KMS version (sys/dev/drm2) supports the same hardware. 20170302: Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 4.0.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20170221: The code that provides support for ZFS .zfs/ directory functionality has been reimplemented. It's not possible now to create a snapshot by mkdir under .zfs/snapshot/. That should be the only user visible change. 20170216: EISA bus support has been removed. The WITH_EISA option is no longer valid. 20170215: MCA bus support has been removed. 20170127: The WITH_LLD_AS_LD / WITHOUT_LLD_AS_LD build knobs have been renamed WITH_LLD_IS_LD / WITHOUT_LLD_IS_LD, for consistency with CLANG_IS_CC. 20170112: The EM_MULTIQUEUE kernel configuration option is deprecated now that the em(4) driver conforms to iflib specifications. 20170109: The igb(4), em(4) and lem(4) ethernet drivers are now implemented via IFLIB. If you have a custom kernel configuration that excludes em(4) but you use igb(4), you need to re-add em(4) to your custom configuration. 20161217: Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 3.9.1. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20161124: Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 3.9.0. Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. 20161119: The layout of the pmap structure has changed for powerpc to put the pmap statistics at the front for all CPU variations. libkvm(3) and all tools that link against it need to be recompiled. 20161030: isl(4) and cyapa(4) drivers now require a new driver, chromebook_platform(4), to work properly on Chromebook-class hardware. On other types of hardware the drivers may need to be configured using device hints. Please see the corresponding manual pages for details. 20161017: The urtwn(4) driver was merged into rtwn(4) and now consists of rtwn(4) main module + rtwn_usb(4) and rtwn_pci(4) bus-specific parts. Also, firmware for RTL8188CE was renamed due to possible name conflict (rtwnrtl8192cU(B) -> rtwnrtl8192cE(B)) 20161015: GNU rcs has been removed from base. It is available as packages: - rcs: Latest GPLv3 GNU rcs version. - rcs57: Copy of the latest version of GNU rcs (GPLv2) before it was removed from base. 20161008: Use of the cc_cdg, cc_chd, cc_hd, or cc_vegas congestion control modules now requires that the kernel configuration contain the TCP_HHOOK option. (This option is included in the GENERIC kernel.) 20161003: The WITHOUT_ELFCOPY_AS_OBJCOPY src.conf(5) knob has been retired. ELF Tool Chain's elfcopy is always installed as /usr/bin/objcopy. 20160924: Relocatable object files with the extension of .So have been renamed to use an extension of .pico instead. The purpose of this change is to avoid a name clash with shared libraries on case-insensitive file systems. On those file systems, foo.So is the same file as foo.so. 20160918: GNU rcs has been turned off by default. It can (temporarily) be built again by adding WITH_RCS knob in src.conf. Otherwise, GNU rcs is available from packages: - rcs: Latest GPLv3 GNU rcs version. - rcs57: Copy of the latest version of GNU rcs (GPLv2) from base. 20160918: The backup_uses_rcs functionality has been removed from rc.subr. 20160908: The queue(3) debugging macro, QUEUE_MACRO_DEBUG, has been split into two separate components, QUEUE_MACRO_DEBUG_TRACE and QUEUE_MACRO_DEBUG_TRASH. Define both for the original QUEUE_MACRO_DEBUG behavior. 20160824: r304787 changed some ioctl interfaces between the iSCSI userspace programs and the kernel. ctladm, ctld, iscsictl, and iscsid must be rebuilt to work with new kernels. __FreeBSD_version has been bumped to 1200005. 20160818: The UDP receive code has been updated to only treat incoming UDP packets that were addressed to an L2 broadcast address as L3 broadcast packets. It is not expected that this will affect any standards-conforming UDP application. The new behaviour can be disabled by setting the sysctl net.inet.udp.require_l2_bcast to 0. 20160818: Remove the openbsd_poll system call. __FreeBSD_version has been bumped because of this. 20160708: The stable/11 branch has been created from head@r302406. After branch N is created, entries older than the N-2 branch point are removed from this file. After stable/14 is branched and current becomes FreeBSD 15, entries older than stable/12 branch point will be removed from current's UPDATING file. COMMON ITEMS: General Notes ------------- Sometimes, obscure build problems are the result of environment poisoning. This can happen because the make utility reads its environment when searching for values for global variables. To run your build attempts in an "environmental clean room", prefix all make commands with 'env -i '. See the env(1) manual page for more details. Occasionally a build failure will occur with "make -j" due to a race condition. If this happens try building again without -j, and please report a bug if it happens consistently. When upgrading from one major version to another it is generally best to upgrade to the latest code in the currently installed branch first, then do an upgrade to the new branch. This is the best-tested upgrade path, and has the highest probability of being successful. Please try this approach if you encounter problems with a major version upgrade. Since the stable 4.x branch point, one has generally been able to upgrade from anywhere in the most recent stable branch to head / current (or even the last couple of stable branches). See the top of this file when there's an exception. The update process will emit an error on an attempt to perform a build or install from a FreeBSD version below the earliest supported version. When updating from an older version the update should be performed one major release at a time, including running `make delete-old` at each step. When upgrading a live system, having a root shell around before installing anything can help undo problems. Not having a root shell around can lead to problems if pam has changed too much from your starting point to allow continued authentication after the upgrade. This file should be read as a log of events. When a later event changes information of a prior event, the prior event should not be deleted. Instead, a pointer to the entry with the new information should be placed in the old entry. Readers of this file should also sanity check older entries before relying on them blindly. Authors of new entries should write them with this in mind. ZFS notes --------- When upgrading the boot ZFS pool to a new version (via zpool upgrade), always follow these three steps: 1) recompile and reinstall the ZFS boot loader and boot block (this is part of "make buildworld" and "make installworld") 2) update the ZFS boot block on your boot drive (only required when doing a zpool upgrade): When booting on x86 via BIOS, use the following to update the ZFS boot block on the freebsd-boot partition of a GPT partitioned drive ada0: gpart bootcode -p /boot/gptzfsboot -i $N ada0 The value $N will typically be 1. For EFI booting, see EFI notes. 3) zpool upgrade the root pool. New bootblocks will work with old pools, but not vice versa, so they need to be updated before any zpool upgrade. Non-boot pools do not need these updates. EFI notes --------- There are two locations the boot loader can be installed into. The current location (and the default) is \efi\freebsd\loader.efi and using efibootmgr(8) to configure it. The old location, that must be used on deficient systems that don't honor efibootmgr(8) protocols, is the fallback location of \EFI\BOOT\BOOTxxx.EFI. Generally, you will copy /boot/loader.efi to this location, but on systems installed a long time ago the ESP may be too small and /boot/boot1.efi may be needed unless the ESP has been expanded in the meantime. Recent systems will have the ESP mounted on /boot/efi, but older ones may not have it mounted at all, or mounted in a different location. Older arm SD images with MBR used /boot/msdos as the mountpoint. The ESP is a MSDOS filesystem. The EFI boot loader rarely needs to be updated. For ZFS booting, however, you must update loader.efi before you do 'zpool upgrade' the root zpool, otherwise the old loader.efi may reject the upgraded zpool since it does not automatically understand some new features. See loader.efi(8) and uefi(8) for more details. To build a kernel ----------------- If you are updating from a prior version of FreeBSD (even one just a few days old), you should follow this procedure. It is the most failsafe as it uses a /usr/obj tree with a fresh mini-buildworld, make kernel-toolchain make -DALWAYS_CHECK_MAKE buildkernel KERNCONF=YOUR_KERNEL_HERE make -DALWAYS_CHECK_MAKE installkernel KERNCONF=YOUR_KERNEL_HERE To test a kernel once --------------------- If you just want to boot a kernel once (because you are not sure if it works, or if you want to boot a known bad kernel to provide debugging information) run make installkernel KERNCONF=YOUR_KERNEL_HERE KODIR=/boot/testkernel nextboot -k testkernel To rebuild everything and install it on the current system. ----------------------------------------------------------- # Note: sometimes if you are running current you gotta do more than # is listed here if you are upgrading from a really old current. make buildworld make buildkernel KERNCONF=YOUR_KERNEL_HERE make installkernel KERNCONF=YOUR_KERNEL_HERE [1] [3] etcupdate -p [5] make installworld etcupdate -B [4] make delete-old [6] To cross-install current onto a separate partition -------------------------------------------------- # In this approach we use a separate partition to hold # current's root, 'usr', and 'var' directories. A partition # holding "/", "/usr" and "/var" should be about 2GB in # size. make buildworld make buildkernel KERNCONF=YOUR_KERNEL_HERE make installworld DESTDIR=${CURRENT_ROOT} -DDB_FROM_SRC make distribution DESTDIR=${CURRENT_ROOT} # if newfs'd make installkernel KERNCONF=YOUR_KERNEL_HERE DESTDIR=${CURRENT_ROOT} cp /etc/fstab ${CURRENT_ROOT}/etc/fstab # if newfs'd To upgrade in-place from stable to current ---------------------------------------------- make buildworld [9] make buildkernel KERNCONF=YOUR_KERNEL_HERE [8] make installkernel KERNCONF=YOUR_KERNEL_HERE [1] [3] etcupdate -p [5] make installworld etcupdate -B [4] make delete-old [6] Make sure that you've read the UPDATING file to understand the tweaks to various things you need. At this point in the life cycle of current, things change often and you are on your own to cope. The defaults can also change, so please read ALL of the UPDATING entries. Also, if you are tracking -current, you must be subscribed to freebsd-current@freebsd.org. Make sure that before you update your sources that you have read and understood all the recent messages there. If in doubt, please track -stable which has much fewer pitfalls. [1] If you have third party modules, such as vmware, you should disable them at this point so they don't crash your system on reboot. Alternatively, you should rebuild all the modules you have in your system and install them as well. If you are running -current, you should seriously consider placing all sources to all the modules for your system (or symlinks to them) in /usr/local/sys/modules so this happens automatically. If all your modules come from ports, then adding the port origin directories to PORTS_MODULES instead is also automatic and effective, eg: PORTS_MODULES+=x11/nvidia-driver [3] From the bootblocks, boot -s, and then do fsck -p mount -u / mount -a sh /etc/rc.d/zfs start # mount zfs filesystem, if needed cd src # full path to source adjkerntz -i # if CMOS is wall time Also, when doing a major release upgrade, it is required that you boot into single user mode to do the installworld. [4] Note: This step is non-optional. Failure to do this step can result in a significant reduction in the functionality of the system. Attempting to do it by hand is not recommended and those that pursue this avenue should read this file carefully, as well as the archives of freebsd-current and freebsd-hackers mailing lists for potential gotchas. See etcupdate(8) for more information. [5] Usually this step is a no-op. However, from time to time you may need to do this if you get unknown user in the following step. [6] This only deletes old files and directories. Old libraries can be deleted by "make delete-old-libs", but you have to make sure that no program is using those libraries anymore. [8] The new kernel must be able to run existing binaries used by an installworld. When upgrading across major versions, the new kernel's configuration must include the correct COMPAT_FREEBSD option for existing binaries (e.g. COMPAT_FREEBSD11 to run 11.x binaries). Failure to do so may leave you with a system that is hard to boot to recover. A GENERIC kernel will include suitable compatibility options to run binaries from older branches. Note that the ability to run binaries from unsupported branches is not guaranteed. Make sure that you merge any new devices from GENERIC since the last time you updated your kernel config file. Options also change over time, so you may need to adjust your custom kernels for these as well. [9] If CPUTYPE is defined in your /etc/make.conf, make sure to use the "?=" instead of the "=" assignment operator, so that buildworld can override the CPUTYPE if it needs to. MAKEOBJDIRPREFIX must be defined in an environment variable, and not on the command line, or in /etc/make.conf. buildworld will warn if it is improperly defined. FORMAT: This file contains a list, in reverse chronological order, of major breakages in tracking -current. It is not guaranteed to be a complete list of such breakages, and only contains entries since September 23, 2011. If you need to see UPDATING entries from before that date, you will need to fetch an UPDATING file from an older FreeBSD release. Copyright information: Copyright 1998-2009 M. Warner Losh Redistribution, publication, translation and use, with or without modification, in full or in part, in any form or format of this document are permitted without further permission from the author. THIS DOCUMENT IS PROVIDED BY WARNER LOSH ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WARNER LOSH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Contact Warner Losh if you have any questions about your use of this document. diff --git a/contrib/one-true-awk/FIXES b/contrib/one-true-awk/FIXES index 8a2befccd624..a13ca50ccde0 100644 --- a/contrib/one-true-awk/FIXES +++ b/contrib/one-true-awk/FIXES @@ -1,1375 +1,70 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ -This file lists all bug fixes, changes, etc., made since the AWK book -was sent to the printers in August, 1987. +This file lists all bug fixes, changes, etc., made since the +second edition of the AWK book was published in September 2023. -July 27, 2021: - As per IEEE Std 1003.1-2008, -F "str" is now consistent with - -v FS="str" when str is null. Thanks to Warner Losh. +Oct 30, 2023: + multiple fixes and a minor code cleanup. + disabled utf-8 for non-multibyte locales, such as C or POSIX. + fixed a bad char * cast that causes incorrect results on big-endian + systems. also fixed an out-of-bounds read for empty CCL. + fixed a buffer overflow in substr with utf-8 strings. + many thanks to Todd C Miller. + -July 24, 2021: - Fix readrec's definition of a record. This fixes an issue - with NetBSD's RS regular expression support that can cause - an infinite read loop. Thanks to Miguel Pineiro Jr. +Sep 24, 2023: + fnematch and getrune have been overhauled to solve issues around + unicode FS and RS. also fixed gsub null match issue with unicode. + big thanks to Arnold Robbins. - Fix regular expression RS ^-anchoring. RS ^-anchoring needs to - know if it is reading the first record of a file. This change - restores a missing line that was overlooked when porting NetBSD's - RS regex functionality. Thanks to Miguel Pineiro Jr. +Sep 12, 2023: + Fixed a length error in u8_byte2char that set RSTART to + incorrect (cannot happen) value for EOL match(str, /$/). - Fix size computation in replace_repeat() for special case - REPEAT_WITH_Q. Thanks to Todd C. Miller. -February 15, 2021: - Small fix so that awk will compile again with g++. Thanks to - Arnold Robbins. +----------------------------------------------------------------- -January 06, 2021: - Fix a decision bug with trailing stuff in lib.c:is_valid_number - after recent changes. Thanks to Ozan Yigit. +[This entry is a summary, not a precise list of changes.] -December 18, 2020: - Fix problems converting inf and NaN values in lib.c:is_valid_number. - Enhance number to string conversion to do the right thing for - NaN and inf values. Things are now pretty much the same as in - gawk. (Found a gawk bug while we're at it.) Added a torture - test for these values. Thanks to Arnold Robbins. Allows closing - of PR #101. + Added --csv option to enable processing of comma-separated + values inputs. When --csv is enabled, fields are separated + by commas, fields may be quoted with " double quotes, fields + may contain embedded newlines. -December 15, 2020: - Merge PR #99, which gets the right header for strcasecmp. - Thanks to GitHub user michaelforney. + If no explicit separator argument is provided, split() uses + the setting of --csv to determine how fields are split. -December 8, 2020: - Merge PR #98: Disallow hex data. Allow only +nan, -nan, - +inf, -inf (case independent) to give NaN and infinity values. - Improve things so that string to double conversion is only - done once, yielding something of a speedup. This obviate - PR #95. Thanks to Arnold Robbins. + Strings may now contain UTF-8 code points (not necessarily + characters). Functions that operate on characters, like + length, substr, index, match, etc., use UTF-8, so the length + of a string of 3 emojis is 3, not 12 as it would be if bytes + were counted. -December 3, 2020: - Fix to argument parsing to avoid printing spurious newlines. - Thanks to Todd Miller. Merges PR #97. - -October 13, 2020: - Add casts before all the calls to malloc/calloc/realloc in order - to get it to compile with g++. Thanks to Arnold Robbins. - -August 16, 2020: - Additional fixes for DJGPP. Thanks to Eli Zaretskii for - the testing. - -August 7, 2020: - Merge PR #93, which adds casts to (void*) for debug prints - using the %p format specifier. Thanks to GitHub user YongHaoWu - ("Chris") for the fixes. - -August 4, 2020: - In run.c, use non-restartable multibyte routines to attain - portability to DJGPP. Should fix Issue 92. Thanks to Albert Wik - for the report and to Todd Miller for the suggested fix. - -July 30, 2020: - Merge PRs 88-91 which fix small bugs. Thanks to Todd Miller and - Tim van der Molen for the fixes. - - In order to make life easier, we move exclusively to bison - as the parser generator. - -July 2, 2020: - Merge PRs 85 and 86 which fix regressions. Thanks to - Tim van der Molen for the fixes. - -June 25, 2020: - Merge PRs 82 and 84. The latter fixes issue #83. Thanks to - Todd Miller and awkfan77. - -June 12, 2020: - Clear errno before calling errcheck to avoid any spurious errors - left over from previous calls that may have set it. Thanks to - Todd Miller for the fix, from PR #80. - - Fix Issue #78 by allowing \r to follow floating point numbers in - lib.c:is_number. Thanks to GitHub user ajcarr for the report - and to Arnold Robbins for the fix. - -June 5, 2020: - In fldbld(), make sure that inputFS is set before trying to - use it. Thanks to Steffen Nurpmeso - for the report. - -May 5, 2020: - Fix checks for compilers that can handle noreturn. Thanks to - GitHub user enh-google for pointing it out. Closes Issue #79. - -April 16, 2020: - Handle old compilers that don't support C11 (for noreturn). - Thanks to Arnold Robbins. - -April 5, 2020: - Use and noreturn instead of GCC attributes. - Thanks to GitHub user awkfan77. Closes PR #77. - -February 28, 2020: - More cleanups from Christos Zoulas: notably backslash continuation - inside strings removes the newline and a fix for RS = "^a". - Fix for address sanitizer-found problem. Thanks to GitHub user - enh-google. - -February 19, 2020: - More small cleanups from Christos Zoulas. - -February 18, 2020: - Additional cleanups from Christos Zoulas. It's no longer necessary - to use the -y flag to bison. - -February 6, 2020: - Additional small cleanups from Christos Zoulas. awk is now - a little more robust about reporting I/O errors upon exit. - -January 31, 2020: - Merge PR #70, which avoids use of variable length arrays. Thanks - to GitHub user michaelforney. Fix issue #60 ({0} in interval - expressions doesn't work). Also get all tests working again. - Thanks to Arnold Robbins. - -January 24, 2020: - A number of small cleanups from Christos Zoulas. Add the close - on exec flag to files/pipes opened for redirection; courtesy of - Arnold Robbins. - -January 19, 2020: - If POSIXLY_CORRECT is set in the environment, then sub and gsub - use POSIX rules for multiple backslashes. This fixes Issue #66, - while maintaining backwards compatibility. - -January 9, 2020: - Input/output errors on closing files are now fatal instead of - mere warnings. Thanks to Martijn Dekker . - -January 5, 2020: - Fix a bug in the concatentation of two string constants into - one done in the grammar. Fixes GitHub issue #61. Thanks - to GitHub user awkfan77 for pointing out the direction for - the fix. New test T.concat added to the test suite. - Fix a few memory leaks reported by valgrind, as well. - -December 27, 2019: - Fix a bug whereby a{0,3} could match four a's. Thanks to - "Anonymous AWK fan" for the report. - -December 11, 2019: - Further printf-related fixes for 32 bit systems. - Thanks again to Christos Zoulas. - -December 8, 2019: - Fix the return value of sprintf("%d") on 32 bit systems. - Thanks to Jim Lowe for the report and to Christos Zoulas - for the fix. - -November 10, 2019: - Convert a number of Boolean integer variables into - actual bools. Convert compile_time variable into an - enum and simplify some of the related code. Thanks - to Arnold Robbins. - -November 8, 2019: - Fix from Ori Bernstein to get UTF-8 characters instead of - bytes when FS = "". This is currently the only bit of - the One True Awk that understands multibyte characters. - From Arnold Robbins, apply some cleanups in the test suite. - -October 25, 2019: - More fixes and cleanups from NetBSD, courtesy of Christos - Zoulas. Merges PRs 54 and 55. - -October 24, 2019: - Import second round of code cleanups from NetBSD. Much thanks - to Christos Zoulas (GitHub user zoulasc). Merges PR 53. - Add an optimization for string concatenation, also from - Christos. - -October 17, 2019: - Import code cleanups from NetBSD. Much thanks to Christos - Zoulas (GitHub user zoulasc). Merges PR 51. - -October 6, 2019: - Import code from NetBSD awk that implements RS as a regular - expression. - -September 10, 2019: - Fixes for various array / memory overruns found via gcc's - -fsanitize=unknown. Thanks to Alexander Richardson (GitHub - user arichardson). Merges PRs 47 and 48. - -July 28, 2019: - Import grammar optimization from NetBSD: Two string constants - concatenated together get turned into a single string. - -July 26, 2019: - Support POSIX-specified C-style escape sequences "\a" (alarm) - and "\v" (vertical tab) in command line arguments and regular - expressions, further to the support for them in strings added on - Apr 9, 1989. These now no longer match as literal "a" and "v" - characters (as they don't on other awk implementations). - Thanks to Martijn Dekker. - -July 17, 2019: - Pull in a number of code cleanups and minor fixes from - Warner Losh's bsd-ota branch. The only user visible change - is the use of random(3) as the random number generator. - Thanks to Warner Losh for collecting all these fixes in - one easy place to get them from. - -July 16, 2019: - Fix field splitting to use FS value as of the time a record - was read or assigned to. Thanks to GitHub user Cody Mello (melloc) - for the fix. (Merged from his branch, via PR #42.) Updated - testdir/T.split per said PR as well. - -June 24, 2019: - Extract awktest.tar into testdir directory. Add some very - simple mechanics to the makefile for running the tests and - for cleaning up. No changes to awk itself. - -June 17, 2019: - Disallow deleting SYMTAB and its elements, which creates - use-after-free bugs. Thanks to GitHub user Cody Mello (melloc) - for the fix. (Merged from PR #43.) - -June 5, 2019: - Allow unmatched right parenthesis in a regular expression to - be treated literally. Fixes Issue #40. Thanks to GitHub user - Warner Losh (bsdimp) for the report. Thanks to Arnold Robbins - for the fix. - -May 29,2019: - Fix check for command line arguments to no longer require that - first character after '=' not be another '='. Reverts change of - August 11, 1989. Thanks to GitHub user Jamie Landeg Jones for - pointing out the issue; from Issue #38. - -Apr 7, 2019: - Update awktest.tar(p.50) to use modern options to sort. Needed - for Android development. Thanks to GitHub user mohd-akram (Mohamed - Akram). From Issue #33. - -Mar 12, 2019: - Added very simplistic support for cross-compiling in the - makefile. We are NOT going to go in the direction of the - autotools, though. Thanks to GitHub user nee-san for - the basic change. (Merged from PR #34.) - -Mar 5, 2019: - Added support for POSIX-standard interval expressions (a.k.a. - bounds, a.k.a. repetition expressions) in regular expressions, - backported (via NetBSD) from Apple awk-24 (20070501). - Thanks to Martijn Dekker for the port. - (Merged from PR #30.) - -Mar 3, 2019: - Merge PRs as follows: - #12: Avoid undefined behaviour when using ctype(3) functions in - relex(). Thanks to GitHub user iamleot. - #31: Make getline handle numeric strings, and update FIXES. Thanks - to GitHub user arnoldrobbins. - #32: maketab: support build systems with read-only source. Thanks - to GitHub user enh. - -Jan 25, 2019: - Make getline handle numeric strings properly in all cases. - (Thanks, Arnold.) - -Jan 21, 2019: - Merged a number of small fixes from GitHub pull requests. - Thanks to GitHub users Arnold Robbins (arnoldrobbins), - Cody Mello (melloc) and Christoph Junghans (junghans). - PR numbers: 13-21, 23, 24, 27. - -Oct 25, 2018: - Added test in maketab.c to prevent generating a proctab entry - for YYSTYPE_IS_DEFINED. It was harmless but some gcc settings - generated a warning message. Thanks to Nan Xiao for report. - -Aug 27, 2018: - Disallow '$' in printf formats; arguments evaluated in order - and printed in order. - - Added some casts to silence warnings on debugging printfs. - (Thanks, Arnold.) - -Aug 23, 2018: - A long list of fixes courtesy of Arnold Robbins, - to whom profound thanks. - - 1. ofs-rebuild: OFS value used to rebuild the record was incorrect. - Fixed August 19, 2014. Revised fix August 2018. - - 2. system-status: Instead of a floating-point division by 256, use - the wait(2) macros to create a reasonable exit status. - Fixed March 12, 2016. - - 3. space: Use provided xisblank() function instead of ispace() for - matching [[:blank:]]. - - 4. a-format: Add POSIX standard %a and %A to supported formats. Check - at runtime that this format is available. - - 5. decr-NF: Decrementing NF did not change $0. This is a decades-old - bug. There are interactions with the old and new value of OFS as well. - Most of the fix came from the NetBSD awk. - - 6. string-conv: String conversions of scalars were sticky. Once a - conversion to string happened, even with OFMT, that value was used until - a new numeric value was assigned, even if OFMT differed from CONVFMT, - and also if CONVFMT changed. - - 7. unary-plus: Unary plus on a string constant returned the string. - Instead, it should convert the value to numeric and give that value. - - Also added Arnold's tests for these to awktest.tar as T.arnold. - -Aug 15, 2018: - fixed mangled awktest.tar (thanks, Arnold), posted all - current (very minor) fixes to github / onetrueawk - -Jun 7, 2018: - (yes, a long layoff) - Updated some broken tests (beebe.tar, T.lilly) - [thanks to Arnold Robbins] - -Mar 26, 2015: - buffer overflow in error reporting; thanks to tobias ulmer - and john-mark gurney for spotting it and the fix. - -Feb 4, 2013: - cleaned up a handful of tests that didn't seem to actually - test for correct behavior: T.latin1, T.gawk. - -Jan 5, 2013: - added ,NULL initializer to static Cells in run.c; not really - needed but cleaner. Thanks to Michael Bombardieri. - -Dec 20, 2012: - fiddled makefile to get correct yacc and bison flags. pick yacc - (linux) or bison (mac) as necessary. - - added __attribute__((__noreturn__)) to a couple of lines in - proto.h, to silence someone's enthusiastic checker. - - fixed obscure call by value bug in split(a[1],a) reported on - 9fans. the management of temporary values is just a mess; i - took a shortcut by making an extra string copy. thanks - to paul patience and arnold robbins for passing it on and for - proposed patches. - - tiny fiddle in setfval to eliminate -0 results in T.expr, which - has irritated me for 20+ years. - -Aug 10, 2011: - another fix to avoid core dump with delete(ARGV); again, many thanks - to ruslan ermilov. - -Aug 7, 2011: - split(s, a, //) now behaves the same as split(s, a, "") - -Jun 12, 2011: - /pat/, \n /pat/ {...} is now legal, though bad style to use. - - added checks to new -v code that permits -vnospace; thanks to - ruslan ermilov for spotting this and providing the patch. - - removed fixed limit on number of open files; thanks to aleksey - cheusov and christos zoulos. - - fixed day 1 bug that resurrected deleted elements of ARGV when - used as filenames (in lib.c). - - minor type fiddles to make gcc -Wall -pedantic happier (but not - totally so); turned on -fno-strict-aliasing in makefile. - -May 6, 2011: - added #ifdef for isblank. - now allows -ffoo as well as -f foo arguments. - (thanks, ruslan) - -May 1, 2011: - after advice from todd miller, kevin lo, ruslan ermilov, - and arnold robbins, changed srand() to return the previous - seed (which is 1 on the first call of srand). the seed is - an Awkfloat internally though converted to unsigned int to - pass to the library srand(). thanks, everyone. - - fixed a subtle (and i hope low-probability) overflow error - in fldbld, by adding space for one extra \0. thanks to - robert bassett for spotting this one and providing a fix. - - removed the files related to compilation on windows. i no - longer have anything like a current windows environment, so - i can't test any of it. - -May 23, 2010: - fixed long-standing overflow bug in run.c; many thanks to - nelson beebe for spotting it and providing the fix. - - fixed bug that didn't parse -vd=1 properly; thanks to santiago - vila for spotting it. - -Feb 8, 2010: - i give up. replaced isblank with isspace in b.c; there are - no consistent header files. - -Nov 26, 2009: - fixed a long-standing issue with when FS takes effect. a - change to FS is now noticed immediately for subsequent splits. - - changed the name getline() to awkgetline() to avoid yet another - name conflict somewhere. - -Feb 11, 2009: - temporarily for now defined HAS_ISBLANK, since that seems to - be the best way through the thicket. isblank arrived in C99, - but seems to be arriving at different systems at different - times. - -Oct 8, 2008: - fixed typo in b.c that set tmpvec wrongly. no one had ever - run into the problem, apparently. thanks to alistair crooks. - -Oct 23, 2007: - minor fix in lib.c: increase inputFS to 100, change malloc - for fields to n+1. - - fixed memory fault caused by out of order test in setsval. - - thanks to david o'brien, freebsd, for both fixes. - -May 1, 2007: - fiddle in makefile to fix for BSD make; thanks to igor sobrado. - -Mar 31, 2007: - fixed some null pointer refs calling adjbuf. - -Feb 21, 2007: - fixed a bug in matching the null RE in sub and gsub. thanks to al aho - who actually did the fix (in b.c), and to wolfgang seeberg for finding - it and providing a very compact test case. - - fixed quotation in b.c; thanks to Hal Pratt and the Princeton Dante - Project. - - removed some no-effect asserts in run.c. - - fiddled maketab.c to not complain about bison-generated values. - - removed the obsolete -V argument; fixed --version to print the - version and exit. - - fixed wording and an outright error in the usage message; thanks to igor - sobrado and jason mcintyre. - - fixed a bug in -d that caused core dump if no program followed. - -Jan 1, 2007: - dropped mac.code from makefile; there are few non-MacOSX - mac's these days. - -Jan 17, 2006: - system() not flagged as unsafe in the unadvertised -safe option. - found it while enhancing tests before shipping the ;login: article. - practice what you preach. - - removed the 9-years-obsolete -mr and -mf flags. - - added -version and --version options. - - core dump on linux with BEGIN {nextfile}, now fixed. - - removed some #ifdef's in run.c and lex.c that appear to no - longer be necessary. - -Apr 24, 2005: - modified lib.c so that values of $0 et al are preserved in the END - block, apparently as required by posix. thanks to havard eidnes - for the report and code. - -Jan 14, 2005: - fixed infinite loop in parsing, originally found by brian tsang. - thanks to arnold robbins for a suggestion that started me - rethinking it. - -Dec 31, 2004: - prevent overflow of -f array in main, head off potential error in - call of SYNTAX(), test malloc return in lib.c, all with thanks to - todd miller. - -Dec 22, 2004: - cranked up size of NCHARS; coverity thinks it can be overrun with - smaller size, and i think that's right. added some assertions to b.c - to catch places where it might overrun. the RE code is still fragile. - -Dec 5, 2004: - fixed a couple of overflow problems with ridiculous field numbers: - e.g., print $(2^32-1). thanks to ruslan ermilov, giorgos keramidas - and david o'brien at freebsd.org for patches. this really should - be re-done from scratch. - -Nov 21, 2004: - fixed another 25-year-old RE bug, in split. it's another failure - to (re-)initialize. thanks to steve fisher for spotting this and - providing a good test case. - -Nov 22, 2003: - fixed a bug in regular expressions that dates (so help me) from 1977; - it's been there from the beginning. an anchored longest match that - was longer than the number of states triggered a failure to initialize - the machine properly. many thanks to moinak ghosh for not only finding - this one but for providing a fix, in some of the most mysterious - code known to man. - - fixed a storage leak in call() that appears to have been there since - 1983 or so -- a function without an explicit return that assigns a - string to a parameter leaked a Cell. thanks to moinak ghosh for - spotting this very subtle one. - -Jul 31, 2003: - fixed, thanks to andrey chernov and ruslan ermilov, a bug in lex.c - that mis-handled the character 255 in input. (it was being compared - to EOF with a signed comparison.) - -Jul 29, 2003: - fixed (i think) the long-standing botch that included the beginning of - line state ^ for RE's in the set of valid characters; this led to a - variety of odd problems, including failure to properly match certain - regular expressions in non-US locales. thanks to ruslan for keeping - at this one. - -Jul 28, 2003: - n-th try at getting internationalization right, with thanks to volker - kiefel, arnold robbins and ruslan ermilov for advice, though they - should not be blamed for the outcome. according to posix, "." is the - radix character in programs and command line arguments regardless of - the locale; otherwise, the locale should prevail for input and output - of numbers. so it's intended to work that way. - - i have rescinded the attempt to use strcoll in expanding shorthands in - regular expressions (cclenter). its properties are much too - surprising; for example [a-c] matches aAbBc in locale en_US but abBcC - in locale fr_CA. i can see how this might arise by implementation - but i cannot explain it to a human user. (this behavior can be seen - in gawk as well; we're leaning on the same library.) - - the issue appears to be that strcoll is meant for sorting, where - merging upper and lower case may make sense (though note that unix - sort does not do this by default either). it is not appropriate - for regular expressions, where the goal is to match specific - patterns of characters. in any case, the notations [:lower:], etc., - are available in awk, and they are more likely to work correctly in - most locales. - - a moratorium is hereby declared on internationalization changes. - i apologize to friends and colleagues in other parts of the world. - i would truly like to get this "right", but i don't know what - that is, and i do not want to keep making changes until it's clear. - -Jul 4, 2003: - fixed bug that permitted non-terminated RE, as in "awk /x". - -Jun 1, 2003: - subtle change to split: if source is empty, number of elems - is always 0 and the array is not set. - -Mar 21, 2003: - added some parens to isblank, in another attempt to make things - internationally portable. - -Mar 14, 2003: - the internationalization changes, somewhat modified, are now - reinstated. in theory awk will now do character comparisons - and case conversions in national language, but "." will always - be the decimal point separator on input and output regardless - of national language. isblank(){} has an #ifndef. - - this no longer compiles on windows: LC_MESSAGES isn't defined - in vc6++. - - fixed subtle behavior in field and record splitting: if FS is - a single character and RS is not empty, \n is NOT a separator. - this tortuous reading is found in the awk book; behavior now - matches gawk and mawk. - -Dec 13, 2002: - for the moment, the internationalization changes of nov 29 are - rolled back -- programs like x = 1.2 don't work in some locales, - because the parser is expecting x = 1,2. until i understand this - better, this will have to wait. - -Nov 29, 2002: - modified b.c (with tiny changes in main and run) to support - locales, using strcoll and iswhatever tests for posix character - classes. thanks to ruslan ermilov (ru@freebsd.org) for code. - the function isblank doesn't seem to have propagated to any - header file near me, so it's there explicitly. not properly - tested on non-ascii character sets by me. - -Jun 28, 2002: - modified run/format() and tran/getsval() to do a slightly better - job on using OFMT for output from print and CONVFMT for other - number->string conversions, as promised by posix and done by - gawk and mawk. there are still places where it doesn't work - right if CONVFMT is changed; by then the STR attribute of the - variable has been irrevocably set. thanks to arnold robbins for - code and examples. - - fixed subtle bug in format that could get core dump. thanks to - Jaromir Dolecek for finding and fixing. - minor cleanup in run.c / format() at the same time. - - added some tests for null pointers to debugging printf's, which - were never intended for external consumption. thanks to dave - kerns (dkerns@lucent.com) for pointing this out. - - GNU compatibility: an empty regexp matches anything (thanks to - dag-erling smorgrav, des@ofug.org). subject to reversion if - this does more harm than good. - - pervasive small changes to make things more const-correct, as - reported by gcc's -Wwrite-strings. as it says in the gcc manual, - this may be more nuisance than useful. provoked by a suggestion - and code from arnaud desitter, arnaud@nimbus.geog.ox.ac.uk - - minor documentation changes to note that this now compiles out - of the box on Mac OS X. - -Feb 10, 2002: - changed types in posix chars structure to quiet solaris cc. - -Jan 1, 2002: - fflush() or fflush("") flushes all files and pipes. - - length(arrayname) returns number of elements; thanks to - arnold robbins for suggestion. - - added a makefile.win to make it easier to build on windows. - based on dan allen's buildwin.bat. - -Nov 16, 2001: - added support for posix character class names like [:digit:], - which are not exactly shorter than [0-9] and perhaps no more - portable. thanks to dag-erling smorgrav for code. - -Feb 16, 2001: - removed -m option; no longer needed, and it was actually - broken (noted thanks to volker kiefel). - -Feb 10, 2001: - fixed an appalling bug in gettok: any sequence of digits, +,-, E, e, - and period was accepted as a valid number if it started with a period. - this would never have happened with the lex version. - - other 1-character botches, now fixed, include a bare $ and a - bare " at the end of the input. - -Feb 7, 2001: - more (const char *) casts in b.c and tran.c to silence warnings. - -Nov 15, 2000: - fixed a bug introduced in august 1997 that caused expressions - like $f[1] to be syntax errors. thanks to arnold robbins for - noticing this and providing a fix. - -Oct 30, 2000: - fixed some nextfile bugs: not handling all cases. thanks to - arnold robbins for pointing this out. new regressions added. - - close() is now a function. it returns whatever the library - fclose returns, and -1 for closing a file or pipe that wasn't - opened. - -Sep 24, 2000: - permit \n explicitly in character classes; won't work right - if comes in as "[\n]" but ok as /[\n]/, because of multiple - processing of \'s. thanks to arnold robbins. - -July 5, 2000: - minor fiddles in tran.c to keep compilers happy about uschar. - thanks to norman wilson. - -May 25, 2000: - yet another attempt at making 8-bit input work, with another - band-aid in b.c (member()), and some (uschar) casts to head - off potential errors in subscripts (like isdigit). also - changed HAT to NCHARS-2. thanks again to santiago vila. - - changed maketab.c to ignore apparently out of range definitions - instead of halting; new freeBSD generates one. thanks to - jon snader for pointing out the problem. - -May 2, 2000: - fixed an 8-bit problem in b.c by making several char*'s into - unsigned char*'s. not clear i have them all yet. thanks to - Santiago Vila for the bug report. - -Apr 21, 2000: - finally found and fixed a memory leak in function call; it's - been there since functions were added ~1983. thanks to - jon bentley for the test case that found it. - - added test in envinit to catch environment "variables" with - names beginning with '='; thanks to Berend Hasselman. - -Jul 28, 1999: - added test in defn() to catch function foo(foo), which - otherwise recurses until core dump. thanks to arnold - robbins for noticing this. - -Jun 20, 1999: - added *bp in gettok in lex.c; appears possible to exit function - without terminating the string. thanks to russ cox. - -Jun 2, 1999: - added function stdinit() to run to initialize files[] array, - in case stdin, etc., are not constants; some compilers care. - -May 10, 1999: - replaced the ERROR ... FATAL, etc., macros with functions - based on vprintf, to avoid problems caused by overrunning - fixed-size errbuf array. thanks to ralph corderoy for the - impetus, and for pointing out a string termination bug in - qstring as well. - -Apr 21, 1999: - fixed bug that caused occasional core dumps with commandline - variable with value ending in \. (thanks to nelson beebe for - the test case.) - -Apr 16, 1999: - with code kindly provided by Bruce Lilly, awk now parses - /=/ and similar constructs more sensibly in more places. - Bruce also provided some helpful test cases. - -Apr 5, 1999: - changed true/false to True/False in run.c to make it - easier to compile with C++. Added some casts on malloc - and realloc to be honest about casts; ditto. changed - ltype int to long in struct rrow to reduce some 64-bit - complaints; other changes scattered throughout for the - same purpose. thanks to Nelson Beebe for these portability - improvements. - - removed some horrible pointer-int casting in b.c and elsewhere - by adding ptoi and itonp to localize the casts, which are - all benign. fixed one incipient bug that showed up on sgi - in 64-bit mode. - - reset lineno for new source file; include filename in error - message. also fixed line number error in continuation lines. - (thanks to Nelson Beebe for both of these.) - -Mar 24, 1999: - Nelson Beebe notes that irix 5.3 yacc dies with a bogus - error; use a newer version or switch to bison, since sgi - is unlikely to fix it. - -Mar 5, 1999: - changed isnumber to is_number to avoid the problem caused by - versions of ctype.h that include the name isnumber. - - distribution now includes a script for building on a Mac, - thanks to Dan Allen. - -Feb 20, 1999: - fixed memory leaks in run.c (call) and tran.c (setfval). - thanks to Stephen Nutt for finding these and providing the fixes. - -Jan 13, 1999: - replaced srand argument by (unsigned int) in run.c; - avoids problem on Mac and potentially on Unix & Windows. - thanks to Dan Allen. - - added a few (int) casts to silence useless compiler warnings. - e.g., errorflag= in run.c jump(). - - added proctab.c to the bundle outout; one less thing - to have to compile out of the box. - - added calls to _popen and _pclose to the win95 stub for - pipes (thanks to Steve Adams for this helpful suggestion). - seems to work, though properties are not well understood - by me, and it appears that under some circumstances the - pipe output is truncated. Be careful. - -Oct 19, 1998: - fixed a couple of bugs in getrec: could fail to update $0 - after a getline var; because inputFS wasn't initialized, - could split $0 on every character, a misleading diversion. - - fixed caching bug in makedfa: LRU was actually removing - least often used. - - thanks to ross ridge for finding these, and for providing - great bug reports. - -May 12, 1998: - fixed potential bug in readrec: might fail to update record - pointer after growing. thanks to dan levy for spotting this - and suggesting the fix. - -Mar 12, 1998: - added -V to print version number and die. - -[notify dave kerns, dkerns@dacsoup.ih.lucent.com] - -Feb 11, 1998: - subtle silent bug in lex.c: if the program ended with a number - longer than 1 digit, part of the input would be pushed back and - parsed again because token buffer wasn't terminated right. - example: awk 'length($0) > 10'. blush. at least i found it - myself. - -Aug 31, 1997: - s/adelete/awkdelete/: SGI uses this in malloc.h. - thanks to nelson beebe for pointing this one out. - -Aug 21, 1997: - fixed some bugs in sub and gsub when replacement includes \\. - this is a dark, horrible corner, but at least now i believe that - the behavior is the same as gawk and the intended posix standard. - thanks to arnold robbins for advice here. - -Aug 9, 1997: - somewhat regretfully, replaced the ancient lex-based lexical - analyzer with one written in C. it's longer, generates less code, - and more portable; the old one depended too much on mysterious - properties of lex that were not preserved in other environments. - in theory these recognize the same language. - - now using strtod to test whether a string is a number, instead of - the convoluted original function. should be more portable and - reliable if strtod is implemented right. - - removed now-pointless optimization in makefile that tries to avoid - recompilation when awkgram.y is changed but symbols are not. - - removed most fixed-size arrays, though a handful remain, some - of which are unchecked. you have been warned. - -Aug 4, 1997: - with some trepidation, replaced the ancient code that managed - fields and $0 in fixed-size arrays with arrays that grow on - demand. there is still some tension between trying to make this - run fast and making it clean; not sure it's right yet. - - the ill-conceived -mr and -mf arguments are now useful only - for debugging. previous dynamic string code removed. - - numerous other minor cleanups along the way. - -Jul 30, 1997: - using code provided by dan levy (to whom profuse thanks), replaced - fixed-size arrays and awkward kludges by a fairly uniform mechanism - to grow arrays as needed for printf, sub, gsub, etc. - -Jul 23, 1997: - falling off the end of a function returns "" and 0, not 0. - thanks to arnold robbins. - -Jun 17, 1997: - replaced several fixed-size arrays by dynamically-created ones - in run.c; added overflow tests to some previously unchecked cases. - getline, toupper, tolower. - - getline code is still broken in that recursive calls may wind - up using the same space. [fixed later] - - increased RECSIZE to 8192 to push problems further over the horizon. - - added \r to \n as input line separator for programs, not data. - damn CRLFs. - - modified format() to permit explicit printf("%c", 0) to include - a null byte in output. thanks to ken stailey for the fix. - - added a "-safe" argument that disables file output (print >, - print >>), process creation (cmd|getline, print |, system), and - access to the environment (ENVIRON). this is a first approximation - to a "safe" version of awk, but don't rely on it too much. thanks - to joan feigenbaum and matt blaze for the inspiration long ago. - -Jul 8, 1996: - fixed long-standing bug in sub, gsub(/a/, "\\\\&"); thanks to - ralph corderoy. - -Jun 29, 1996: - fixed awful bug in new field splitting; didn't get all the places - where input was done. - -Jun 28, 1996: - changed field-splitting to conform to posix definition: fields are - split using the value of FS at the time of input; it used to be - the value when the field or NF was first referred to, a much less - predictable definition. thanks to arnold robbins for encouragement - to do the right thing. - -May 28, 1996: - fixed appalling but apparently unimportant bug in parsing octal - numbers in reg exprs. - - explicit hex in reg exprs now limited to 2 chars: \xa, \xaa. - -May 27, 1996: - cleaned up some declarations so gcc -Wall is now almost silent. - - makefile now includes backup copies of ytab.c and lexyy.c in case - one makes before looking; it also avoids recreating lexyy.c unless - really needed. - - s/aprintf/awkprint, s/asprintf/awksprintf/ to avoid some name clashes - with unwisely-written header files. - - thanks to jeffrey friedl for several of these. - -May 26, 1996: - an attempt to rationalize the (unsigned) char issue. almost all - instances of unsigned char have been removed; the handful of places - in b.c where chars are used as table indices have been hand-crafted. - added some latin-1 tests to the regression, but i'm not confident; - none of my compilers seem to care much. thanks to nelson beebe for - pointing out some others that do care. - -May 2, 1996: - removed all register declarations. - - enhanced split(), as in gawk, etc: split(s, a, "") splits s into - a[1]...a[length(s)] with each character a single element. - - made the same changes for field-splitting if FS is "". - - added nextfile, as in gawk: causes immediate advance to next - input file. (thanks to arnold robbins for inspiration and code). - - small fixes to regexpr code: can now handle []], [[], and - variants; [] is now a syntax error, rather than matching - everything; [z-a] is now empty, not z. far from complete - or correct, however. (thanks to jeffrey friedl for pointing out - some awful behaviors.) - -Apr 29, 1996: - replaced uchar by uschar everywhere; apparently some compilers - usurp this name and this causes conflicts. - - fixed call to time in run.c (bltin); arg is time_t *. - - replaced horrible pointer/long punning in b.c by a legitimate - union. should be safer on 64-bit machines and cleaner everywhere. - (thanks to nelson beebe for pointing out some of these problems.) - - replaced nested comments by #if 0...#endif in run.c, lib.c. - - removed getsval, setsval, execute macros from run.c and lib.c. - machines are 100x faster than they were when these macros were - first used. - - revised filenames: awk.g.y => awkgram.y, awk.lx.l => awklex.l, - y.tab.[ch] => ytab.[ch], lex.yy.c => lexyy.c, all in the aid of - portability to nameless systems. - - "make bundle" now includes yacc and lex output files for recipients - who don't have yacc or lex. - -Aug 15, 1995: - initialized Cells in setsymtab more carefully; some fields - were not set. (thanks to purify, all of whose complaints i - think i now understand.) - - fixed at least one error in gsub that looked at -1-th element - of an array when substituting for a null match (e.g., $). - - delete arrayname is now legal; it clears the elements but leaves - the array, which may not be the right behavior. - - modified makefile: my current make can't cope with the test used - to avoid unnecessary yacc invocations. - -Jul 17, 1995: - added dynamically growing strings to awk.lx.l and b.c - to permit regular expressions to be much bigger. - the state arrays can still overflow. - -Aug 24, 1994: - detect duplicate arguments in function definitions (mdm). - -May 11, 1994: - trivial fix to printf to limit string size in sub(). - -Apr 22, 1994: - fixed yet another subtle self-assignment problem: - $1 = $2; $1 = $1 clobbered $1. - - Regression tests now use private echo, to avoid quoting problems. - -Feb 2, 1994: - changed error() to print line number as %d, not %g. - -Jul 23, 1993: - cosmetic changes: increased sizes of some arrays, - reworded some error messages. - - added CONVFMT as in posix (just replaced OFMT in getsval) - - FILENAME is now "" until the first thing that causes a file - to be opened. - -Nov 28, 1992: - deleted yyunput and yyoutput from proto.h; - different versions of lex give these different declarations. - -May 31, 1992: - added -mr N and -mf N options: more record and fields. - these really ought to adjust automatically. - - cleaned up some error messages; "out of space" now means - malloc returned NULL in all cases. - - changed rehash so that if it runs out, it just returns; - things will continue to run slow, but maybe a bit longer. - -Apr 24, 1992: - remove redundant close of stdin when using -f -. - - got rid of core dump with -d; awk -d just prints date. - -Apr 12, 1992: - added explicit check for /dev/std(in,out,err) in redirection. - unlike gawk, no /dev/fd/n yet. - - added (file/pipe) builtin. hard to test satisfactorily. - not posix. - -Feb 20, 1992: - recompile after abortive changes; should be unchanged. - -Dec 2, 1991: - die-casting time: converted to ansi C, installed that. - -Nov 30, 1991: - fixed storage leak in freefa, failing to recover [N]CCL. - thanks to Bill Jones (jones@cs.usask.ca) - -Nov 19, 1991: - use RAND_MAX instead of literal in builtin(). - -Nov 12, 1991: - cranked up some fixed-size arrays in b.c, and added a test for - overflow in penter. thanks to mark larsen. - -Sep 24, 1991: - increased buffer in gsub. a very crude fix to a general problem. - and again on Sep 26. - -Aug 18, 1991: - enforce variable name syntax for commandline variables: has to - start with letter or _. - -Jul 27, 1991: - allow newline after ; in for statements. - -Jul 21, 1991: - fixed so that in self-assignment like $1=$1, side effects - like recomputing $0 take place. (this is getting subtle.) - -Jun 30, 1991: - better test for detecting too-long output record. - -Jun 2, 1991: - better defense against very long printf strings. - made break and continue illegal outside of loops. - -May 13, 1991: - removed extra arg on gettemp, tempfree. minor error message rewording. - -May 6, 1991: - fixed silly bug in hex parsing in hexstr(). - removed an apparently unnecessary test in isnumber(). - warn about weird printf conversions. - fixed unchecked array overwrite in relex(). - - changed for (i in array) to access elements in sorted order. - then unchanged it -- it really does run slower in too many cases. - left the code in place, commented out. - -Feb 10, 1991: - check error status on all writes, to avoid banging on full disks. - -Jan 28, 1991: - awk -f - reads the program from stdin. - -Jan 11, 1991: - failed to set numeric state on $0 in cmd|getline context in run.c. - -Nov 2, 1990: - fixed sleazy test for integrality in getsval; use modf. - -Oct 29, 1990: - fixed sleazy buggy code in lib.c that looked (incorrectly) for - too long input lines. - -Oct 14, 1990: - fixed the bug on p. 198 in which it couldn't deduce that an - argument was an array in some contexts. replaced the error - message in intest() by code that damn well makes it an array. - -Oct 8, 1990: - fixed horrible bug: types and values were not preserved in - some kinds of self-assignment. (in assign().) - -Aug 24, 1990: - changed NCHARS to 256 to handle 8-bit characters in strings - presented to match(), etc. - -Jun 26, 1990: - changed struct rrow (awk.h) to use long instead of int for lval, - since cfoll() stores a pointer in it. now works better when int's - are smaller than pointers! - -May 6, 1990: - AVA fixed the grammar so that ! is uniformly of the same precedence as - unary + and -. This renders illegal some constructs like !x=y, which - now has to be parenthesized as !(x=y), and makes others work properly: - !x+y is (!x)+y, and x!y is x !y, not two pattern-action statements. - (These problems were pointed out by Bob Lenk of Posix.) - - Added \x to regular expressions (already in strings). - Limited octal to octal digits; \8 and \9 are not octal. - Centralized the code for parsing escapes in regular expressions. - Added a bunch of tests to T.re and T.sub to verify some of this. - -Feb 9, 1990: - fixed null pointer dereference bug in main.c: -F[nothing]. sigh. - - restored srand behavior: it returns the current seed. - -Jan 18, 1990: - srand now returns previous seed value (0 to start). - -Jan 5, 1990: - fix potential problem in tran.c -- something was freed, - then used in freesymtab. - -Oct 18, 1989: - another try to get the max number of open files set with - relatively machine-independent code. - - small fix to input() in case of multiple reads after EOF. - -Oct 11, 1989: - FILENAME is now defined in the BEGIN block -- too many old - programs broke. - - "-" means stdin in getline as well as on the commandline. - - added a bunch of casts to the code to tell the truth about - char * vs. unsigned char *, a right royal pain. added a - setlocale call to the front of main, though probably no one - has it usefully implemented yet. - -Aug 24, 1989: - removed redundant relational tests against nullnode if parse - tree already had a relational at that point. - -Aug 11, 1989: - fixed bug: commandline variable assignment has to look like - var=something. (consider the man page for =, in file =.1) - - changed number of arguments to functions to static arrays - to avoid repeated malloc calls. - -Aug 2, 1989: - restored -F (space) separator - -Jul 30, 1989: - added -v x=1 y=2 ... for immediate commandline variable assignment; - done before the BEGIN block for sure. they have to precede the - program if the program is on the commandline. - Modified Aug 2 to require a separate -v for each assignment. - -Jul 10, 1989: - fixed ref-thru-zero bug in environment code in tran.c - -Jun 23, 1989: - add newline to usage message. - -Jun 14, 1989: - added some missing ansi printf conversion letters: %i %X %E %G. - no sensible meaning for h or L, so they may not do what one expects. - - made %* conversions work. - - changed x^y so that if n is a positive integer, it's done - by explicit multiplication, thus achieving maximum accuracy. - (this should be done by pow() but it seems not to be locally.) - done to x ^= y as well. - -Jun 4, 1989: - ENVIRON array contains environment: if shell variable V=thing, - ENVIRON["V"] is "thing" - - multiple -f arguments permitted. error reporting is naive. - (they were permitted before, but only the last was used.) - - fixed a really stupid botch in the debugging macro dprintf - - fixed order of evaluation of commandline assignments to match - what the book claims: an argument of the form x=e is evaluated - at the time it would have been opened if it were a filename (p 63). - this invalidates the suggested answer to ex 4-1 (p 195). - - removed some code that permitted -F (space) fieldseparator, - since it didn't quite work right anyway. (restored aug 2) - -Apr 27, 1989: - Line number now accumulated correctly for comment lines. - -Apr 26, 1989: - Debugging output now includes a version date, - if one compiles it into the source each time. - -Apr 9, 1989: - Changed grammar to prohibit constants as 3rd arg of sub and gsub; - prevents class of overwriting-a-constant errors. (Last one?) - This invalidates the "banana" example on page 43 of the book. - - Added \a ("alert"), \v (vertical tab), \xhhh (hexadecimal), - as in ANSI, for strings. Rescinded the sloppiness that permitted - non-octal digits in \ooo. Warning: not all compilers and libraries - will be able to deal with \x correctly. - -Jan 9, 1989: - Fixed bug that caused tempcell list to contain a duplicate. - The fix is kludgy. - -Dec 17, 1988: - Catches some more commandline errors in main. - Removed redundant decl of modf in run.c (confuses some compilers). - Warning: there's no single declaration of malloc, etc., in awk.h - that seems to satisfy all compilers. - -Dec 7, 1988: - Added a bit of code to error printing to avoid printing nulls. - (Not clear that it actually would.) - -Nov 27, 1988: - With fear and trembling, modified the grammar to permit - multiple pattern-action statements on one line without - an explicit separator. By definition, this capitulation - to the ghost of ancient implementations remains undefined - and thus subject to change without notice or apology. - DO NOT COUNT ON IT. - -Oct 30, 1988: - Fixed bug in call() that failed to recover storage. - - A warning is now generated if there are more arguments - in the call than in the definition (in lieu of fixing - another storage leak). - -Oct 20, 1988: - Fixed %c: if expr is numeric, use numeric value; - otherwise print 1st char of string value. still - doesn't work if the value is 0 -- won't print \0. - - Added a few more checks for running out of malloc. - -Oct 12, 1988: - Fixed bug in call() that freed local arrays twice. - - Fixed to handle deletion of non-existent array right; - complains about attempt to delete non-array element. - -Sep 30, 1988: - Now guarantees to evaluate all arguments of built-in - functions, as in C; the appearance is that arguments - are evaluated before the function is called. Places - affected are sub (gsub was ok), substr, printf, and - all the built-in arithmetic functions in bltin(). - A warning is generated if a bltin() is called with - the wrong number of arguments. - - This requires changing makeprof on p167 of the book. - -Aug 23, 1988: - setting FILENAME in BEGIN caused core dump, apparently - because it was freeing space not allocated by malloc. - -July 24, 1988: - fixed egregious error in toupper/tolower functions. - still subject to rescinding, however. - -July 2, 1988: - flush stdout before opening file or pipe - -July 2, 1988: - performance bug in b.c/cgoto(): not freeing some sets of states. - partial fix only right now, and the number of states increased - to make it less obvious. - -June 1, 1988: - check error status on close - -May 28, 1988: - srand returns seed value it's using. - see 1/18/90 - -May 22, 1988: - Removed limit on depth of function calls. - -May 10, 1988: - Fixed lib.c to permit _ in commandline variable names. - -Mar 25, 1988: - main.c fixed to recognize -- as terminator of command- - line options. Illegal options flagged. - Error reporting slightly cleaned up. - -Dec 2, 1987: - Newer C compilers apply a strict scope rule to extern - declarations within functions. Two extern declarations in - lib.c and tran.c have been moved to obviate this problem. - -Oct xx, 1987: - Reluctantly added toupper and tolower functions. - Subject to rescinding without notice. - -Sep 17, 1987: - Error-message printer had printf(s) instead of - printf("%s",s); got core dumps when the message - included a %. - -Sep 12, 1987: - Very long printf strings caused core dump; - fixed aprintf, asprintf, format to catch them. - Can still get a core dump in printf itself. + Regular expressions are processes as UTF-8. + Unicode literals can be written as \u followed by one + to eight hexadecimal digits. These may appear in strings and + regular expressions. diff --git a/contrib/one-true-awk/FIXES b/contrib/one-true-awk/FIXES.1e similarity index 95% copy from contrib/one-true-awk/FIXES copy to contrib/one-true-awk/FIXES.1e index 8a2befccd624..8cbd6ac1a097 100644 --- a/contrib/one-true-awk/FIXES +++ b/contrib/one-true-awk/FIXES.1e @@ -1,1375 +1,1429 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ This file lists all bug fixes, changes, etc., made since the AWK book -was sent to the printers in August, 1987. +was sent to the printers in August 1987. + +Sep 06, 2023: + Fix edge case where FS is changed on commandline. Thanks to + Gordon Shephard and Miguel Pineiro Jr. + + Fix regular expression clobbering in the lexer, where lexer does + not make a copy of regexp literals. also makedfa memory leaks have + been plugged. Thanks to Miguel Pineiro Jr. + +Dec 15, 2022: + Force hex escapes in strings to be no more than two characters, + as they already are in regular expressions. This brings internal + consistency, as well as consistency with gawk. Thanks to + Arnold Robbins. + +Sep 12, 2022: + adjbuf minlen error (cannot be 0) in cat, resulting in NULL pbuf. + discovered by todd miller. also use-after-free issue with + tempfree in cat, thanks to Miguel Pineiro Jr and valgrind. + +Aug 30, 2022: + Various leaks and use-after-free issues plugged/fixed. + Thanks to Miguel Pineiro Jr. . + +May 23, 2022: + Memory leak when assigning a string to some of the built-in + variables. allocated string erroneously marked DONTFREE. + Thanks to Miguel Pineiro Jr. . + +Mar 14, 2022: + Historic bug: command-line "name=value" assignment had been + truncating its entry in ARGV. (circa 1989) Thanks to + Miguel Pineiro Jr. . + +Mar 3, 2022: + Fixed file management memory leak that appears to have been + there since the files array was first initialized with stdin, + stdout, and stderr (circa 1992). Thanks to Miguel Pineiro Jr. + . + +December 8, 2021: + The error handling in closefile and closeall was mangled. Long + standing warnings had been made fatal and some fatal errors went + undetected. Thanks to Miguel Pineiro Jr. . + +Nov 03, 2021: + getline accesses uninitialized data after getrec() + returns 0 on EOF and leaves the contents of buf unchanged. + Thanks to Volodymyr Gubarkov, and Todd C Miller. + +Oct 12, 2021: + The fix for #83 changed the code to insert 2 chars, but the + call to adjbuf just above it only allows for 1 char. This can + cause a heap buffer overflow. July 27, 2021: As per IEEE Std 1003.1-2008, -F "str" is now consistent with -v FS="str" when str is null. Thanks to Warner Losh. July 24, 2021: Fix readrec's definition of a record. This fixes an issue with NetBSD's RS regular expression support that can cause an infinite read loop. Thanks to Miguel Pineiro Jr. Fix regular expression RS ^-anchoring. RS ^-anchoring needs to know if it is reading the first record of a file. This change restores a missing line that was overlooked when porting NetBSD's RS regex functionality. Thanks to Miguel Pineiro Jr. Fix size computation in replace_repeat() for special case REPEAT_WITH_Q. Thanks to Todd C. Miller. February 15, 2021: Small fix so that awk will compile again with g++. Thanks to Arnold Robbins. January 06, 2021: Fix a decision bug with trailing stuff in lib.c:is_valid_number after recent changes. Thanks to Ozan Yigit. December 18, 2020: Fix problems converting inf and NaN values in lib.c:is_valid_number. Enhance number to string conversion to do the right thing for NaN and inf values. Things are now pretty much the same as in gawk. (Found a gawk bug while we're at it.) Added a torture test for these values. Thanks to Arnold Robbins. Allows closing of PR #101. December 15, 2020: Merge PR #99, which gets the right header for strcasecmp. Thanks to GitHub user michaelforney. December 8, 2020: Merge PR #98: Disallow hex data. Allow only +nan, -nan, +inf, -inf (case independent) to give NaN and infinity values. Improve things so that string to double conversion is only done once, yielding something of a speedup. This obviate PR #95. Thanks to Arnold Robbins. December 3, 2020: Fix to argument parsing to avoid printing spurious newlines. Thanks to Todd Miller. Merges PR #97. October 13, 2020: Add casts before all the calls to malloc/calloc/realloc in order to get it to compile with g++. Thanks to Arnold Robbins. August 16, 2020: Additional fixes for DJGPP. Thanks to Eli Zaretskii for the testing. August 7, 2020: Merge PR #93, which adds casts to (void*) for debug prints using the %p format specifier. Thanks to GitHub user YongHaoWu ("Chris") for the fixes. August 4, 2020: In run.c, use non-restartable multibyte routines to attain portability to DJGPP. Should fix Issue 92. Thanks to Albert Wik for the report and to Todd Miller for the suggested fix. July 30, 2020: Merge PRs 88-91 which fix small bugs. Thanks to Todd Miller and Tim van der Molen for the fixes. In order to make life easier, we move exclusively to bison as the parser generator. July 2, 2020: Merge PRs 85 and 86 which fix regressions. Thanks to Tim van der Molen for the fixes. June 25, 2020: Merge PRs 82 and 84. The latter fixes issue #83. Thanks to Todd Miller and awkfan77. June 12, 2020: Clear errno before calling errcheck to avoid any spurious errors left over from previous calls that may have set it. Thanks to Todd Miller for the fix, from PR #80. Fix Issue #78 by allowing \r to follow floating point numbers in lib.c:is_number. Thanks to GitHub user ajcarr for the report and to Arnold Robbins for the fix. June 5, 2020: In fldbld(), make sure that inputFS is set before trying to use it. Thanks to Steffen Nurpmeso for the report. May 5, 2020: Fix checks for compilers that can handle noreturn. Thanks to GitHub user enh-google for pointing it out. Closes Issue #79. April 16, 2020: Handle old compilers that don't support C11 (for noreturn). Thanks to Arnold Robbins. April 5, 2020: Use and noreturn instead of GCC attributes. Thanks to GitHub user awkfan77. Closes PR #77. February 28, 2020: More cleanups from Christos Zoulas: notably backslash continuation inside strings removes the newline and a fix for RS = "^a". Fix for address sanitizer-found problem. Thanks to GitHub user enh-google. February 19, 2020: More small cleanups from Christos Zoulas. February 18, 2020: Additional cleanups from Christos Zoulas. It's no longer necessary to use the -y flag to bison. February 6, 2020: Additional small cleanups from Christos Zoulas. awk is now a little more robust about reporting I/O errors upon exit. January 31, 2020: Merge PR #70, which avoids use of variable length arrays. Thanks to GitHub user michaelforney. Fix issue #60 ({0} in interval expressions doesn't work). Also get all tests working again. Thanks to Arnold Robbins. January 24, 2020: A number of small cleanups from Christos Zoulas. Add the close on exec flag to files/pipes opened for redirection; courtesy of Arnold Robbins. January 19, 2020: If POSIXLY_CORRECT is set in the environment, then sub and gsub use POSIX rules for multiple backslashes. This fixes Issue #66, while maintaining backwards compatibility. January 9, 2020: Input/output errors on closing files are now fatal instead of mere warnings. Thanks to Martijn Dekker . January 5, 2020: Fix a bug in the concatentation of two string constants into one done in the grammar. Fixes GitHub issue #61. Thanks to GitHub user awkfan77 for pointing out the direction for the fix. New test T.concat added to the test suite. Fix a few memory leaks reported by valgrind, as well. December 27, 2019: Fix a bug whereby a{0,3} could match four a's. Thanks to "Anonymous AWK fan" for the report. December 11, 2019: Further printf-related fixes for 32 bit systems. Thanks again to Christos Zoulas. December 8, 2019: Fix the return value of sprintf("%d") on 32 bit systems. Thanks to Jim Lowe for the report and to Christos Zoulas for the fix. November 10, 2019: Convert a number of Boolean integer variables into actual bools. Convert compile_time variable into an enum and simplify some of the related code. Thanks to Arnold Robbins. November 8, 2019: Fix from Ori Bernstein to get UTF-8 characters instead of bytes when FS = "". This is currently the only bit of the One True Awk that understands multibyte characters. From Arnold Robbins, apply some cleanups in the test suite. October 25, 2019: More fixes and cleanups from NetBSD, courtesy of Christos Zoulas. Merges PRs 54 and 55. October 24, 2019: Import second round of code cleanups from NetBSD. Much thanks to Christos Zoulas (GitHub user zoulasc). Merges PR 53. Add an optimization for string concatenation, also from Christos. October 17, 2019: Import code cleanups from NetBSD. Much thanks to Christos Zoulas (GitHub user zoulasc). Merges PR 51. October 6, 2019: Import code from NetBSD awk that implements RS as a regular expression. September 10, 2019: Fixes for various array / memory overruns found via gcc's -fsanitize=unknown. Thanks to Alexander Richardson (GitHub user arichardson). Merges PRs 47 and 48. July 28, 2019: Import grammar optimization from NetBSD: Two string constants concatenated together get turned into a single string. July 26, 2019: Support POSIX-specified C-style escape sequences "\a" (alarm) and "\v" (vertical tab) in command line arguments and regular expressions, further to the support for them in strings added on Apr 9, 1989. These now no longer match as literal "a" and "v" characters (as they don't on other awk implementations). Thanks to Martijn Dekker. July 17, 2019: Pull in a number of code cleanups and minor fixes from Warner Losh's bsd-ota branch. The only user visible change is the use of random(3) as the random number generator. Thanks to Warner Losh for collecting all these fixes in one easy place to get them from. July 16, 2019: Fix field splitting to use FS value as of the time a record was read or assigned to. Thanks to GitHub user Cody Mello (melloc) for the fix. (Merged from his branch, via PR #42.) Updated testdir/T.split per said PR as well. June 24, 2019: Extract awktest.tar into testdir directory. Add some very simple mechanics to the makefile for running the tests and for cleaning up. No changes to awk itself. June 17, 2019: Disallow deleting SYMTAB and its elements, which creates use-after-free bugs. Thanks to GitHub user Cody Mello (melloc) for the fix. (Merged from PR #43.) June 5, 2019: Allow unmatched right parenthesis in a regular expression to be treated literally. Fixes Issue #40. Thanks to GitHub user Warner Losh (bsdimp) for the report. Thanks to Arnold Robbins for the fix. May 29,2019: Fix check for command line arguments to no longer require that first character after '=' not be another '='. Reverts change of August 11, 1989. Thanks to GitHub user Jamie Landeg Jones for pointing out the issue; from Issue #38. Apr 7, 2019: Update awktest.tar(p.50) to use modern options to sort. Needed for Android development. Thanks to GitHub user mohd-akram (Mohamed Akram). From Issue #33. Mar 12, 2019: Added very simplistic support for cross-compiling in the makefile. We are NOT going to go in the direction of the autotools, though. Thanks to GitHub user nee-san for the basic change. (Merged from PR #34.) Mar 5, 2019: Added support for POSIX-standard interval expressions (a.k.a. bounds, a.k.a. repetition expressions) in regular expressions, backported (via NetBSD) from Apple awk-24 (20070501). Thanks to Martijn Dekker for the port. (Merged from PR #30.) Mar 3, 2019: Merge PRs as follows: #12: Avoid undefined behaviour when using ctype(3) functions in relex(). Thanks to GitHub user iamleot. #31: Make getline handle numeric strings, and update FIXES. Thanks - to GitHub user arnoldrobbins. + to GitHub user Arnold Robbins (arnoldrobbins) #32: maketab: support build systems with read-only source. Thanks to GitHub user enh. Jan 25, 2019: Make getline handle numeric strings properly in all cases. (Thanks, Arnold.) Jan 21, 2019: Merged a number of small fixes from GitHub pull requests. Thanks to GitHub users Arnold Robbins (arnoldrobbins), Cody Mello (melloc) and Christoph Junghans (junghans). PR numbers: 13-21, 23, 24, 27. Oct 25, 2018: Added test in maketab.c to prevent generating a proctab entry for YYSTYPE_IS_DEFINED. It was harmless but some gcc settings generated a warning message. Thanks to Nan Xiao for report. Aug 27, 2018: Disallow '$' in printf formats; arguments evaluated in order and printed in order. Added some casts to silence warnings on debugging printfs. (Thanks, Arnold.) Aug 23, 2018: A long list of fixes courtesy of Arnold Robbins, to whom profound thanks. 1. ofs-rebuild: OFS value used to rebuild the record was incorrect. Fixed August 19, 2014. Revised fix August 2018. 2. system-status: Instead of a floating-point division by 256, use the wait(2) macros to create a reasonable exit status. Fixed March 12, 2016. 3. space: Use provided xisblank() function instead of ispace() for matching [[:blank:]]. 4. a-format: Add POSIX standard %a and %A to supported formats. Check at runtime that this format is available. 5. decr-NF: Decrementing NF did not change $0. This is a decades-old bug. There are interactions with the old and new value of OFS as well. Most of the fix came from the NetBSD awk. 6. string-conv: String conversions of scalars were sticky. Once a conversion to string happened, even with OFMT, that value was used until a new numeric value was assigned, even if OFMT differed from CONVFMT, and also if CONVFMT changed. 7. unary-plus: Unary plus on a string constant returned the string. Instead, it should convert the value to numeric and give that value. Also added Arnold's tests for these to awktest.tar as T.arnold. Aug 15, 2018: fixed mangled awktest.tar (thanks, Arnold), posted all current (very minor) fixes to github / onetrueawk Jun 7, 2018: (yes, a long layoff) Updated some broken tests (beebe.tar, T.lilly) [thanks to Arnold Robbins] Mar 26, 2015: buffer overflow in error reporting; thanks to tobias ulmer and john-mark gurney for spotting it and the fix. Feb 4, 2013: cleaned up a handful of tests that didn't seem to actually test for correct behavior: T.latin1, T.gawk. Jan 5, 2013: added ,NULL initializer to static Cells in run.c; not really needed but cleaner. Thanks to Michael Bombardieri. Dec 20, 2012: fiddled makefile to get correct yacc and bison flags. pick yacc (linux) or bison (mac) as necessary. added __attribute__((__noreturn__)) to a couple of lines in proto.h, to silence someone's enthusiastic checker. fixed obscure call by value bug in split(a[1],a) reported on 9fans. the management of temporary values is just a mess; i took a shortcut by making an extra string copy. thanks to paul patience and arnold robbins for passing it on and for proposed patches. tiny fiddle in setfval to eliminate -0 results in T.expr, which has irritated me for 20+ years. Aug 10, 2011: another fix to avoid core dump with delete(ARGV); again, many thanks to ruslan ermilov. Aug 7, 2011: split(s, a, //) now behaves the same as split(s, a, "") Jun 12, 2011: /pat/, \n /pat/ {...} is now legal, though bad style to use. added checks to new -v code that permits -vnospace; thanks to ruslan ermilov for spotting this and providing the patch. removed fixed limit on number of open files; thanks to aleksey cheusov and christos zoulos. fixed day 1 bug that resurrected deleted elements of ARGV when used as filenames (in lib.c). minor type fiddles to make gcc -Wall -pedantic happier (but not totally so); turned on -fno-strict-aliasing in makefile. May 6, 2011: added #ifdef for isblank. now allows -ffoo as well as -f foo arguments. (thanks, ruslan) May 1, 2011: after advice from todd miller, kevin lo, ruslan ermilov, and arnold robbins, changed srand() to return the previous seed (which is 1 on the first call of srand). the seed is an Awkfloat internally though converted to unsigned int to pass to the library srand(). thanks, everyone. fixed a subtle (and i hope low-probability) overflow error in fldbld, by adding space for one extra \0. thanks to robert bassett for spotting this one and providing a fix. removed the files related to compilation on windows. i no longer have anything like a current windows environment, so i can't test any of it. May 23, 2010: fixed long-standing overflow bug in run.c; many thanks to nelson beebe for spotting it and providing the fix. fixed bug that didn't parse -vd=1 properly; thanks to santiago vila for spotting it. Feb 8, 2010: i give up. replaced isblank with isspace in b.c; there are no consistent header files. Nov 26, 2009: fixed a long-standing issue with when FS takes effect. a change to FS is now noticed immediately for subsequent splits. changed the name getline() to awkgetline() to avoid yet another name conflict somewhere. Feb 11, 2009: temporarily for now defined HAS_ISBLANK, since that seems to be the best way through the thicket. isblank arrived in C99, but seems to be arriving at different systems at different times. Oct 8, 2008: fixed typo in b.c that set tmpvec wrongly. no one had ever run into the problem, apparently. thanks to alistair crooks. Oct 23, 2007: minor fix in lib.c: increase inputFS to 100, change malloc for fields to n+1. fixed memory fault caused by out of order test in setsval. thanks to david o'brien, freebsd, for both fixes. May 1, 2007: fiddle in makefile to fix for BSD make; thanks to igor sobrado. Mar 31, 2007: fixed some null pointer refs calling adjbuf. Feb 21, 2007: fixed a bug in matching the null RE in sub and gsub. thanks to al aho who actually did the fix (in b.c), and to wolfgang seeberg for finding it and providing a very compact test case. fixed quotation in b.c; thanks to Hal Pratt and the Princeton Dante Project. removed some no-effect asserts in run.c. fiddled maketab.c to not complain about bison-generated values. removed the obsolete -V argument; fixed --version to print the version and exit. fixed wording and an outright error in the usage message; thanks to igor sobrado and jason mcintyre. fixed a bug in -d that caused core dump if no program followed. Jan 1, 2007: dropped mac.code from makefile; there are few non-MacOSX mac's these days. Jan 17, 2006: system() not flagged as unsafe in the unadvertised -safe option. found it while enhancing tests before shipping the ;login: article. practice what you preach. removed the 9-years-obsolete -mr and -mf flags. added -version and --version options. core dump on linux with BEGIN {nextfile}, now fixed. removed some #ifdef's in run.c and lex.c that appear to no longer be necessary. Apr 24, 2005: modified lib.c so that values of $0 et al are preserved in the END block, apparently as required by posix. thanks to havard eidnes for the report and code. Jan 14, 2005: fixed infinite loop in parsing, originally found by brian tsang. thanks to arnold robbins for a suggestion that started me rethinking it. Dec 31, 2004: prevent overflow of -f array in main, head off potential error in call of SYNTAX(), test malloc return in lib.c, all with thanks to todd miller. Dec 22, 2004: cranked up size of NCHARS; coverity thinks it can be overrun with smaller size, and i think that's right. added some assertions to b.c to catch places where it might overrun. the RE code is still fragile. Dec 5, 2004: fixed a couple of overflow problems with ridiculous field numbers: e.g., print $(2^32-1). thanks to ruslan ermilov, giorgos keramidas and david o'brien at freebsd.org for patches. this really should be re-done from scratch. Nov 21, 2004: fixed another 25-year-old RE bug, in split. it's another failure to (re-)initialize. thanks to steve fisher for spotting this and providing a good test case. Nov 22, 2003: fixed a bug in regular expressions that dates (so help me) from 1977; it's been there from the beginning. an anchored longest match that was longer than the number of states triggered a failure to initialize the machine properly. many thanks to moinak ghosh for not only finding this one but for providing a fix, in some of the most mysterious code known to man. fixed a storage leak in call() that appears to have been there since 1983 or so -- a function without an explicit return that assigns a string to a parameter leaked a Cell. thanks to moinak ghosh for spotting this very subtle one. Jul 31, 2003: fixed, thanks to andrey chernov and ruslan ermilov, a bug in lex.c that mis-handled the character 255 in input. (it was being compared to EOF with a signed comparison.) Jul 29, 2003: fixed (i think) the long-standing botch that included the beginning of line state ^ for RE's in the set of valid characters; this led to a variety of odd problems, including failure to properly match certain regular expressions in non-US locales. thanks to ruslan for keeping at this one. Jul 28, 2003: n-th try at getting internationalization right, with thanks to volker kiefel, arnold robbins and ruslan ermilov for advice, though they should not be blamed for the outcome. according to posix, "." is the radix character in programs and command line arguments regardless of the locale; otherwise, the locale should prevail for input and output of numbers. so it's intended to work that way. i have rescinded the attempt to use strcoll in expanding shorthands in regular expressions (cclenter). its properties are much too surprising; for example [a-c] matches aAbBc in locale en_US but abBcC in locale fr_CA. i can see how this might arise by implementation but i cannot explain it to a human user. (this behavior can be seen in gawk as well; we're leaning on the same library.) the issue appears to be that strcoll is meant for sorting, where merging upper and lower case may make sense (though note that unix sort does not do this by default either). it is not appropriate for regular expressions, where the goal is to match specific patterns of characters. in any case, the notations [:lower:], etc., are available in awk, and they are more likely to work correctly in most locales. a moratorium is hereby declared on internationalization changes. i apologize to friends and colleagues in other parts of the world. i would truly like to get this "right", but i don't know what that is, and i do not want to keep making changes until it's clear. Jul 4, 2003: fixed bug that permitted non-terminated RE, as in "awk /x". Jun 1, 2003: subtle change to split: if source is empty, number of elems is always 0 and the array is not set. Mar 21, 2003: added some parens to isblank, in another attempt to make things internationally portable. Mar 14, 2003: the internationalization changes, somewhat modified, are now reinstated. in theory awk will now do character comparisons and case conversions in national language, but "." will always be the decimal point separator on input and output regardless of national language. isblank(){} has an #ifndef. this no longer compiles on windows: LC_MESSAGES isn't defined in vc6++. fixed subtle behavior in field and record splitting: if FS is a single character and RS is not empty, \n is NOT a separator. this tortuous reading is found in the awk book; behavior now matches gawk and mawk. Dec 13, 2002: for the moment, the internationalization changes of nov 29 are rolled back -- programs like x = 1.2 don't work in some locales, because the parser is expecting x = 1,2. until i understand this better, this will have to wait. Nov 29, 2002: modified b.c (with tiny changes in main and run) to support locales, using strcoll and iswhatever tests for posix character classes. thanks to ruslan ermilov (ru@freebsd.org) for code. the function isblank doesn't seem to have propagated to any header file near me, so it's there explicitly. not properly tested on non-ascii character sets by me. Jun 28, 2002: modified run/format() and tran/getsval() to do a slightly better job on using OFMT for output from print and CONVFMT for other number->string conversions, as promised by posix and done by gawk and mawk. there are still places where it doesn't work right if CONVFMT is changed; by then the STR attribute of the variable has been irrevocably set. thanks to arnold robbins for code and examples. fixed subtle bug in format that could get core dump. thanks to Jaromir Dolecek for finding and fixing. minor cleanup in run.c / format() at the same time. added some tests for null pointers to debugging printf's, which were never intended for external consumption. thanks to dave kerns (dkerns@lucent.com) for pointing this out. GNU compatibility: an empty regexp matches anything (thanks to dag-erling smorgrav, des@ofug.org). subject to reversion if this does more harm than good. pervasive small changes to make things more const-correct, as reported by gcc's -Wwrite-strings. as it says in the gcc manual, this may be more nuisance than useful. provoked by a suggestion and code from arnaud desitter, arnaud@nimbus.geog.ox.ac.uk minor documentation changes to note that this now compiles out of the box on Mac OS X. Feb 10, 2002: changed types in posix chars structure to quiet solaris cc. Jan 1, 2002: fflush() or fflush("") flushes all files and pipes. length(arrayname) returns number of elements; thanks to arnold robbins for suggestion. added a makefile.win to make it easier to build on windows. based on dan allen's buildwin.bat. Nov 16, 2001: added support for posix character class names like [:digit:], which are not exactly shorter than [0-9] and perhaps no more portable. thanks to dag-erling smorgrav for code. Feb 16, 2001: removed -m option; no longer needed, and it was actually broken (noted thanks to volker kiefel). Feb 10, 2001: fixed an appalling bug in gettok: any sequence of digits, +,-, E, e, and period was accepted as a valid number if it started with a period. this would never have happened with the lex version. other 1-character botches, now fixed, include a bare $ and a bare " at the end of the input. Feb 7, 2001: more (const char *) casts in b.c and tran.c to silence warnings. Nov 15, 2000: fixed a bug introduced in august 1997 that caused expressions like $f[1] to be syntax errors. thanks to arnold robbins for noticing this and providing a fix. Oct 30, 2000: fixed some nextfile bugs: not handling all cases. thanks to arnold robbins for pointing this out. new regressions added. close() is now a function. it returns whatever the library fclose returns, and -1 for closing a file or pipe that wasn't opened. Sep 24, 2000: permit \n explicitly in character classes; won't work right if comes in as "[\n]" but ok as /[\n]/, because of multiple processing of \'s. thanks to arnold robbins. July 5, 2000: minor fiddles in tran.c to keep compilers happy about uschar. thanks to norman wilson. May 25, 2000: yet another attempt at making 8-bit input work, with another band-aid in b.c (member()), and some (uschar) casts to head off potential errors in subscripts (like isdigit). also changed HAT to NCHARS-2. thanks again to santiago vila. changed maketab.c to ignore apparently out of range definitions instead of halting; new freeBSD generates one. thanks to jon snader for pointing out the problem. May 2, 2000: fixed an 8-bit problem in b.c by making several char*'s into unsigned char*'s. not clear i have them all yet. thanks to Santiago Vila for the bug report. Apr 21, 2000: finally found and fixed a memory leak in function call; it's been there since functions were added ~1983. thanks to jon bentley for the test case that found it. added test in envinit to catch environment "variables" with names beginning with '='; thanks to Berend Hasselman. Jul 28, 1999: added test in defn() to catch function foo(foo), which otherwise recurses until core dump. thanks to arnold robbins for noticing this. Jun 20, 1999: added *bp in gettok in lex.c; appears possible to exit function without terminating the string. thanks to russ cox. Jun 2, 1999: added function stdinit() to run to initialize files[] array, in case stdin, etc., are not constants; some compilers care. May 10, 1999: replaced the ERROR ... FATAL, etc., macros with functions based on vprintf, to avoid problems caused by overrunning fixed-size errbuf array. thanks to ralph corderoy for the impetus, and for pointing out a string termination bug in qstring as well. Apr 21, 1999: fixed bug that caused occasional core dumps with commandline variable with value ending in \. (thanks to nelson beebe for the test case.) Apr 16, 1999: with code kindly provided by Bruce Lilly, awk now parses /=/ and similar constructs more sensibly in more places. Bruce also provided some helpful test cases. Apr 5, 1999: changed true/false to True/False in run.c to make it easier to compile with C++. Added some casts on malloc and realloc to be honest about casts; ditto. changed ltype int to long in struct rrow to reduce some 64-bit complaints; other changes scattered throughout for the same purpose. thanks to Nelson Beebe for these portability improvements. removed some horrible pointer-int casting in b.c and elsewhere by adding ptoi and itonp to localize the casts, which are all benign. fixed one incipient bug that showed up on sgi in 64-bit mode. reset lineno for new source file; include filename in error message. also fixed line number error in continuation lines. (thanks to Nelson Beebe for both of these.) Mar 24, 1999: Nelson Beebe notes that irix 5.3 yacc dies with a bogus error; use a newer version or switch to bison, since sgi is unlikely to fix it. Mar 5, 1999: changed isnumber to is_number to avoid the problem caused by versions of ctype.h that include the name isnumber. distribution now includes a script for building on a Mac, thanks to Dan Allen. Feb 20, 1999: fixed memory leaks in run.c (call) and tran.c (setfval). thanks to Stephen Nutt for finding these and providing the fixes. Jan 13, 1999: replaced srand argument by (unsigned int) in run.c; avoids problem on Mac and potentially on Unix & Windows. thanks to Dan Allen. added a few (int) casts to silence useless compiler warnings. e.g., errorflag= in run.c jump(). added proctab.c to the bundle outout; one less thing to have to compile out of the box. added calls to _popen and _pclose to the win95 stub for pipes (thanks to Steve Adams for this helpful suggestion). seems to work, though properties are not well understood by me, and it appears that under some circumstances the pipe output is truncated. Be careful. Oct 19, 1998: fixed a couple of bugs in getrec: could fail to update $0 after a getline var; because inputFS wasn't initialized, could split $0 on every character, a misleading diversion. fixed caching bug in makedfa: LRU was actually removing least often used. thanks to ross ridge for finding these, and for providing great bug reports. May 12, 1998: fixed potential bug in readrec: might fail to update record pointer after growing. thanks to dan levy for spotting this and suggesting the fix. Mar 12, 1998: added -V to print version number and die. [notify dave kerns, dkerns@dacsoup.ih.lucent.com] Feb 11, 1998: subtle silent bug in lex.c: if the program ended with a number longer than 1 digit, part of the input would be pushed back and parsed again because token buffer wasn't terminated right. example: awk 'length($0) > 10'. blush. at least i found it myself. Aug 31, 1997: s/adelete/awkdelete/: SGI uses this in malloc.h. thanks to nelson beebe for pointing this one out. Aug 21, 1997: fixed some bugs in sub and gsub when replacement includes \\. this is a dark, horrible corner, but at least now i believe that the behavior is the same as gawk and the intended posix standard. thanks to arnold robbins for advice here. Aug 9, 1997: somewhat regretfully, replaced the ancient lex-based lexical analyzer with one written in C. it's longer, generates less code, and more portable; the old one depended too much on mysterious properties of lex that were not preserved in other environments. in theory these recognize the same language. now using strtod to test whether a string is a number, instead of the convoluted original function. should be more portable and reliable if strtod is implemented right. removed now-pointless optimization in makefile that tries to avoid recompilation when awkgram.y is changed but symbols are not. removed most fixed-size arrays, though a handful remain, some of which are unchecked. you have been warned. Aug 4, 1997: with some trepidation, replaced the ancient code that managed fields and $0 in fixed-size arrays with arrays that grow on demand. there is still some tension between trying to make this run fast and making it clean; not sure it's right yet. the ill-conceived -mr and -mf arguments are now useful only for debugging. previous dynamic string code removed. numerous other minor cleanups along the way. Jul 30, 1997: using code provided by dan levy (to whom profuse thanks), replaced fixed-size arrays and awkward kludges by a fairly uniform mechanism to grow arrays as needed for printf, sub, gsub, etc. Jul 23, 1997: falling off the end of a function returns "" and 0, not 0. thanks to arnold robbins. Jun 17, 1997: replaced several fixed-size arrays by dynamically-created ones in run.c; added overflow tests to some previously unchecked cases. getline, toupper, tolower. getline code is still broken in that recursive calls may wind up using the same space. [fixed later] increased RECSIZE to 8192 to push problems further over the horizon. added \r to \n as input line separator for programs, not data. damn CRLFs. modified format() to permit explicit printf("%c", 0) to include a null byte in output. thanks to ken stailey for the fix. added a "-safe" argument that disables file output (print >, print >>), process creation (cmd|getline, print |, system), and access to the environment (ENVIRON). this is a first approximation to a "safe" version of awk, but don't rely on it too much. thanks to joan feigenbaum and matt blaze for the inspiration long ago. Jul 8, 1996: fixed long-standing bug in sub, gsub(/a/, "\\\\&"); thanks to ralph corderoy. Jun 29, 1996: fixed awful bug in new field splitting; didn't get all the places where input was done. Jun 28, 1996: changed field-splitting to conform to posix definition: fields are split using the value of FS at the time of input; it used to be the value when the field or NF was first referred to, a much less predictable definition. thanks to arnold robbins for encouragement to do the right thing. May 28, 1996: fixed appalling but apparently unimportant bug in parsing octal numbers in reg exprs. explicit hex in reg exprs now limited to 2 chars: \xa, \xaa. May 27, 1996: cleaned up some declarations so gcc -Wall is now almost silent. makefile now includes backup copies of ytab.c and lexyy.c in case one makes before looking; it also avoids recreating lexyy.c unless really needed. s/aprintf/awkprint, s/asprintf/awksprintf/ to avoid some name clashes with unwisely-written header files. thanks to jeffrey friedl for several of these. May 26, 1996: an attempt to rationalize the (unsigned) char issue. almost all instances of unsigned char have been removed; the handful of places in b.c where chars are used as table indices have been hand-crafted. added some latin-1 tests to the regression, but i'm not confident; none of my compilers seem to care much. thanks to nelson beebe for pointing out some others that do care. May 2, 1996: removed all register declarations. enhanced split(), as in gawk, etc: split(s, a, "") splits s into a[1]...a[length(s)] with each character a single element. made the same changes for field-splitting if FS is "". added nextfile, as in gawk: causes immediate advance to next input file. (thanks to arnold robbins for inspiration and code). small fixes to regexpr code: can now handle []], [[], and variants; [] is now a syntax error, rather than matching everything; [z-a] is now empty, not z. far from complete or correct, however. (thanks to jeffrey friedl for pointing out some awful behaviors.) Apr 29, 1996: replaced uchar by uschar everywhere; apparently some compilers usurp this name and this causes conflicts. fixed call to time in run.c (bltin); arg is time_t *. replaced horrible pointer/long punning in b.c by a legitimate union. should be safer on 64-bit machines and cleaner everywhere. (thanks to nelson beebe for pointing out some of these problems.) replaced nested comments by #if 0...#endif in run.c, lib.c. removed getsval, setsval, execute macros from run.c and lib.c. machines are 100x faster than they were when these macros were first used. revised filenames: awk.g.y => awkgram.y, awk.lx.l => awklex.l, y.tab.[ch] => ytab.[ch], lex.yy.c => lexyy.c, all in the aid of portability to nameless systems. "make bundle" now includes yacc and lex output files for recipients who don't have yacc or lex. Aug 15, 1995: initialized Cells in setsymtab more carefully; some fields were not set. (thanks to purify, all of whose complaints i think i now understand.) fixed at least one error in gsub that looked at -1-th element of an array when substituting for a null match (e.g., $). delete arrayname is now legal; it clears the elements but leaves the array, which may not be the right behavior. modified makefile: my current make can't cope with the test used to avoid unnecessary yacc invocations. Jul 17, 1995: added dynamically growing strings to awk.lx.l and b.c to permit regular expressions to be much bigger. the state arrays can still overflow. Aug 24, 1994: detect duplicate arguments in function definitions (mdm). May 11, 1994: trivial fix to printf to limit string size in sub(). Apr 22, 1994: fixed yet another subtle self-assignment problem: $1 = $2; $1 = $1 clobbered $1. Regression tests now use private echo, to avoid quoting problems. Feb 2, 1994: changed error() to print line number as %d, not %g. Jul 23, 1993: cosmetic changes: increased sizes of some arrays, reworded some error messages. added CONVFMT as in posix (just replaced OFMT in getsval) FILENAME is now "" until the first thing that causes a file to be opened. Nov 28, 1992: deleted yyunput and yyoutput from proto.h; different versions of lex give these different declarations. May 31, 1992: added -mr N and -mf N options: more record and fields. these really ought to adjust automatically. cleaned up some error messages; "out of space" now means malloc returned NULL in all cases. changed rehash so that if it runs out, it just returns; things will continue to run slow, but maybe a bit longer. Apr 24, 1992: remove redundant close of stdin when using -f -. got rid of core dump with -d; awk -d just prints date. Apr 12, 1992: added explicit check for /dev/std(in,out,err) in redirection. unlike gawk, no /dev/fd/n yet. added (file/pipe) builtin. hard to test satisfactorily. not posix. Feb 20, 1992: recompile after abortive changes; should be unchanged. Dec 2, 1991: die-casting time: converted to ansi C, installed that. Nov 30, 1991: fixed storage leak in freefa, failing to recover [N]CCL. thanks to Bill Jones (jones@cs.usask.ca) Nov 19, 1991: use RAND_MAX instead of literal in builtin(). Nov 12, 1991: cranked up some fixed-size arrays in b.c, and added a test for overflow in penter. thanks to mark larsen. Sep 24, 1991: increased buffer in gsub. a very crude fix to a general problem. and again on Sep 26. Aug 18, 1991: enforce variable name syntax for commandline variables: has to start with letter or _. Jul 27, 1991: allow newline after ; in for statements. Jul 21, 1991: fixed so that in self-assignment like $1=$1, side effects like recomputing $0 take place. (this is getting subtle.) Jun 30, 1991: better test for detecting too-long output record. Jun 2, 1991: better defense against very long printf strings. made break and continue illegal outside of loops. May 13, 1991: removed extra arg on gettemp, tempfree. minor error message rewording. May 6, 1991: fixed silly bug in hex parsing in hexstr(). removed an apparently unnecessary test in isnumber(). warn about weird printf conversions. fixed unchecked array overwrite in relex(). changed for (i in array) to access elements in sorted order. then unchanged it -- it really does run slower in too many cases. left the code in place, commented out. Feb 10, 1991: check error status on all writes, to avoid banging on full disks. Jan 28, 1991: awk -f - reads the program from stdin. Jan 11, 1991: failed to set numeric state on $0 in cmd|getline context in run.c. Nov 2, 1990: fixed sleazy test for integrality in getsval; use modf. Oct 29, 1990: fixed sleazy buggy code in lib.c that looked (incorrectly) for too long input lines. Oct 14, 1990: fixed the bug on p. 198 in which it couldn't deduce that an argument was an array in some contexts. replaced the error message in intest() by code that damn well makes it an array. Oct 8, 1990: fixed horrible bug: types and values were not preserved in some kinds of self-assignment. (in assign().) Aug 24, 1990: changed NCHARS to 256 to handle 8-bit characters in strings presented to match(), etc. Jun 26, 1990: changed struct rrow (awk.h) to use long instead of int for lval, since cfoll() stores a pointer in it. now works better when int's are smaller than pointers! May 6, 1990: AVA fixed the grammar so that ! is uniformly of the same precedence as unary + and -. This renders illegal some constructs like !x=y, which now has to be parenthesized as !(x=y), and makes others work properly: !x+y is (!x)+y, and x!y is x !y, not two pattern-action statements. (These problems were pointed out by Bob Lenk of Posix.) Added \x to regular expressions (already in strings). Limited octal to octal digits; \8 and \9 are not octal. Centralized the code for parsing escapes in regular expressions. Added a bunch of tests to T.re and T.sub to verify some of this. Feb 9, 1990: fixed null pointer dereference bug in main.c: -F[nothing]. sigh. restored srand behavior: it returns the current seed. Jan 18, 1990: srand now returns previous seed value (0 to start). Jan 5, 1990: fix potential problem in tran.c -- something was freed, then used in freesymtab. Oct 18, 1989: another try to get the max number of open files set with relatively machine-independent code. small fix to input() in case of multiple reads after EOF. Oct 11, 1989: FILENAME is now defined in the BEGIN block -- too many old programs broke. "-" means stdin in getline as well as on the commandline. added a bunch of casts to the code to tell the truth about char * vs. unsigned char *, a right royal pain. added a setlocale call to the front of main, though probably no one has it usefully implemented yet. Aug 24, 1989: removed redundant relational tests against nullnode if parse tree already had a relational at that point. Aug 11, 1989: fixed bug: commandline variable assignment has to look like var=something. (consider the man page for =, in file =.1) changed number of arguments to functions to static arrays to avoid repeated malloc calls. Aug 2, 1989: restored -F (space) separator Jul 30, 1989: added -v x=1 y=2 ... for immediate commandline variable assignment; done before the BEGIN block for sure. they have to precede the program if the program is on the commandline. Modified Aug 2 to require a separate -v for each assignment. Jul 10, 1989: fixed ref-thru-zero bug in environment code in tran.c Jun 23, 1989: add newline to usage message. Jun 14, 1989: added some missing ansi printf conversion letters: %i %X %E %G. no sensible meaning for h or L, so they may not do what one expects. made %* conversions work. changed x^y so that if n is a positive integer, it's done by explicit multiplication, thus achieving maximum accuracy. (this should be done by pow() but it seems not to be locally.) done to x ^= y as well. Jun 4, 1989: ENVIRON array contains environment: if shell variable V=thing, ENVIRON["V"] is "thing" multiple -f arguments permitted. error reporting is naive. (they were permitted before, but only the last was used.) fixed a really stupid botch in the debugging macro dprintf fixed order of evaluation of commandline assignments to match what the book claims: an argument of the form x=e is evaluated at the time it would have been opened if it were a filename (p 63). this invalidates the suggested answer to ex 4-1 (p 195). removed some code that permitted -F (space) fieldseparator, since it didn't quite work right anyway. (restored aug 2) Apr 27, 1989: Line number now accumulated correctly for comment lines. Apr 26, 1989: Debugging output now includes a version date, if one compiles it into the source each time. Apr 9, 1989: Changed grammar to prohibit constants as 3rd arg of sub and gsub; prevents class of overwriting-a-constant errors. (Last one?) This invalidates the "banana" example on page 43 of the book. Added \a ("alert"), \v (vertical tab), \xhhh (hexadecimal), as in ANSI, for strings. Rescinded the sloppiness that permitted non-octal digits in \ooo. Warning: not all compilers and libraries will be able to deal with \x correctly. Jan 9, 1989: Fixed bug that caused tempcell list to contain a duplicate. The fix is kludgy. Dec 17, 1988: Catches some more commandline errors in main. Removed redundant decl of modf in run.c (confuses some compilers). Warning: there's no single declaration of malloc, etc., in awk.h that seems to satisfy all compilers. Dec 7, 1988: Added a bit of code to error printing to avoid printing nulls. (Not clear that it actually would.) Nov 27, 1988: With fear and trembling, modified the grammar to permit multiple pattern-action statements on one line without an explicit separator. By definition, this capitulation to the ghost of ancient implementations remains undefined and thus subject to change without notice or apology. DO NOT COUNT ON IT. Oct 30, 1988: Fixed bug in call() that failed to recover storage. A warning is now generated if there are more arguments in the call than in the definition (in lieu of fixing another storage leak). Oct 20, 1988: Fixed %c: if expr is numeric, use numeric value; otherwise print 1st char of string value. still doesn't work if the value is 0 -- won't print \0. Added a few more checks for running out of malloc. Oct 12, 1988: Fixed bug in call() that freed local arrays twice. Fixed to handle deletion of non-existent array right; complains about attempt to delete non-array element. Sep 30, 1988: Now guarantees to evaluate all arguments of built-in functions, as in C; the appearance is that arguments are evaluated before the function is called. Places affected are sub (gsub was ok), substr, printf, and all the built-in arithmetic functions in bltin(). A warning is generated if a bltin() is called with the wrong number of arguments. This requires changing makeprof on p167 of the book. Aug 23, 1988: setting FILENAME in BEGIN caused core dump, apparently because it was freeing space not allocated by malloc. July 24, 1988: fixed egregious error in toupper/tolower functions. still subject to rescinding, however. July 2, 1988: flush stdout before opening file or pipe July 2, 1988: performance bug in b.c/cgoto(): not freeing some sets of states. partial fix only right now, and the number of states increased to make it less obvious. June 1, 1988: check error status on close May 28, 1988: srand returns seed value it's using. see 1/18/90 May 22, 1988: Removed limit on depth of function calls. May 10, 1988: Fixed lib.c to permit _ in commandline variable names. Mar 25, 1988: main.c fixed to recognize -- as terminator of command- line options. Illegal options flagged. Error reporting slightly cleaned up. Dec 2, 1987: Newer C compilers apply a strict scope rule to extern declarations within functions. Two extern declarations in lib.c and tran.c have been moved to obviate this problem. Oct xx, 1987: Reluctantly added toupper and tolower functions. Subject to rescinding without notice. Sep 17, 1987: Error-message printer had printf(s) instead of printf("%s",s); got core dumps when the message included a %. Sep 12, 1987: Very long printf strings caused core dump; fixed aprintf, asprintf, format to catch them. Can still get a core dump in printf itself. diff --git a/contrib/one-true-awk/README.md b/contrib/one-true-awk/README.md index 76ae3d48c983..daace23e166e 100644 --- a/contrib/one-true-awk/README.md +++ b/contrib/one-true-awk/README.md @@ -1,123 +1,148 @@ # The One True Awk This is the version of `awk` described in _The AWK Programming Language_, -by Al Aho, Brian Kernighan, and Peter Weinberger -(Addison-Wesley, 1988, ISBN 0-201-07981-X). +Second Edition, by Al Aho, Brian Kernighan, and Peter Weinberger +(Addison-Wesley, 2024, ISBN-13 978-0138269722, ISBN-10 0138269726). + +## What's New? ## + +This version of Awk handles UTF-8 and comma-separated values (CSV) input. + +### Strings ### + +Functions that process strings now count Unicode code points, not bytes; +this affects `length`, `substr`, `index`, `match`, `split`, +`sub`, `gsub`, and others. Note that code +points are not necessarily characters. + +UTF-8 sequences may appear in literal strings and regular expressions. +Aribtrary characters may be included with `\u` followed by 1 to 8 hexadecimal digits. + +### Regular expressions ### + +Regular expressions may include UTF-8 code points, including `\u`. +Character classes are likely to be limited to about 256 characters +when expanded. + +### CSV ### + +The option `--csv` turns on CSV processing of input: +fields are separated by commas, fields may be quoted with +double-quote (`"`) characters, quoted fields may contain embedded newlines. +In CSV mode, `FS` is ignored. + +If no explicit separator argument is provided, +field-splitting in `split` is determined by CSV mode. ## Copyright Copyright (C) Lucent Technologies 1997
All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ## Distribution and Reporting Problems Changes, mostly bug fixes and occasional enhancements, are listed in `FIXES`. If you distribute this code further, please please please distribute `FIXES` with it. If you find errors, please report them -to bwk@cs.princeton.edu. +to the current maintainer, ozan.yigit@gmail.com. Please _also_ open an issue in the GitHub issue tracker, to make it easy to track issues. Thanks. ## Submitting Pull Requests Pull requests are welcome. Some guidelines: * Please do not use functions or facilities that are not standard (e.g., `strlcpy()`, `fpurge()`). * Please run the test suite and make sure that your changes pass before posting the pull request. To do so: 1. Save the previous version of `awk` somewhere in your path. Call it `nawk` (for example). 1. Run `oldawk=nawk make check > check.out 2>&1`. 1. Search for `BAD` or `error` in the result. In general, look over it manually to make sure there are no errors. * Please create the pull request with a request to merge into the `staging` branch instead of into the `master` branch. This allows us to do testing, and to make any additional edits or changes after the merge but before merging to `master`. ## Building The program itself is created by make which should produce a sequence of messages roughly like this: - yacc -d awkgram.y - conflicts: 43 shift/reduce, 85 reduce/reduce - mv y.tab.c ytab.c - mv y.tab.h ytab.h - cc -c ytab.c - cc -c b.c - cc -c main.c - cc -c parse.c - cc maketab.c -o maketab - ./maketab >proctab.c - cc -c proctab.c - cc -c tran.c - cc -c lib.c - cc -c run.c - cc -c lex.c - cc ytab.o b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o -lm + bison -d awkgram.y + awkgram.y: warning: 44 shift/reduce conflicts [-Wconflicts-sr] + awkgram.y: warning: 85 reduce/reduce conflicts [-Wconflicts-rr] + awkgram.y: note: rerun with option '-Wcounterexamples' to generate conflict counterexamples + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o awkgram.tab.o awkgram.tab.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o b.o b.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o main.o main.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o parse.o parse.c + gcc -g -Wall -pedantic -Wcast-qual -O2 maketab.c -o maketab + ./maketab awkgram.tab.h >proctab.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o proctab.o proctab.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o tran.o tran.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o lib.o lib.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o run.o run.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o lex.o lex.c + gcc -g -Wall -pedantic -Wcast-qual -O2 awkgram.tab.o b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o -lm This produces an executable `a.out`; you will eventually want to move this to some place like `/usr/bin/awk`. If your system does not have `yacc` or `bison` (the GNU equivalent), you need to install one of them first. -NOTE: This version uses ANSI C (C 99), as you should also. We have +NOTE: This version uses ISO/IEC C99, as you should also. We have compiled this without any changes using `gcc -Wall` and/or local C compilers on a variety of systems, but new systems or compilers may raise some new complaint; reports of difficulties are welcome. This compiles without change on Macintosh OS X using `gcc` and the standard developer tools. You can also use `make CC=g++` to build with the GNU C++ compiler, should you choose to do so. -The version of `malloc` that comes with some systems is sometimes -astonishly slow. If `awk` seems slow, you might try fixing that. -More generally, turning on optimization can significantly improve -`awk`'s speed, perhaps by 1/3 for highest levels. - ## A Note About Releases -We don't do releases. +We don't usually do releases. ## A Note About Maintenance NOTICE! Maintenance of this program is on a ''best effort'' basis. We try to get to issues and pull requests as quickly as we can. Unfortunately, however, keeping this program going is not at the top of our priority list. #### Last Updated -Sat Jul 25 14:00:07 EDT 2021 +Sun 15 Oct 2023 06:28:36 IDT diff --git a/contrib/one-true-awk/awk.1 b/contrib/one-true-awk/awk.1 index b3698eb07d1a..f814e2348343 100644 --- a/contrib/one-true-awk/awk.1 +++ b/contrib/one-true-awk/awk.1 @@ -1,684 +1,693 @@ .de EX .nf .ft CW .. .de EE .br .fi .ft 1 .. .de TF .IP "" "\w'\fB\\$1\ \ \fP'u" .PD 0 .. .TH AWK 1 .CT 1 files prog_other .SH NAME awk \- pattern-directed scanning and processing language .SH SYNOPSIS .B awk [ .BI \-F .I fs +| +.B \-\^\-csv ] [ .BI \-v .I var=value ] [ .I 'prog' | .BI \-f .I progfile ] [ .I file ... ] .SH DESCRIPTION .I Awk scans each input .I file for lines that match any of a set of patterns specified literally in .I prog or in one or more files specified as .B \-f .IR progfile . With each pattern there can be an associated action that will be performed when a line of a .I file matches the pattern. Each line is matched against the pattern portion of every pattern-action statement; the associated action is performed for each matched pattern. The file name .B \- means the standard input. Any .I file of the form .I var=value is treated as an assignment, not a filename, and is executed at the time it would have been opened if it were a filename. The option .B \-v followed by .I var=value is an assignment to be done before .I prog is executed; any number of .B \-v options may be present. The .B \-F .I fs option defines the input field separator to be the regular expression .IR fs . +The +.B \-\^\-csv +option causes +.I awk +to process records using (more or less) standard comma-separated values +(CSV) format. .PP An input line is normally made up of fields separated by white space, or by the regular expression .BR FS . The fields are denoted .BR $1 , .BR $2 , \&..., while .B $0 refers to the entire line. If .BR FS is null, the input line is split into one field per character. .PP A pattern-action statement has the form: .IP .IB pattern " { " action " } .PP A missing .BI { " action " } means print the line; a missing pattern always matches. Pattern-action statements are separated by newlines or semicolons. .PP An action is a sequence of statements. A statement can be one of the following: .PP .EX .ta \w'\f(CWdelete array[expression]\fR'u .RS .nf .ft CW if(\fI expression \fP)\fI statement \fP\fR[ \fPelse\fI statement \fP\fR]\fP while(\fI expression \fP)\fI statement\fP for(\fI expression \fP;\fI expression \fP;\fI expression \fP)\fI statement\fP for(\fI var \fPin\fI array \fP)\fI statement\fP do\fI statement \fPwhile(\fI expression \fP) break continue {\fR [\fP\fI statement ... \fP\fR] \fP} \fIexpression\fP #\fR commonly\fP\fI var = expression\fP print\fR [ \fP\fIexpression-list \fP\fR] \fP\fR[ \fP>\fI expression \fP\fR]\fP printf\fI format \fP\fR[ \fP,\fI expression-list \fP\fR] \fP\fR[ \fP>\fI expression \fP\fR]\fP return\fR [ \fP\fIexpression \fP\fR]\fP next #\fR skip remaining patterns on this input line\fP nextfile #\fR skip rest of this file, open next, start at top\fP delete\fI array\fP[\fI expression \fP] #\fR delete an array element\fP delete\fI array\fP #\fR delete all elements of array\fP exit\fR [ \fP\fIexpression \fP\fR]\fP #\fR exit immediately; status is \fP\fIexpression\fP .fi .RE .EE .DT .PP Statements are terminated by semicolons, newlines or right braces. An empty .I expression-list stands for .BR $0 . String constants are quoted \&\f(CW"\ "\fR, with the usual C escapes recognized within. Expressions take on string or numeric values as appropriate, and are built using the operators .B + \- * / % ^ (exponentiation), and concatenation (indicated by white space). The operators .B ! ++ \-\- += \-= *= /= %= ^= > >= < <= == != ?: are also available in expressions. Variables may be scalars, array elements (denoted .IB x [ i ] \fR) or fields. Variables are initialized to the null string. Array subscripts may be any string, not necessarily numeric; this allows for a form of associative memory. Multiple subscripts such as .B [i,j,k] are permitted; the constituents are concatenated, separated by the value of .BR SUBSEP . .PP The .B print statement prints its arguments on the standard output (or on a file if .BI > " file or .BI >> " file is present or on a pipe if .BI | " cmd is present), separated by the current output field separator, and terminated by the output record separator. .I file and .I cmd may be literal names or parenthesized expressions; identical string values in different statements denote the same open file. The .B printf statement formats its expression list according to the .I format (see .IR printf (3)). The built-in function .BI close( expr ) closes the file or pipe .IR expr . The built-in function .BI fflush( expr ) flushes any buffered output for the file or pipe .IR expr . .PP The mathematical functions .BR atan2 , .BR cos , .BR exp , .BR log , .BR sin , and .B sqrt are built in. Other built-in functions: -.TF length +.TF "\fBlength(\fR[\fIv\^\fR]\fB)\fR" .TP -.B length +\fBlength(\fR[\fIv\^\fR]\fB)\fR the length of its argument taken as a string, number of elements in an array for an array argument, or length of .B $0 if no argument. .TP -.B rand +.B rand() random number on [0,1). .TP -.B srand +\fBsrand(\fR[\fIs\^\fR]\fB)\fR sets seed for .B rand and returns the previous seed. .TP -.B int +.BI int( x\^ ) truncates to an integer value. .TP \fBsubstr(\fIs\fB, \fIm\fR [\fB, \fIn\^\fR]\fB)\fR the .IR n -character substring of .I s that begins at position .I m counted from 1. If no .IR n , use the rest of the string. .TP .BI index( s , " t" ) the position in .I s where the string .I t occurs, or 0 if it does not. .TP .BI match( s , " r" ) the position in .I s where the regular expression .I r occurs, or 0 if it does not. The variables .B RSTART and .B RLENGTH are set to the position and length of the matched string. .TP \fBsplit(\fIs\fB, \fIa \fR[\fB, \fIfs\^\fR]\fB)\fR splits the string .I s into array elements .IB a [1] \fR, .IB a [2] \fR, \&..., .IB a [ n ] \fR, and returns .IR n . The separation is done with the regular expression .I fs or with the field separator .B FS if .I fs is not given. An empty string as field separator splits the string into one array element per character. .TP \fBsub(\fIr\fB, \fIt \fR[, \fIs\^\fR]\fB) substitutes .I t for the first occurrence of the regular expression .I r in the string .IR s . If .I s is not given, .B $0 is used. .TP \fBgsub(\fIr\fB, \fIt \fR[, \fIs\^\fR]\fB) same as .B sub except that all occurrences of the regular expression are replaced; .B sub and .B gsub return the number of replacements. .TP \fBgensub(\fIpat\fB, \fIrepl\fB, \fIhow\fR [\fB, \fItarget\fR]\fB)\fR replaces instances of .I pat in .I target with .IR repl . If .I how is \fB"g"\fR or \fB"G"\fR, do so globally. Otherwise, .I how is a number indicating which occurrence to replace. If no .IR target , use .BR $0 . Return the resulting string; .I target is not modified. .TP .BI sprintf( fmt , " expr" , " ...\fB) the string resulting from formatting .I expr ... according to the .IR printf (3) format .IR fmt . .TP .B systime() returns the current date and time as a standard ``seconds since the epoch'' value. .TP .BI strftime( fmt ", " timestamp\^ ) formats .I timestamp (a value in seconds since the epoch) according to .IR fmt , which is a format string as supported by .IR strftime (3). Both .I timestamp and .I fmt may be omitted; if no .IR timestamp , the current time of day is used, and if no .IR fmt , a default format of \fB"%a %b %e %H:%M:%S %Z %Y"\fR is used. .TP .BI system( cmd ) executes .I cmd and returns its exit status. This will be \-1 upon error, .IR cmd 's exit status upon a normal exit, 256 + .I sig upon death-by-signal, where .I sig is the number of the murdering signal, or 512 + .I sig if there was a core dump. .TP .BI tolower( str ) returns a copy of .I str with all upper-case characters translated to their corresponding lower-case equivalents. .TP .BI toupper( str ) returns a copy of .I str with all lower-case characters translated to their corresponding upper-case equivalents. .PD .PP The ``function'' .B getline sets .B $0 to the next input record from the current input file; .B getline .BI < " file sets .B $0 to the next record from .IR file . .B getline .I x sets variable .I x instead. Finally, .IB cmd " | getline pipes the output of .I cmd into .BR getline ; each call of .B getline returns the next line of output from .IR cmd . In all cases, .B getline returns 1 for a successful input, 0 for end of file, and \-1 for an error. .PP The functions .BR compl , .BR and , .BR or , .BR xor , .BR lshift , and .B rshift peform the corresponding bitwise operations on their operands, which are first truncated to integer. .PP Patterns are arbitrary Boolean combinations (with .BR "! || &&" ) of regular expressions and relational expressions. Regular expressions are as in .IR egrep ; see .IR grep (1). Isolated regular expressions in a pattern apply to the entire line. Regular expressions may also occur in relational expressions, using the operators .B ~ and .BR !~ . .BI / re / is a constant regular expression; any string (constant or variable) may be used as a regular expression, except in the position of an isolated regular expression in a pattern. .PP A pattern may consist of two patterns separated by a comma; in this case, the action is performed for all lines from an occurrence of the first pattern -though an occurrence of the second. +through an occurrence of the second, inclusive. .PP A relational expression is one of the following: .IP .I expression matchop regular-expression .br .I expression relop expression .br .IB expression " in " array-name .br -.BI ( expr , expr,... ") in " array-name +.BI ( expr ,\| expr ,\| ... ") in " array-name .PP where a .I relop is any of the six relational operators in C, and a .I matchop is either .B ~ (matches) or .B !~ (does not match). A conditional is an arithmetic expression, a relational expression, or a Boolean combination of these. .PP The special patterns .B BEGIN and .B END may be used to capture control before the first input line is read and after the last. .B BEGIN and .B END do not combine with other patterns. They may appear multiple times in a program and execute in the order they are read by .IR awk . .PP Variable names with special meanings: .TF FILENAME .TP .B ARGC argument count, assignable. .TP .B ARGV argument array, assignable; non-null members are taken as filenames. .TP .B CONVFMT conversion format used when converting numbers (default .BR "%.6g" ). .TP .B ENVIRON array of environment variables; subscripts are names. .TP .B FILENAME the name of the current input file. .TP .B FNR ordinal number of the current record in the current file. .TP .B FS regular expression used to separate fields; also settable by option .BI \-F fs\fR. .TP .BR NF number of fields in the current record. .TP .B NR ordinal number of the current record. .TP .B OFMT output format for numbers (default .BR "%.6g" ). .TP .B OFS output field separator (default space). .TP .B ORS output record separator (default newline). .TP .B RLENGTH the length of a string matched by .BR match . .TP .B RS input record separator (default newline). If empty, blank lines separate records. If more than one character long, .B RS is treated as a regular expression, and records are separated by text matching the expression. .TP .B RSTART the start position of a string matched by .BR match . .TP .B SUBSEP separates multiple subscripts (default 034). .PD .PP Functions may be defined (at the position of a pattern-action statement) thus: .IP .B -function foo(a, b, c) { ...; return x } +function foo(a, b, c) { ... } .PP Parameters are passed by value if scalar and by reference if array name; functions may be called recursively. Parameters are local to the function; all other variables are global. Thus local variables may be created by providing excess parameters in the function definition. .SH ENVIRONMENT VARIABLES If .B POSIXLY_CORRECT is set in the environment, then .I awk follows the POSIX rules for .B sub and .B gsub with respect to consecutive backslashes and ampersands. .SH EXAMPLES .TP .EX length($0) > 72 .EE Print lines longer than 72 characters. .TP .EX { print $2, $1 } .EE Print first two fields in opposite order. .PP .EX BEGIN { FS = ",[ \et]*|[ \et]+" } { print $2, $1 } .EE .ns .IP Same, with input fields separated by comma and/or spaces and tabs. .PP .EX .nf { s += $1 } END { print "sum is", s, " average is", s/NR } .fi .EE .ns .IP Add up first column, print sum and average. .TP .EX /start/, /stop/ .EE Print all lines between start/stop pairs. .PP .EX .nf BEGIN { # Simulate echo(1) for (i = 1; i < ARGC; i++) printf "%s ", ARGV[i] printf "\en" exit } .fi .EE .SH SEE ALSO .IR grep (1), .IR lex (1), .IR sed (1) .br A. V. Aho, B. W. Kernighan, P. J. Weinberger, -.IR "The AWK Programming Language" , -Addison-Wesley, 1988. ISBN 0-201-07981-X. +.IR "The AWK Programming Language, Second Edition" , +Addison-Wesley, 2024. ISBN 978-0-13-826972-2, 0-13-826972-6. .SH BUGS There are no explicit conversions between numbers and strings. To force an expression to be treated as a number add 0 to it; to force it to be treated as a string concatenate \&\f(CW""\fP to it. .PP The scope rules for variables in functions are a botch; the syntax is worse. .PP -Only eight-bit characters sets are handled correctly. +Input is expected to be UTF-8 encoded. Other multibyte +character sets are not handled. .SH UNUSUAL FLOATING-POINT VALUES .I Awk was designed before IEEE 754 arithmetic defined Not-A-Number (NaN) and Infinity values, which are supported by all modern floating-point hardware. .PP Because .I awk uses .IR strtod (3) and .IR atof (3) to convert string values to double-precision floating-point values, modern C libraries also convert strings starting with .B inf and .B nan into infinity and NaN values respectively. This led to strange results, with something like this: .PP .EX .nf echo nancy | awk '{ print $1 + 0 }' .fi .EE .PP printing .B nan instead of zero. .PP .I Awk now follows GNU AWK, and prefilters string values before attempting to convert them to numbers, as follows: .TP .I "Hexadecimal values" Hexadecimal values (allowed since C99) convert to zero, as they did prior to C99. .TP .I "NaN values" The two strings .B +nan and .B \-nan (case independent) convert to NaN. No others do. (NaNs can have signs.) .TP .I "Infinity values" The two strings .B +inf and .B \-inf (case independent) convert to positive and negative infinity, respectively. No others do. diff --git a/contrib/one-true-awk/awk.h b/contrib/one-true-awk/awk.h index 51c00df9f279..4e066b9706c5 100644 --- a/contrib/one-true-awk/awk.h +++ b/contrib/one-true-awk/awk.h @@ -1,266 +1,278 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #include #include #include #if __STDC_VERSION__ <= 199901L #define noreturn #else #include #endif typedef double Awkfloat; /* unsigned char is more trouble than it's worth */ typedef unsigned char uschar; -#define xfree(a) { if ((a) != NULL) { free((void *)(intptr_t)(a)); (a) = NULL; } } +#define xfree(a) { free((void *)(intptr_t)(a)); (a) = NULL; } /* * We sometimes cheat writing read-only pointers to NUL-terminate them * and then put back the original value */ #define setptr(ptr, a) (*(char *)(intptr_t)(ptr)) = (a) #define NN(p) ((p) ? (p) : "(null)") /* guaranteed non-null for DPRINTF */ #define DEBUG #ifdef DEBUG # define DPRINTF(...) if (dbg) printf(__VA_ARGS__) #else # define DPRINTF(...) #endif extern enum compile_states { RUNNING, COMPILING, ERROR_PRINTING } compile_time; extern bool safe; /* false => unsafe, true => safe */ #define RECSIZE (8 * 1024) /* sets limit on records, fields, etc., etc. */ extern int recsize; /* size of current record, orig RECSIZE */ +extern size_t awk_mb_cur_max; /* max size of a multi-byte character */ + extern char EMPTY[]; /* this avoid -Wwritable-strings issues */ extern char **FS; extern char **RS; extern char **ORS; extern char **OFS; extern char **OFMT; extern Awkfloat *NR; extern Awkfloat *FNR; extern Awkfloat *NF; extern char **FILENAME; extern char **SUBSEP; extern Awkfloat *RSTART; extern Awkfloat *RLENGTH; +extern bool CSV; /* true for csv input */ + extern char *record; /* points to $0 */ extern int lineno; /* line number in awk program */ extern int errorflag; /* 1 if error has occurred */ extern bool donefld; /* true if record broken into fields */ extern bool donerec; /* true if record is valid (no fld has changed */ extern int dbg; extern const char *patbeg; /* beginning of pattern matched */ extern int patlen; /* length of pattern matched. set in b.c */ /* Cell: all information about a variable or constant */ typedef struct Cell { uschar ctype; /* OCELL, OBOOL, OJUMP, etc. */ uschar csub; /* CCON, CTEMP, CFLD, etc. */ char *nval; /* name, for variables only */ char *sval; /* string value */ Awkfloat fval; /* value as number */ int tval; /* type info: STR|NUM|ARR|FCN|FLD|CON|DONTFREE|CONVC|CONVO */ char *fmt; /* CONVFMT/OFMT value used to convert from number */ struct Cell *cnext; /* ptr to next if chained */ } Cell; typedef struct Array { /* symbol table array */ int nelem; /* elements in table right now */ int size; /* size of tab */ Cell **tab; /* hash table pointers */ } Array; #define NSYMTAB 50 /* initial size of a symbol table */ extern Array *symtab; extern Cell *nrloc; /* NR */ extern Cell *fnrloc; /* FNR */ extern Cell *fsloc; /* FS */ extern Cell *nfloc; /* NF */ extern Cell *ofsloc; /* OFS */ extern Cell *orsloc; /* ORS */ extern Cell *rsloc; /* RS */ extern Cell *rstartloc; /* RSTART */ extern Cell *rlengthloc; /* RLENGTH */ extern Cell *subseploc; /* SUBSEP */ extern Cell *symtabloc; /* SYMTAB */ /* Cell.tval values: */ #define NUM 01 /* number value is valid */ #define STR 02 /* string value is valid */ #define DONTFREE 04 /* string space is not freeable */ #define CON 010 /* this is a constant */ #define ARR 020 /* this is an array */ #define FCN 040 /* this is a function name */ #define FLD 0100 /* this is a field $1, $2, ... */ #define REC 0200 /* this is $0 */ #define CONVC 0400 /* string was converted from number via CONVFMT */ #define CONVO 01000 /* string was converted from number via OFMT */ /* function types */ #define FLENGTH 1 #define FSQRT 2 #define FEXP 3 #define FLOG 4 #define FINT 5 #define FSYSTEM 6 #define FRAND 7 #define FSRAND 8 #define FSIN 9 #define FCOS 10 #define FATAN 11 #define FTOUPPER 12 #define FTOLOWER 13 #define FFLUSH 14 #define FAND 15 #define FFOR 16 #define FXOR 17 #define FCOMPL 18 #define FLSHIFT 19 #define FRSHIFT 20 #define FSYSTIME 21 #define FSTRFTIME 22 /* Node: parse tree is made of nodes, with Cell's at bottom */ typedef struct Node { int ntype; struct Node *nnext; int lineno; int nobj; struct Node *narg[1]; /* variable: actual size set by calling malloc */ } Node; #define NIL ((Node *) 0) extern Node *winner; extern Node *nullstat; extern Node *nullnode; /* ctypes */ #define OCELL 1 #define OBOOL 2 #define OJUMP 3 /* Cell subtypes: csub */ #define CFREE 7 #define CCOPY 6 #define CCON 5 #define CTEMP 4 #define CNAME 3 #define CVAR 2 #define CFLD 1 #define CUNK 0 /* bool subtypes */ #define BTRUE 11 #define BFALSE 12 /* jump subtypes */ #define JEXIT 21 #define JNEXT 22 #define JBREAK 23 #define JCONT 24 #define JRET 25 #define JNEXTFILE 26 /* node types */ #define NVALUE 1 #define NSTAT 2 #define NEXPR 3 extern int pairstack[], paircnt; #define notlegal(n) (n <= FIRSTTOKEN || n >= LASTTOKEN || proctab[n-FIRSTTOKEN] == nullproc) #define isvalue(n) ((n)->ntype == NVALUE) #define isexpr(n) ((n)->ntype == NEXPR) #define isjump(n) ((n)->ctype == OJUMP) #define isexit(n) ((n)->csub == JEXIT) #define isbreak(n) ((n)->csub == JBREAK) #define iscont(n) ((n)->csub == JCONT) #define isnext(n) ((n)->csub == JNEXT || (n)->csub == JNEXTFILE) #define isret(n) ((n)->csub == JRET) #define isrec(n) ((n)->tval & REC) #define isfld(n) ((n)->tval & FLD) #define isstr(n) ((n)->tval & STR) #define isnum(n) ((n)->tval & NUM) #define isarr(n) ((n)->tval & ARR) #define isfcn(n) ((n)->tval & FCN) #define istrue(n) ((n)->csub == BTRUE) #define istemp(n) ((n)->csub == CTEMP) #define isargument(n) ((n)->nobj == ARG) /* #define freeable(p) (!((p)->tval & DONTFREE)) */ #define freeable(p) ( ((p)->tval & (STR|DONTFREE)) == STR ) /* structures used by regular expression matching machinery, mostly b.c: */ -#define NCHARS (256+3) /* 256 handles 8-bit chars; 128 does 7-bit */ +#define NCHARS (1256+3) /* 256 handles 8-bit chars; 128 does 7-bit */ + /* BUG: some overflows (caught) if we use 256 */ /* watch out in match(), etc. */ #define HAT (NCHARS+2) /* matches ^ in regular expr */ #define NSTATES 32 typedef struct rrow { long ltype; /* long avoids pointer warnings on 64-bit */ union { int i; Node *np; uschar *up; + int *rp; /* rune representation of char class */ } lval; /* because Al stores a pointer in it! */ int *lfollow; } rrow; +typedef struct gtt { /* gototab entry */ + unsigned int ch; + unsigned int state; +} gtt; + typedef struct fa { - unsigned int **gototab; + gtt **gototab; + int gototab_len; uschar *out; uschar *restr; int **posns; int state_count; bool anchor; int use; int initstat; int curstat; int accept; struct rrow re[1]; /* variable: actual size set by calling malloc */ } fa; #include "proto.h" diff --git a/contrib/one-true-awk/awkgram.y b/contrib/one-true-awk/awkgram.y index f37073d1f9ac..233253a4307b 100644 --- a/contrib/one-true-awk/awkgram.y +++ b/contrib/one-true-awk/awkgram.y @@ -1,508 +1,515 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ %{ #include #include #include "awk.h" void checkdup(Node *list, Cell *item); int yywrap(void) { return(1); } Node *beginloc = 0; Node *endloc = 0; bool infunc = false; /* = true if in arglist or body of func */ int inloop = 0; /* >= 1 if in while, for, do; can't be bool, since loops can next */ char *curfname = 0; /* current function name */ Node *arglist = 0; /* list of args for current function */ %} %union { Node *p; Cell *cp; int i; char *s; } %token FIRSTTOKEN /* must be first */ %token

PROGRAM PASTAT PASTAT2 XBEGIN XEND %token NL ',' '{' '(' '|' ';' '/' ')' '}' '[' ']' %token ARRAY %token MATCH NOTMATCH MATCHOP %token FINAL DOT ALL CCL NCCL CHAR OR STAR QUEST PLUS EMPTYRE ZERO %token AND BOR APPEND EQ GE GT LE LT NE IN %token ARG BLTIN BREAK CLOSE CONTINUE DELETE DO EXIT FOR FUNC %token GENSUB SUB GSUB IF INDEX LSUBSTR MATCHFCN NEXT NEXTFILE %token ADD MINUS MULT DIVIDE MOD %token ASSIGN ASGNOP ADDEQ SUBEQ MULTEQ DIVEQ MODEQ POWEQ %token PRINT PRINTF SPRINTF %token

ELSE INTEST CONDEXPR %token POSTINCR PREINCR POSTDECR PREDECR %token VAR IVAR VARNF CALL NUMBER STRING %token REGEXPR %type

pas pattern ppattern plist pplist patlist prarg term re %type

pa_pat pa_stat pa_stats %type reg_expr %type

simple_stmt opt_simple_stmt stmt stmtlist %type

var varname funcname varlist %type

for if else while %type do st %type pst opt_pst lbrace rbrace rparen comma nl opt_nl and bor %type subop print %type string %right ASGNOP %right '?' %right ':' %left BOR %left AND %left GETLINE %nonassoc APPEND EQ GE GT LE LT NE MATCHOP IN '|' %left ARG BLTIN BREAK CALL CLOSE CONTINUE DELETE DO EXIT FOR FUNC %left GSUB IF INDEX LSUBSTR MATCHFCN NEXT NUMBER %left PRINT PRINTF RETURN SPLIT SPRINTF STRING SUB SUBSTR %left REGEXPR VAR VARNF IVAR WHILE '(' %left CAT %left '+' '-' %left '*' '/' '%' %left NOT UMINUS UPLUS %right POWER %right DECR INCR %left INDIRECT %token LASTTOKEN /* must be last */ %% program: pas { if (errorflag==0) winner = (Node *)stat3(PROGRAM, beginloc, $1, endloc); } | error { yyclearin; bracecheck(); SYNTAX("bailing out"); } ; and: AND | and NL ; bor: BOR | bor NL ; comma: ',' | comma NL ; do: DO | do NL ; else: ELSE | else NL ; for: FOR '(' opt_simple_stmt ';' opt_nl pattern ';' opt_nl opt_simple_stmt rparen {inloop++;} stmt { --inloop; $$ = stat4(FOR, $3, notnull($6), $9, $12); } | FOR '(' opt_simple_stmt ';' ';' opt_nl opt_simple_stmt rparen {inloop++;} stmt { --inloop; $$ = stat4(FOR, $3, NIL, $7, $10); } | FOR '(' varname IN varname rparen {inloop++;} stmt { --inloop; $$ = stat3(IN, $3, makearr($5), $8); } ; funcname: VAR { setfname($1); } | CALL { setfname($1); } ; if: IF '(' pattern rparen { $$ = notnull($3); } ; lbrace: '{' | lbrace NL ; nl: NL | nl NL ; opt_nl: /* empty */ { $$ = 0; } | nl ; opt_pst: /* empty */ { $$ = 0; } | pst ; opt_simple_stmt: /* empty */ { $$ = 0; } | simple_stmt ; pas: opt_pst { $$ = 0; } | opt_pst pa_stats opt_pst { $$ = $2; } ; pa_pat: pattern { $$ = notnull($1); } ; pa_stat: pa_pat { $$ = stat2(PASTAT, $1, stat2(PRINT, rectonode(), NIL)); } | pa_pat lbrace stmtlist '}' { $$ = stat2(PASTAT, $1, $3); } | pa_pat ',' opt_nl pa_pat { $$ = pa2stat($1, $4, stat2(PRINT, rectonode(), NIL)); } | pa_pat ',' opt_nl pa_pat lbrace stmtlist '}' { $$ = pa2stat($1, $4, $6); } | lbrace stmtlist '}' { $$ = stat2(PASTAT, NIL, $2); } | XBEGIN lbrace stmtlist '}' { beginloc = linkum(beginloc, $3); $$ = 0; } | XEND lbrace stmtlist '}' { endloc = linkum(endloc, $3); $$ = 0; } | FUNC funcname '(' varlist rparen {infunc = true;} lbrace stmtlist '}' { infunc = false; curfname=0; defn((Cell *)$2, $4, $8); $$ = 0; } ; pa_stats: pa_stat | pa_stats opt_pst pa_stat { $$ = linkum($1, $3); } ; patlist: pattern | patlist comma pattern { $$ = linkum($1, $3); } ; ppattern: var ASGNOP ppattern { $$ = op2($2, $1, $3); } | ppattern '?' ppattern ':' ppattern %prec '?' { $$ = op3(CONDEXPR, notnull($1), $3, $5); } | ppattern bor ppattern %prec BOR { $$ = op2(BOR, notnull($1), notnull($3)); } | ppattern and ppattern %prec AND { $$ = op2(AND, notnull($1), notnull($3)); } - | ppattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); } + | ppattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); free($3); } | ppattern MATCHOP ppattern - { if (constnode($3)) + { if (constnode($3)) { $$ = op3($2, NIL, $1, (Node*)makedfa(strnode($3), 0)); - else + free($3); + } else $$ = op3($2, (Node *)1, $1, $3); } | ppattern IN varname { $$ = op2(INTEST, $1, makearr($3)); } | '(' plist ')' IN varname { $$ = op2(INTEST, $2, makearr($5)); } | ppattern term %prec CAT { $$ = op2(CAT, $1, $2); } | re | term ; pattern: var ASGNOP pattern { $$ = op2($2, $1, $3); } | pattern '?' pattern ':' pattern %prec '?' { $$ = op3(CONDEXPR, notnull($1), $3, $5); } | pattern bor pattern %prec BOR { $$ = op2(BOR, notnull($1), notnull($3)); } | pattern and pattern %prec AND { $$ = op2(AND, notnull($1), notnull($3)); } | pattern EQ pattern { $$ = op2($2, $1, $3); } | pattern GE pattern { $$ = op2($2, $1, $3); } | pattern GT pattern { $$ = op2($2, $1, $3); } | pattern LE pattern { $$ = op2($2, $1, $3); } | pattern LT pattern { $$ = op2($2, $1, $3); } | pattern NE pattern { $$ = op2($2, $1, $3); } - | pattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); } + | pattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); free($3); } | pattern MATCHOP pattern - { if (constnode($3)) + { if (constnode($3)) { $$ = op3($2, NIL, $1, (Node*)makedfa(strnode($3), 0)); - else + free($3); + } else $$ = op3($2, (Node *)1, $1, $3); } | pattern IN varname { $$ = op2(INTEST, $1, makearr($3)); } | '(' plist ')' IN varname { $$ = op2(INTEST, $2, makearr($5)); } | pattern '|' GETLINE var { if (safe) SYNTAX("cmd | getline is unsafe"); else $$ = op3(GETLINE, $4, itonp($2), $1); } | pattern '|' GETLINE { if (safe) SYNTAX("cmd | getline is unsafe"); else $$ = op3(GETLINE, (Node*)0, itonp($2), $1); } | pattern term %prec CAT { $$ = op2(CAT, $1, $2); } | re | term ; plist: pattern comma pattern { $$ = linkum($1, $3); } | plist comma pattern { $$ = linkum($1, $3); } ; pplist: ppattern | pplist comma ppattern { $$ = linkum($1, $3); } ; prarg: /* empty */ { $$ = rectonode(); } | pplist | '(' plist ')' { $$ = $2; } ; print: PRINT | PRINTF ; pst: NL | ';' | pst NL | pst ';' ; rbrace: '}' | rbrace NL ; re: reg_expr - { $$ = op3(MATCH, NIL, rectonode(), (Node*)makedfa($1, 0)); } + { $$ = op3(MATCH, NIL, rectonode(), (Node*)makedfa($1, 0)); free($1); } | NOT re { $$ = op1(NOT, notnull($2)); } ; reg_expr: '/' {startreg();} REGEXPR '/' { $$ = $3; } ; rparen: ')' | rparen NL ; simple_stmt: print prarg '|' term { if (safe) SYNTAX("print | is unsafe"); else $$ = stat3($1, $2, itonp($3), $4); } | print prarg APPEND term { if (safe) SYNTAX("print >> is unsafe"); else $$ = stat3($1, $2, itonp($3), $4); } | print prarg GT term { if (safe) SYNTAX("print > is unsafe"); else $$ = stat3($1, $2, itonp($3), $4); } | print prarg { $$ = stat3($1, $2, NIL, NIL); } | DELETE varname '[' patlist ']' { $$ = stat2(DELETE, makearr($2), $4); } | DELETE varname { $$ = stat2(DELETE, makearr($2), 0); } | pattern { $$ = exptostat($1); } | error { yyclearin; SYNTAX("illegal statement"); } ; st: nl | ';' opt_nl ; stmt: BREAK st { if (!inloop) SYNTAX("break illegal outside of loops"); $$ = stat1(BREAK, NIL); } | CONTINUE st { if (!inloop) SYNTAX("continue illegal outside of loops"); $$ = stat1(CONTINUE, NIL); } | do {inloop++;} stmt {--inloop;} WHILE '(' pattern ')' st { $$ = stat2(DO, $3, notnull($7)); } | EXIT pattern st { $$ = stat1(EXIT, $2); } | EXIT st { $$ = stat1(EXIT, NIL); } | for | if stmt else stmt { $$ = stat3(IF, $1, $2, $4); } | if stmt { $$ = stat3(IF, $1, $2, NIL); } | lbrace stmtlist rbrace { $$ = $2; } | NEXT st { if (infunc) SYNTAX("next is illegal inside a function"); $$ = stat1(NEXT, NIL); } | NEXTFILE st { if (infunc) SYNTAX("nextfile is illegal inside a function"); $$ = stat1(NEXTFILE, NIL); } | RETURN pattern st { $$ = stat1(RETURN, $2); } | RETURN st { $$ = stat1(RETURN, NIL); } | simple_stmt st | while {inloop++;} stmt { --inloop; $$ = stat2(WHILE, $1, $3); } | ';' opt_nl { $$ = 0; } ; stmtlist: stmt | stmtlist stmt { $$ = linkum($1, $2); } ; subop: SUB | GSUB ; string: STRING | string STRING { $$ = catstr($1, $2); } ; term: term '/' ASGNOP term { $$ = op2(DIVEQ, $1, $4); } | term '+' term { $$ = op2(ADD, $1, $3); } | term '-' term { $$ = op2(MINUS, $1, $3); } | term '*' term { $$ = op2(MULT, $1, $3); } | term '/' term { $$ = op2(DIVIDE, $1, $3); } | term '%' term { $$ = op2(MOD, $1, $3); } | term POWER term { $$ = op2(POWER, $1, $3); } | '-' term %prec UMINUS { $$ = op1(UMINUS, $2); } | '+' term %prec UMINUS { $$ = op1(UPLUS, $2); } | NOT term %prec UMINUS { $$ = op1(NOT, notnull($2)); } | BLTIN '(' ')' { $$ = op2(BLTIN, itonp($1), rectonode()); } | BLTIN '(' patlist ')' { $$ = op2(BLTIN, itonp($1), $3); } | BLTIN { $$ = op2(BLTIN, itonp($1), rectonode()); } | CALL '(' ')' { $$ = op2(CALL, celltonode($1,CVAR), NIL); } | CALL '(' patlist ')' { $$ = op2(CALL, celltonode($1,CVAR), $3); } | CLOSE term { $$ = op1(CLOSE, $2); } | DECR var { $$ = op1(PREDECR, $2); } | INCR var { $$ = op1(PREINCR, $2); } | var DECR { $$ = op1(POSTDECR, $1); } | var INCR { $$ = op1(POSTINCR, $1); } | GENSUB '(' reg_expr comma pattern comma pattern ')' { $$ = op5(GENSUB, NIL, (Node*)makedfa($3, 1), $5, $7, rectonode()); } | GENSUB '(' pattern comma pattern comma pattern ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op5(GENSUB, NIL, (Node *)makedfa(strnode($3), 1), $5, $7, rectonode()); - else + free($3); + } else $$ = op5(GENSUB, (Node *)1, $3, $5, $7, rectonode()); } | GENSUB '(' reg_expr comma pattern comma pattern comma pattern ')' { $$ = op5(GENSUB, NIL, (Node*)makedfa($3, 1), $5, $7, $9); } | GENSUB '(' pattern comma pattern comma pattern comma pattern ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op5(GENSUB, NIL, (Node *)makedfa(strnode($3),1), $5,$7,$9); - else + free($3); + } else $$ = op5(GENSUB, (Node *)1, $3, $5, $7, $9); } | GETLINE var LT term { $$ = op3(GETLINE, $2, itonp($3), $4); } | GETLINE LT term { $$ = op3(GETLINE, NIL, itonp($2), $3); } | GETLINE var { $$ = op3(GETLINE, $2, NIL, NIL); } | GETLINE { $$ = op3(GETLINE, NIL, NIL, NIL); } | INDEX '(' pattern comma pattern ')' { $$ = op2(INDEX, $3, $5); } | INDEX '(' pattern comma reg_expr ')' { SYNTAX("index() doesn't permit regular expressions"); $$ = op2(INDEX, $3, (Node*)$5); } | '(' pattern ')' { $$ = $2; } | MATCHFCN '(' pattern comma reg_expr ')' - { $$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa($5, 1)); } + { $$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa($5, 1)); free($5); } | MATCHFCN '(' pattern comma pattern ')' - { if (constnode($5)) + { if (constnode($5)) { $$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa(strnode($5), 1)); - else + free($5); + } else $$ = op3(MATCHFCN, (Node *)1, $3, $5); } | NUMBER { $$ = celltonode($1, CCON); } | SPLIT '(' pattern comma varname comma pattern ')' /* string */ { $$ = op4(SPLIT, $3, makearr($5), $7, (Node*)STRING); } | SPLIT '(' pattern comma varname comma reg_expr ')' /* const /regexp/ */ - { $$ = op4(SPLIT, $3, makearr($5), (Node*)makedfa($7, 1), (Node *)REGEXPR); } + { $$ = op4(SPLIT, $3, makearr($5), (Node*)makedfa($7, 1), (Node *)REGEXPR); free($7); } | SPLIT '(' pattern comma varname ')' { $$ = op4(SPLIT, $3, makearr($5), NIL, (Node*)STRING); } /* default */ | SPRINTF '(' patlist ')' { $$ = op1($1, $3); } | string { $$ = celltonode($1, CCON); } | subop '(' reg_expr comma pattern ')' - { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, rectonode()); } + { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, rectonode()); free($3); } | subop '(' pattern comma pattern ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op4($1, NIL, (Node*)makedfa(strnode($3), 1), $5, rectonode()); - else + free($3); + } else $$ = op4($1, (Node *)1, $3, $5, rectonode()); } | subop '(' reg_expr comma pattern comma var ')' - { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, $7); } + { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, $7); free($3); } | subop '(' pattern comma pattern comma var ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op4($1, NIL, (Node*)makedfa(strnode($3), 1), $5, $7); - else + free($3); + } else $$ = op4($1, (Node *)1, $3, $5, $7); } | SUBSTR '(' pattern comma pattern comma pattern ')' { $$ = op3(SUBSTR, $3, $5, $7); } | SUBSTR '(' pattern comma pattern ')' { $$ = op3(SUBSTR, $3, $5, NIL); } | var ; var: varname | varname '[' patlist ']' { $$ = op2(ARRAY, makearr($1), $3); } | IVAR { $$ = op1(INDIRECT, celltonode($1, CVAR)); } | INDIRECT term { $$ = op1(INDIRECT, $2); } ; varlist: /* nothing */ { arglist = $$ = 0; } | VAR { arglist = $$ = celltonode($1,CVAR); } | varlist comma VAR { checkdup($1, $3); arglist = $$ = linkum($1,celltonode($3,CVAR)); } ; varname: VAR { $$ = celltonode($1, CVAR); } | ARG { $$ = op1(ARG, itonp($1)); } | VARNF { $$ = op1(VARNF, (Node *) $1); } ; while: WHILE '(' pattern rparen { $$ = notnull($3); } ; %% void setfname(Cell *p) { if (isarr(p)) SYNTAX("%s is an array, not a function", p->nval); else if (isfcn(p)) SYNTAX("you can't define function %s more than once", p->nval); curfname = p->nval; } int constnode(Node *p) { return isvalue(p) && ((Cell *) (p->narg[0]))->csub == CCON; } char *strnode(Node *p) { return ((Cell *)(p->narg[0]))->sval; } Node *notnull(Node *n) { switch (n->nobj) { case LE: case LT: case EQ: case NE: case GT: case GE: case BOR: case AND: case NOT: return n; default: return op2(NE, n, nullnode); } } void checkdup(Node *vl, Cell *cp) /* check if name already in list */ { char *s = cp->nval; for ( ; vl; vl = vl->nnext) { if (strcmp(s, ((Cell *)(vl->narg[0]))->nval) == 0) { SYNTAX("duplicate argument %s", s); break; } } } diff --git a/contrib/one-true-awk/b.c b/contrib/one-true-awk/b.c index 804236656e22..aa07d59fabac 100644 --- a/contrib/one-true-awk/b.c +++ b/contrib/one-true-awk/b.c @@ -1,1341 +1,1527 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ /* lasciate ogne speranza, voi ch'intrate. */ #define DEBUG #include #include #include #include #include #include "awk.h" #include "awkgram.tab.h" #define MAXLIN 22 #define type(v) (v)->nobj /* badly overloaded here */ #define info(v) (v)->ntype /* badly overloaded here */ #define left(v) (v)->narg[0] #define right(v) (v)->narg[1] #define parent(v) (v)->nnext #define LEAF case CCL: case NCCL: case CHAR: case DOT: case FINAL: case ALL: #define ELEAF case EMPTYRE: /* empty string in regexp */ #define UNARY case STAR: case PLUS: case QUEST: /* encoding in tree Nodes: leaf (CCL, NCCL, CHAR, DOT, FINAL, ALL, EMPTYRE): left is index, right contains value or pointer to value unary (STAR, PLUS, QUEST): left is child, right is null binary (CAT, OR): left and right are children parent contains pointer to parent */ int *setvec; int *tmpset; int maxsetvec = 0; int rtok; /* next token in current re */ int rlxval; static const uschar *rlxstr; static const uschar *prestr; /* current position in current re */ static const uschar *lastre; /* origin of last re */ static const uschar *lastatom; /* origin of last Atom */ static const uschar *starttok; static const uschar *basestr; /* starts with original, replaced during repetition processing */ static const uschar *firstbasestr; static int setcnt; static int poscnt; const char *patbeg; int patlen; #define NFA 128 /* cache this many dynamic fa's */ fa *fatab[NFA]; int nfatab = 0; /* entries in fatab */ +extern int u8_nextlen(const char *s); + + +/* utf-8 mechanism: + + For most of Awk, utf-8 strings just "work", since they look like + null-terminated sequences of 8-bit bytes. + + Functions like length(), index(), and substr() have to operate + in units of utf-8 characters. The u8_* functions in run.c + handle this. + + Regular expressions are more complicated, since the basic + mechanism of the goto table used 8-bit byte indices into the + gototab entries to compute the next state. Unicode is a lot + bigger, so the gototab entries are now structs with a character + and a next state, and there is a linear search of the characters + to find the state. (Yes, this is slower, by a significant + amount. Tough.) + + Throughout the RE mechanism in b.c, utf-8 characters are + converted to their utf-32 value. This mostly shows up in + cclenter, which expands character class ranges like a-z and now + alpha-omega. The size of a gototab array is still about 256. + This should be dynamic, but for now things work ok for a single + code page of Unicode, which is the most likely case. + + The code changes are localized in run.c and b.c. I have added a + handful of functions to somewhat better hide the implementation, + but a lot more could be done. + + */ + +static int get_gototab(fa*, int, int); +static int set_gototab(fa*, int, int, int); +extern int u8_rune(int *, const uschar *); + static int * intalloc(size_t n, const char *f) { int *p = (int *) calloc(n, sizeof(int)); if (p == NULL) overflo(f); return p; } static void resizesetvec(const char *f) { if (maxsetvec == 0) maxsetvec = MAXLIN; else maxsetvec *= 4; setvec = (int *) realloc(setvec, maxsetvec * sizeof(*setvec)); tmpset = (int *) realloc(tmpset, maxsetvec * sizeof(*tmpset)); if (setvec == NULL || tmpset == NULL) overflo(f); } static void resize_state(fa *f, int state) { - unsigned int **p; + gtt **p; uschar *p2; int **p3; int i, new_count; if (++state < f->state_count) return; new_count = state + 10; /* needs to be tuned */ - p = (unsigned int **) realloc(f->gototab, new_count * sizeof(f->gototab[0])); + p = (gtt **) realloc(f->gototab, new_count * sizeof(f->gototab[0])); if (p == NULL) goto out; f->gototab = p; p2 = (uschar *) realloc(f->out, new_count * sizeof(f->out[0])); if (p2 == NULL) goto out; f->out = p2; p3 = (int **) realloc(f->posns, new_count * sizeof(f->posns[0])); if (p3 == NULL) goto out; f->posns = p3; for (i = f->state_count; i < new_count; ++i) { - f->gototab[i] = (unsigned int *) calloc(NCHARS, sizeof(**f->gototab)); + f->gototab[i] = (gtt *) calloc(NCHARS, sizeof(**f->gototab)); if (f->gototab[i] == NULL) goto out; f->out[i] = 0; f->posns[i] = NULL; } + f->gototab_len = NCHARS; /* should be variable, growable */ f->state_count = new_count; return; out: overflo(__func__); } fa *makedfa(const char *s, bool anchor) /* returns dfa for reg expr s */ { int i, use, nuse; fa *pfa; static int now = 1; if (setvec == NULL) { /* first time through any RE */ resizesetvec(__func__); } if (compile_time != RUNNING) /* a constant for sure */ return mkdfa(s, anchor); for (i = 0; i < nfatab; i++) /* is it there already? */ if (fatab[i]->anchor == anchor && strcmp((const char *) fatab[i]->restr, s) == 0) { fatab[i]->use = now++; return fatab[i]; } pfa = mkdfa(s, anchor); if (nfatab < NFA) { /* room for another */ fatab[nfatab] = pfa; fatab[nfatab]->use = now++; nfatab++; return pfa; } use = fatab[0]->use; /* replace least-recently used */ nuse = 0; for (i = 1; i < nfatab; i++) if (fatab[i]->use < use) { use = fatab[i]->use; nuse = i; } freefa(fatab[nuse]); fatab[nuse] = pfa; pfa->use = now++; return pfa; } fa *mkdfa(const char *s, bool anchor) /* does the real work of making a dfa */ /* anchor = true for anchored matches, else false */ { Node *p, *p1; fa *f; firstbasestr = (const uschar *) s; basestr = firstbasestr; p = reparse(s); p1 = op2(CAT, op2(STAR, op2(ALL, NIL, NIL), NIL), p); /* put ALL STAR in front of reg. exp. */ p1 = op2(CAT, p1, op2(FINAL, NIL, NIL)); /* put FINAL after reg. exp. */ poscnt = 0; penter(p1); /* enter parent pointers and leaf indices */ if ((f = (fa *) calloc(1, sizeof(fa) + poscnt * sizeof(rrow))) == NULL) overflo(__func__); f->accept = poscnt-1; /* penter has computed number of positions in re */ cfoll(f, p1); /* set up follow sets */ freetr(p1); resize_state(f, 1); f->posns[0] = intalloc(*(f->re[0].lfollow), __func__); f->posns[1] = intalloc(1, __func__); *f->posns[1] = 0; f->initstat = makeinit(f, anchor); f->anchor = anchor; f->restr = (uschar *) tostring(s); if (firstbasestr != basestr) { if (basestr) xfree(basestr); } return f; } int makeinit(fa *f, bool anchor) { int i, k; f->curstat = 2; f->out[2] = 0; k = *(f->re[0].lfollow); xfree(f->posns[2]); f->posns[2] = intalloc(k + 1, __func__); for (i = 0; i <= k; i++) { (f->posns[2])[i] = (f->re[0].lfollow)[i]; } if ((f->posns[2])[1] == f->accept) f->out[2] = 1; for (i = 0; i < NCHARS; i++) - f->gototab[2][i] = 0; + set_gototab(f, 2, 0, 0); /* f->gototab[2][i] = 0; */ f->curstat = cgoto(f, 2, HAT); if (anchor) { *f->posns[2] = k-1; /* leave out position 0 */ for (i = 0; i < k; i++) { (f->posns[0])[i] = (f->posns[2])[i]; } f->out[0] = f->out[2]; if (f->curstat != 2) --(*f->posns[f->curstat]); } return f->curstat; } void penter(Node *p) /* set up parent pointers and leaf indices */ { switch (type(p)) { ELEAF LEAF info(p) = poscnt; poscnt++; break; UNARY penter(left(p)); parent(left(p)) = p; break; case CAT: case OR: penter(left(p)); penter(right(p)); parent(left(p)) = p; parent(right(p)) = p; break; case ZERO: break; default: /* can't happen */ FATAL("can't happen: unknown type %d in penter", type(p)); break; } } void freetr(Node *p) /* free parse tree */ { switch (type(p)) { ELEAF LEAF xfree(p); break; UNARY case ZERO: freetr(left(p)); xfree(p); break; case CAT: case OR: freetr(left(p)); freetr(right(p)); xfree(p); break; default: /* can't happen */ FATAL("can't happen: unknown type %d in freetr", type(p)); break; } } /* in the parsing of regular expressions, metacharacters like . have */ /* to be seen literally; \056 is not a metacharacter. */ -int hexstr(const uschar **pp) /* find and eval hex string at pp, return new p */ +int hexstr(const uschar **pp, int max) /* find and eval hex string at pp, return new p */ { /* only pick up one 8-bit byte (2 chars) */ const uschar *p; int n = 0; int i; - for (i = 0, p = *pp; i < 2 && isxdigit(*p); i++, p++) { + for (i = 0, p = *pp; i < max && isxdigit(*p); i++, p++) { if (isdigit(*p)) n = 16 * n + *p - '0'; else if (*p >= 'a' && *p <= 'f') n = 16 * n + *p - 'a' + 10; else if (*p >= 'A' && *p <= 'F') n = 16 * n + *p - 'A' + 10; } *pp = p; return n; } + + #define isoctdigit(c) ((c) >= '0' && (c) <= '7') /* multiple use of arg */ int quoted(const uschar **pp) /* pick up next thing after a \\ */ /* and increment *pp */ { const uschar *p = *pp; int c; - if ((c = *p++) == 't') +/* BUG: should advance by utf-8 char even if makes no sense */ + + if ((c = *p++) == 't') { c = '\t'; - else if (c == 'n') + } else if (c == 'n') { c = '\n'; - else if (c == 'f') + } else if (c == 'f') { c = '\f'; - else if (c == 'r') + } else if (c == 'r') { c = '\r'; - else if (c == 'b') + } else if (c == 'b') { c = '\b'; - else if (c == 'v') + } else if (c == 'v') { c = '\v'; - else if (c == 'a') + } else if (c == 'a') { c = '\a'; - else if (c == '\\') + } else if (c == '\\') { c = '\\'; - else if (c == 'x') { /* hexadecimal goo follows */ - c = hexstr(&p); /* this adds a null if number is invalid */ + } else if (c == 'x') { /* 2 hex digits follow */ + c = hexstr(&p, 2); /* this adds a null if number is invalid */ + } else if (c == 'u') { /* unicode char number up to 8 hex digits */ + c = hexstr(&p, 8); } else if (isoctdigit(c)) { /* \d \dd \ddd */ int n = c - '0'; if (isoctdigit(*p)) { n = 8 * n + *p++ - '0'; if (isoctdigit(*p)) n = 8 * n + *p++ - '0'; } c = n; } /* else */ /* c = c; */ *pp = p; return c; } -char *cclenter(const char *argp) /* add a character class */ +int *cclenter(const char *argp) /* add a character class */ { int i, c, c2; - const uschar *op, *p = (const uschar *) argp; - uschar *bp; - static uschar *buf = NULL; + int n; + const uschar *p = (const uschar *) argp; + int *bp, *retp; + static int *buf = NULL; static int bufsz = 100; - op = p; - if (buf == NULL && (buf = (uschar *) malloc(bufsz)) == NULL) + if (buf == NULL && (buf = (int *) calloc(bufsz, sizeof(int))) == NULL) FATAL("out of space for character class [%.10s...] 1", p); bp = buf; - for (i = 0; (c = *p++) != 0; ) { + for (i = 0; *p != 0; ) { + n = u8_rune(&c, p); + p += n; if (c == '\\') { c = quoted(&p); } else if (c == '-' && i > 0 && bp[-1] != 0) { if (*p != 0) { c = bp[-1]; - c2 = *p++; + /* c2 = *p++; */ + n = u8_rune(&c2, p); + p += n; if (c2 == '\\') - c2 = quoted(&p); + c2 = quoted(&p); /* BUG: sets p, has to be u8 size */ if (c > c2) { /* empty; ignore */ bp--; i--; continue; } while (c < c2) { - if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "cclenter1")) - FATAL("out of space for character class [%.10s...] 2", p); + if (i >= bufsz) { + bufsz *= 2; + buf = (int *) realloc(buf, bufsz * sizeof(int)); + if (buf == NULL) + FATAL("out of space for character class [%.10s...] 2", p); + bp = buf + i; + } *bp++ = ++c; i++; } continue; } } - if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "cclenter2")) - FATAL("out of space for character class [%.10s...] 3", p); + if (i >= bufsz) { + bufsz *= 2; + buf = (int *) realloc(buf, bufsz * sizeof(int)); + if (buf == NULL) + FATAL("out of space for character class [%.10s...] 2", p); + bp = buf + i; + } *bp++ = c; i++; } *bp = 0; - DPRINTF("cclenter: in = |%s|, out = |%s|\n", op, buf); - xfree(op); - return (char *) tostring((char *) buf); + /* DPRINTF("cclenter: in = |%s|, out = |%s|\n", op, buf); BUG: can't print array of int */ + /* xfree(op); BUG: what are we freeing here? */ + retp = (int *) calloc(bp-buf+1, sizeof(int)); + for (i = 0; i < bp-buf+1; i++) + retp[i] = buf[i]; + return retp; } void overflo(const char *s) { FATAL("regular expression too big: out of space in %.30s...", s); } void cfoll(fa *f, Node *v) /* enter follow set of each leaf of vertex v into lfollow[leaf] */ { int i; int *p; switch (type(v)) { ELEAF LEAF f->re[info(v)].ltype = type(v); f->re[info(v)].lval.np = right(v); while (f->accept >= maxsetvec) { /* guessing here! */ resizesetvec(__func__); } for (i = 0; i <= f->accept; i++) setvec[i] = 0; setcnt = 0; follow(v); /* computes setvec and setcnt */ p = intalloc(setcnt + 1, __func__); f->re[info(v)].lfollow = p; *p = setcnt; for (i = f->accept; i >= 0; i--) if (setvec[i] == 1) *++p = i; break; UNARY cfoll(f,left(v)); break; case CAT: case OR: cfoll(f,left(v)); cfoll(f,right(v)); break; case ZERO: break; default: /* can't happen */ FATAL("can't happen: unknown type %d in cfoll", type(v)); } } int first(Node *p) /* collects initially active leaves of p into setvec */ /* returns 0 if p matches empty string */ { int b, lp; switch (type(p)) { ELEAF LEAF lp = info(p); /* look for high-water mark of subscripts */ while (setcnt >= maxsetvec || lp >= maxsetvec) { /* guessing here! */ resizesetvec(__func__); } if (type(p) == EMPTYRE) { setvec[lp] = 0; return(0); } if (setvec[lp] != 1) { setvec[lp] = 1; setcnt++; } - if (type(p) == CCL && (*(char *) right(p)) == '\0') + if (type(p) == CCL && (*(int *) right(p)) == 0) return(0); /* empty CCL */ return(1); case PLUS: if (first(left(p)) == 0) return(0); return(1); case STAR: case QUEST: first(left(p)); return(0); case CAT: if (first(left(p)) == 0 && first(right(p)) == 0) return(0); return(1); case OR: b = first(right(p)); if (first(left(p)) == 0 || b == 0) return(0); return(1); case ZERO: return 0; } FATAL("can't happen: unknown type %d in first", type(p)); /* can't happen */ return(-1); } void follow(Node *v) /* collects leaves that can follow v into setvec */ { Node *p; if (type(v) == FINAL) return; p = parent(v); switch (type(p)) { case STAR: case PLUS: first(v); follow(p); return; case OR: case QUEST: follow(p); return; case CAT: if (v == left(p)) { /* v is left child of p */ if (first(right(p)) == 0) { follow(p); return; } } else /* v is right child */ follow(p); return; } } -int member(int c, const char *sarg) /* is c in s? */ +int member(int c, int *sarg) /* is c in s? */ { - const uschar *s = (const uschar *) sarg; + int *s = (int *) sarg; while (*s) if (c == *s++) return(1); return(0); } +static int get_gototab(fa *f, int state, int ch) /* hide gototab inplementation */ +{ + int i; + for (i = 0; i < f->gototab_len; i++) { + if (f->gototab[state][i].ch == 0) + break; + if (f->gototab[state][i].ch == ch) + return f->gototab[state][i].state; + } + return 0; +} + +static int set_gototab(fa *f, int state, int ch, int val) /* hide gototab inplementation */ +{ + int i; + for (i = 0; i < f->gototab_len; i++) { + if (f->gototab[state][i].ch == 0 || f->gototab[state][i].ch == ch) { + f->gototab[state][i].ch = ch; + f->gototab[state][i].state = val; + return val; + } + } + overflo(__func__); + return val; /* not used anywhere at the moment */ +} + int match(fa *f, const char *p0) /* shortest match ? */ { int s, ns; + int n; + int rune; const uschar *p = (const uschar *) p0; + /* return pmatch(f, p0); does it matter whether longest or shortest? */ + s = f->initstat; assert (s < f->state_count); if (f->out[s]) return(1); do { /* assert(*p < NCHARS); */ - if ((ns = f->gototab[s][*p]) != 0) + n = u8_rune(&rune, p); + if ((ns = get_gototab(f, s, rune)) != 0) s = ns; else - s = cgoto(f, s, *p); + s = cgoto(f, s, rune); if (f->out[s]) return(1); - } while (*p++ != 0); + if (*p == 0) + break; + p += n; + } while (1); /* was *p++ != 0 */ return(0); } int pmatch(fa *f, const char *p0) /* longest match, for sub */ { int s, ns; + int n; + int rune; const uschar *p = (const uschar *) p0; const uschar *q; s = f->initstat; assert(s < f->state_count); patbeg = (const char *)p; patlen = -1; do { q = p; do { if (f->out[s]) /* final state */ patlen = q-p; /* assert(*q < NCHARS); */ - if ((ns = f->gototab[s][*q]) != 0) + n = u8_rune(&rune, q); + if ((ns = get_gototab(f, s, rune)) != 0) s = ns; else - s = cgoto(f, s, *q); + s = cgoto(f, s, rune); assert(s < f->state_count); if (s == 1) { /* no transition */ if (patlen >= 0) { patbeg = (const char *) p; return(1); } else goto nextin; /* no match */ } - } while (*q++ != 0); + if (*q == 0) + break; + q += n; + } while (1); + q++; /* was *q++ */ if (f->out[s]) patlen = q-p-1; /* don't count $ */ if (patlen >= 0) { patbeg = (const char *) p; return(1); } nextin: s = 2; - } while (*p++); + if (*p == 0) + break; + n = u8_rune(&rune, p); + p += n; + } while (1); /* was *p++ */ return (0); } int nematch(fa *f, const char *p0) /* non-empty match, for sub */ { int s, ns; + int n; + int rune; const uschar *p = (const uschar *) p0; const uschar *q; s = f->initstat; assert(s < f->state_count); patbeg = (const char *)p; patlen = -1; while (*p) { q = p; do { if (f->out[s]) /* final state */ patlen = q-p; /* assert(*q < NCHARS); */ - if ((ns = f->gototab[s][*q]) != 0) + n = u8_rune(&rune, q); + if ((ns = get_gototab(f, s, rune)) != 0) s = ns; else - s = cgoto(f, s, *q); + s = cgoto(f, s, rune); if (s == 1) { /* no transition */ if (patlen > 0) { patbeg = (const char *) p; return(1); } else goto nnextin; /* no nonempty match */ } - } while (*q++ != 0); + if (*q == 0) + break; + q += n; + } while (1); + q++; if (f->out[s]) patlen = q-p-1; /* don't count $ */ if (patlen > 0 ) { patbeg = (const char *) p; return(1); } nnextin: s = 2; p++; } return (0); } +#define MAX_UTF_BYTES 4 // UTF-8 is up to 4 bytes long + +// Read one rune at a time from the given FILE*. Return both +// the bytes and the actual rune. + +struct runedata { + int rune; + size_t len; + char bytes[6]; +}; + +struct runedata getrune(FILE *fp) +{ + struct runedata result; + int c, next; + + memset(&result, 0, sizeof(result)); + + c = getc(fp); + if (c == EOF) + return result; // result.rune == 0 --> EOF + else if (c < 128 || awk_mb_cur_max == 1) { + result.bytes[0] = c; + result.len = 1; + result.rune = c; + + return result; + } + + // need to get bytes and fill things in + result.bytes[0] = c; + result.len = 1; + + next = 1; + for (int i = 1; i < MAX_UTF_BYTES; i++) { + c = getc(fp); + if (c == EOF) + break; + result.bytes[next++] = c; + result.len++; + } + + // put back any extra input bytes + int actual_len = u8_nextlen(result.bytes); + while (result.len > actual_len) { + ungetc(result.bytes[--result.len], fp); + } + + result.bytes[result.len] = '\0'; + (void) u8_rune(& result.rune, (uschar *) result.bytes); + + return result; +} + + /* * NAME * fnematch * * DESCRIPTION * A stream-fed version of nematch which transfers characters to a * null-terminated buffer. All characters up to and including the last * character of the matching text or EOF are placed in the buffer. If * a match is found, patbeg and patlen are set appropriately. * * RETURN VALUES * false No match found. * true Match found. */ bool fnematch(fa *pfa, FILE *f, char **pbuf, int *pbufsize, int quantum) { char *buf = *pbuf; int bufsize = *pbufsize; - int c, i, j, k, ns, s; + int i, j, k, ns, s; + struct runedata r; s = pfa->initstat; patlen = 0; /* * All indices relative to buf. * i <= j <= k <= bufsize * - * i: origin of active substring - * j: current character + * i: origin of active substring (first byte of first character) + * j: current character (last byte of current character) * k: destination of next getc() */ i = -1, k = 0; do { j = i++; do { - if (++j == k) { - if (k == bufsize) + r = getrune(f); + if ((++j + r.len) >= k) { + if (k >= bufsize) if (!adjbuf((char **) &buf, &bufsize, bufsize+1, quantum, 0, "fnematch")) FATAL("stream '%.30s...' too long", buf); - buf[k++] = (c = getc(f)) != EOF ? c : 0; } - c = (uschar)buf[j]; - /* assert(c < NCHARS); */ + memcpy(buf + k, r.bytes, r.len); + j += r.len - 1; // incremented next time around the loop + k += r.len; - if ((ns = pfa->gototab[s][c]) != 0) + if ((ns = get_gototab(pfa, s, r.rune)) != 0) s = ns; else - s = cgoto(pfa, s, c); + s = cgoto(pfa, s, r.rune); if (pfa->out[s]) { /* final state */ patlen = j - i + 1; - if (c == 0) /* don't count $ */ + if (r.rune == 0) /* don't count $ */ patlen--; } } while (buf[j] && s != 1); s = 2; + if (r.len > 1) + i += r.len - 1; // i incremented around the loop } while (buf[i] && !patlen); /* adjbuf() may have relocated a resized buffer. Inform the world. */ *pbuf = buf; *pbufsize = bufsize; if (patlen) { patbeg = (char *) buf + i; /* * Under no circumstances is the last character fed to * the automaton part of the match. It is EOF's nullbyte, * or it sent the automaton into a state with no further * transitions available (s==1), or both. Room for a * terminating nullbyte is guaranteed. * * ungetc any chars after the end of matching text * (except for EOF's nullbyte, if present) and null * terminate the buffer. */ do - if (buf[--k] && ungetc(buf[k], f) == EOF) - FATAL("unable to ungetc '%c'", buf[k]); + for (int ii = r.len; ii > 0; ii--) + if (buf[--k] && ungetc(buf[k], f) == EOF) + FATAL("unable to ungetc '%c'", buf[k]); while (k > i + patlen); buf[k] = '\0'; return true; } else return false; } Node *reparse(const char *p) /* parses regular expression pointed to by p */ { /* uses relex() to scan regular expression */ Node *np; DPRINTF("reparse <%s>\n", p); lastre = prestr = (const uschar *) p; /* prestr points to string to be parsed */ rtok = relex(); /* GNU compatibility: an empty regexp matches anything */ if (rtok == '\0') { /* FATAL("empty regular expression"); previous */ return(op2(EMPTYRE, NIL, NIL)); } np = regexp(); if (rtok != '\0') FATAL("syntax error in regular expression %s at %s", lastre, prestr); return(np); } Node *regexp(void) /* top-level parse of reg expr */ { return (alt(concat(primary()))); } Node *primary(void) { Node *np; int savelastatom; switch (rtok) { case CHAR: lastatom = starttok; np = op2(CHAR, NIL, itonp(rlxval)); rtok = relex(); return (unary(np)); case ALL: rtok = relex(); return (unary(op2(ALL, NIL, NIL))); case EMPTYRE: rtok = relex(); return (unary(op2(EMPTYRE, NIL, NIL))); case DOT: lastatom = starttok; rtok = relex(); return (unary(op2(DOT, NIL, NIL))); case CCL: np = op2(CCL, NIL, (Node*) cclenter((const char *) rlxstr)); lastatom = starttok; rtok = relex(); return (unary(np)); case NCCL: np = op2(NCCL, NIL, (Node *) cclenter((const char *) rlxstr)); lastatom = starttok; rtok = relex(); return (unary(np)); case '^': rtok = relex(); return (unary(op2(CHAR, NIL, itonp(HAT)))); case '$': rtok = relex(); return (unary(op2(CHAR, NIL, NIL))); case '(': lastatom = starttok; savelastatom = starttok - basestr; /* Retain over recursion */ rtok = relex(); if (rtok == ')') { /* special pleading for () */ rtok = relex(); - return unary(op2(CCL, NIL, (Node *) tostring(""))); + return unary(op2(CCL, NIL, (Node *) cclenter(""))); } np = regexp(); if (rtok == ')') { lastatom = basestr + savelastatom; /* Restore */ rtok = relex(); return (unary(np)); } else FATAL("syntax error in regular expression %s at %s", lastre, prestr); default: FATAL("illegal primary in regular expression %s at %s", lastre, prestr); } return 0; /*NOTREACHED*/ } Node *concat(Node *np) { switch (rtok) { case CHAR: case DOT: case ALL: case CCL: case NCCL: case '$': case '(': return (concat(op2(CAT, np, primary()))); case EMPTYRE: rtok = relex(); - return (concat(op2(CAT, op2(CCL, NIL, (Node *) tostring("")), + return (concat(op2(CAT, op2(CCL, NIL, (Node *) cclenter("")), primary()))); } return (np); } Node *alt(Node *np) { if (rtok == OR) { rtok = relex(); return (alt(op2(OR, np, concat(primary())))); } return (np); } Node *unary(Node *np) { switch (rtok) { case STAR: rtok = relex(); return (unary(op2(STAR, np, NIL))); case PLUS: rtok = relex(); return (unary(op2(PLUS, np, NIL))); case QUEST: rtok = relex(); return (unary(op2(QUEST, np, NIL))); case ZERO: rtok = relex(); return (unary(op2(ZERO, np, NIL))); default: return (np); } } /* * Character class definitions conformant to the POSIX locale as * defined in IEEE P1003.1 draft 7 of June 2001, assuming the source * and operating character sets are both ASCII (ISO646) or supersets * thereof. * * Note that to avoid overflowing the temporary buffer used in * relex(), the expanded character class (prior to range expansion) * must be less than twice the size of their full name. */ /* Because isblank doesn't show up in any of the header files on any * system i use, it's defined here. if some other locale has a richer * definition of "blank", define HAS_ISBLANK and provide your own * version. * the parentheses here are an attempt to find a path through the maze * of macro definition and/or function and/or version provided. thanks * to nelson beebe for the suggestion; let's see if it works everywhere. */ /* #define HAS_ISBLANK */ #ifndef HAS_ISBLANK int (xisblank)(int c) { return c==' ' || c=='\t'; } #endif static const struct charclass { const char *cc_name; int cc_namelen; int (*cc_func)(int); } charclasses[] = { { "alnum", 5, isalnum }, { "alpha", 5, isalpha }, #ifndef HAS_ISBLANK { "blank", 5, xisblank }, #else { "blank", 5, isblank }, #endif { "cntrl", 5, iscntrl }, { "digit", 5, isdigit }, { "graph", 5, isgraph }, { "lower", 5, islower }, { "print", 5, isprint }, { "punct", 5, ispunct }, { "space", 5, isspace }, { "upper", 5, isupper }, { "xdigit", 6, isxdigit }, { NULL, 0, NULL }, }; #define REPEAT_SIMPLE 0 #define REPEAT_PLUS_APPENDED 1 #define REPEAT_WITH_Q 2 #define REPEAT_ZERO 3 static int replace_repeat(const uschar *reptok, int reptoklen, const uschar *atom, int atomlen, int firstnum, int secondnum, int special_case) { int i, j; uschar *buf = 0; int ret = 1; int init_q = (firstnum == 0); /* first added char will be ? */ int n_q_reps = secondnum-firstnum; /* m>n, so reduce until {1,m-n} left */ int prefix_length = reptok - basestr; /* prefix includes first rep */ int suffix_length = strlen((const char *) reptok) - reptoklen; /* string after rep specifier */ int size = prefix_length + suffix_length; if (firstnum > 1) { /* add room for reps 2 through firstnum */ size += atomlen*(firstnum-1); } /* Adjust size of buffer for special cases */ if (special_case == REPEAT_PLUS_APPENDED) { size++; /* for the final + */ } else if (special_case == REPEAT_WITH_Q) { size += init_q + (atomlen+1)* (n_q_reps-init_q); } else if (special_case == REPEAT_ZERO) { size += 2; /* just a null ERE: () */ } if ((buf = (uschar *) malloc(size + 1)) == NULL) FATAL("out of space in reg expr %.10s..", lastre); memcpy(buf, basestr, prefix_length); /* copy prefix */ j = prefix_length; if (special_case == REPEAT_ZERO) { j -= atomlen; buf[j++] = '('; buf[j++] = ')'; } for (i = 1; i < firstnum; i++) { /* copy x reps */ memcpy(&buf[j], atom, atomlen); j += atomlen; } if (special_case == REPEAT_PLUS_APPENDED) { buf[j++] = '+'; } else if (special_case == REPEAT_WITH_Q) { if (init_q) buf[j++] = '?'; for (i = init_q; i < n_q_reps; i++) { /* copy x? reps */ memcpy(&buf[j], atom, atomlen); j += atomlen; buf[j++] = '?'; } } memcpy(&buf[j], reptok+reptoklen, suffix_length); j += suffix_length; buf[j] = '\0'; /* free old basestr */ if (firstbasestr != basestr) { if (basestr) xfree(basestr); } basestr = buf; prestr = buf + prefix_length; if (special_case == REPEAT_ZERO) { prestr -= atomlen; ret++; } return ret; } static int repeat(const uschar *reptok, int reptoklen, const uschar *atom, int atomlen, int firstnum, int secondnum) { /* In general, the repetition specifier or "bound" is replaced here by an equivalent ERE string, repeating the immediately previous atom and appending ? and + as needed. Note that the first copy of the atom is left in place, except in the special_case of a zero-repeat (i.e., {0}). */ if (secondnum < 0) { /* means {n,} -> repeat n-1 times followed by PLUS */ if (firstnum < 2) { /* 0 or 1: should be handled before you get here */ FATAL("internal error"); } else { return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_PLUS_APPENDED); } } else if (firstnum == secondnum) { /* {n} or {n,n} -> simply repeat n-1 times */ if (firstnum == 0) { /* {0} or {0,0} */ /* This case is unusual because the resulting replacement string might actually be SMALLER than the original ERE */ return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_ZERO); } else { /* (firstnum >= 1) */ return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_SIMPLE); } } else if (firstnum < secondnum) { /* {n,m} -> repeat n-1 times then alternate */ /* x{n,m} => xx...x{1, m-n+1} => xx...x?x?x?..x? */ return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_WITH_Q); } else { /* Error - shouldn't be here (n>m) */ FATAL("internal error"); } return 0; } +extern int u8_rune(int *, const uschar *); /* run.c; should be in header file */ + int relex(void) /* lexical analyzer for reparse */ { int c, n; int cflag; static uschar *buf = NULL; static int bufsz = 100; uschar *bp; const struct charclass *cc; int i; int num, m; bool commafound, digitfound; const uschar *startreptok; static int parens = 0; rescan: starttok = prestr; + if ((n = u8_rune(&rlxval, prestr)) > 1) { + prestr += n; + starttok = prestr; + return CHAR; + } + switch (c = *prestr++) { case '|': return OR; case '*': return STAR; case '+': return PLUS; case '?': return QUEST; case '.': return DOT; case '\0': prestr--; return '\0'; case '^': case '$': return c; case '(': parens++; return c; case ')': if (parens) { parens--; return c; } /* unmatched close parenthesis; per POSIX, treat as literal */ rlxval = c; return CHAR; case '\\': rlxval = quoted(&prestr); return CHAR; default: rlxval = c; return CHAR; case '[': if (buf == NULL && (buf = (uschar *) malloc(bufsz)) == NULL) FATAL("out of space in reg expr %.10s..", lastre); bp = buf; if (*prestr == '^') { cflag = 1; prestr++; } else cflag = 0; - n = 2 * strlen((const char *) prestr)+1; + n = 5 * strlen((const char *) prestr)+1; /* BUG: was 2. what value? */ if (!adjbuf((char **) &buf, &bufsz, n, n, (char **) &bp, "relex1")) FATAL("out of space for reg expr %.10s...", lastre); for (; ; ) { + if ((n = u8_rune(&rlxval, prestr)) > 1) { + for (i = 0; i < n; i++) + *bp++ = *prestr++; + continue; + } if ((c = *prestr++) == '\\') { *bp++ = '\\'; if ((c = *prestr++) == '\0') FATAL("nonterminated character class %.20s...", lastre); *bp++ = c; /* } else if (c == '\n') { */ /* FATAL("newline in character class %.20s...", lastre); */ } else if (c == '[' && *prestr == ':') { /* POSIX char class names, Dag-Erling Smorgrav, des@ofug.org */ for (cc = charclasses; cc->cc_name; cc++) if (strncmp((const char *) prestr + 1, (const char *) cc->cc_name, cc->cc_namelen) == 0) break; if (cc->cc_name != NULL && prestr[1 + cc->cc_namelen] == ':' && prestr[2 + cc->cc_namelen] == ']') { prestr += cc->cc_namelen + 3; /* * BUG: We begin at 1, instead of 0, since we * would otherwise prematurely terminate the * string for classes like [[:cntrl:]]. This * means that we can't match the NUL character, * not without first adapting the entire * program to track each string's length. */ for (i = 1; i <= UCHAR_MAX; i++) { - if (!adjbuf((char **) &buf, &bufsz, bp-buf+1, 100, (char **) &bp, "relex2")) + if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "relex2")) FATAL("out of space for reg expr %.10s...", lastre); if (cc->cc_func(i)) { /* escape backslash */ if (i == '\\') { *bp++ = '\\'; n++; } *bp++ = i; n++; } } } else *bp++ = c; } else if (c == '[' && *prestr == '.') { char collate_char; prestr++; collate_char = *prestr++; if (*prestr == '.' && prestr[1] == ']') { prestr += 2; /* Found it: map via locale TBD: for now, simply return this char. This is sufficient to pass conformance test awk.ex 156 */ if (*prestr == ']') { prestr++; rlxval = collate_char; return CHAR; } } } else if (c == '[' && *prestr == '=') { char equiv_char; prestr++; equiv_char = *prestr++; if (*prestr == '=' && prestr[1] == ']') { prestr += 2; /* Found it: map via locale TBD: for now simply return this char. This is sufficient to pass conformance test awk.ex 156 */ if (*prestr == ']') { prestr++; rlxval = equiv_char; return CHAR; } } } else if (c == '\0') { FATAL("nonterminated character class %.20s", lastre); } else if (bp == buf) { /* 1st char is special */ *bp++ = c; } else if (c == ']') { *bp++ = 0; rlxstr = (uschar *) tostring((char *) buf); if (cflag == 0) return CCL; else return NCCL; } else *bp++ = c; } break; case '{': if (isdigit(*(prestr))) { num = 0; /* Process as a repetition */ n = -1; m = -1; commafound = false; digitfound = false; startreptok = prestr-1; /* Remember start of previous atom here ? */ } else { /* just a { char, not a repetition */ rlxval = c; return CHAR; } for (; ; ) { if ((c = *prestr++) == '}') { if (commafound) { if (digitfound) { /* {n,m} */ m = num; if (m < n) FATAL("illegal repetition expression: class %.20s", lastre); if (n == 0 && m == 1) { return QUEST; } } else { /* {n,} */ if (n == 0) return STAR; else if (n == 1) return PLUS; } } else { if (digitfound) { /* {n} same as {n,n} */ n = num; m = num; } else { /* {} */ FATAL("illegal repetition expression: class %.20s", lastre); } } if (repeat(starttok, prestr-starttok, lastatom, startreptok - lastatom, n, m) > 0) { if (n == 0 && m == 0) { return ZERO; } /* must rescan input for next token */ goto rescan; } /* Failed to replace: eat up {...} characters and treat like just PLUS */ return PLUS; } else if (c == '\0') { FATAL("nonterminated character class %.20s", lastre); } else if (isdigit(c)) { num = 10 * num + c - '0'; digitfound = true; } else if (c == ',') { if (commafound) FATAL("illegal repetition expression: class %.20s", lastre); /* looking for {n,} or {n,m} */ commafound = true; n = num; digitfound = false; /* reset */ num = 0; } else { FATAL("illegal repetition expression: class %.20s", lastre); } } break; } } int cgoto(fa *f, int s, int c) { int *p, *q; int i, j, k; - assert(c == HAT || c < NCHARS); + /* assert(c == HAT || c < NCHARS); BUG: seg fault if disable test */ while (f->accept >= maxsetvec) { /* guessing here! */ resizesetvec(__func__); } for (i = 0; i <= f->accept; i++) setvec[i] = 0; setcnt = 0; resize_state(f, s); /* compute positions of gototab[s,c] into setvec */ p = f->posns[s]; for (i = 1; i <= *p; i++) { if ((k = f->re[p[i]].ltype) != FINAL) { if ((k == CHAR && c == ptoi(f->re[p[i]].lval.np)) || (k == DOT && c != 0 && c != HAT) || (k == ALL && c != 0) || (k == EMPTYRE && c != 0) - || (k == CCL && member(c, (char *) f->re[p[i]].lval.up)) - || (k == NCCL && !member(c, (char *) f->re[p[i]].lval.up) && c != 0 && c != HAT)) { + || (k == CCL && member(c, (int *) f->re[p[i]].lval.rp)) + || (k == NCCL && !member(c, (int *) f->re[p[i]].lval.rp) && c != 0 && c != HAT)) { q = f->re[p[i]].lfollow; for (j = 1; j <= *q; j++) { if (q[j] >= maxsetvec) { resizesetvec(__func__); } if (setvec[q[j]] == 0) { setcnt++; setvec[q[j]] = 1; } } } } } /* determine if setvec is a previous state */ tmpset[0] = setcnt; j = 1; for (i = f->accept; i >= 0; i--) if (setvec[i]) { tmpset[j++] = i; } resize_state(f, f->curstat > s ? f->curstat : s); /* tmpset == previous state? */ for (i = 1; i <= f->curstat; i++) { p = f->posns[i]; if ((k = tmpset[0]) != p[0]) goto different; for (j = 1; j <= k; j++) if (tmpset[j] != p[j]) goto different; /* setvec is state i */ if (c != HAT) - f->gototab[s][c] = i; + set_gototab(f, s, c, i); return i; different:; } /* add tmpset to current set of states */ ++(f->curstat); resize_state(f, f->curstat); for (i = 0; i < NCHARS; i++) - f->gototab[f->curstat][i] = 0; + set_gototab(f, f->curstat, 0, 0); xfree(f->posns[f->curstat]); p = intalloc(setcnt + 1, __func__); f->posns[f->curstat] = p; if (c != HAT) - f->gototab[s][c] = f->curstat; + set_gototab(f, s, c, f->curstat); for (i = 0; i <= setcnt; i++) p[i] = tmpset[i]; if (setvec[f->accept]) f->out[f->curstat] = 1; else f->out[f->curstat] = 0; return f->curstat; } void freefa(fa *f) /* free a finite automaton */ { int i; if (f == NULL) return; for (i = 0; i < f->state_count; i++) xfree(f->gototab[i]) for (i = 0; i <= f->curstat; i++) xfree(f->posns[i]); for (i = 0; i <= f->accept; i++) { xfree(f->re[i].lfollow); if (f->re[i].ltype == CCL || f->re[i].ltype == NCCL) xfree(f->re[i].lval.np); } xfree(f->restr); xfree(f->out); xfree(f->posns); xfree(f->gototab); xfree(f); } diff --git a/contrib/one-true-awk/bugs-fixed/getline-corruption.awk b/contrib/one-true-awk/bugs-fixed/getline-corruption.awk new file mode 100644 index 000000000000..461e551cfff5 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/getline-corruption.awk @@ -0,0 +1,5 @@ +BEGIN { + getline l + getline l + print (s=substr(l,1,10)) " len=" length(s) +} diff --git a/contrib/one-true-awk/bugs-fixed/getline-corruption.in b/contrib/one-true-awk/bugs-fixed/getline-corruption.in new file mode 100644 index 000000000000..78981922613b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/getline-corruption.in @@ -0,0 +1 @@ +a diff --git a/contrib/one-true-awk/bugs-fixed/getline-corruption.ok b/contrib/one-true-awk/bugs-fixed/getline-corruption.ok new file mode 100644 index 000000000000..3efb54597c6d --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/getline-corruption.ok @@ -0,0 +1 @@ +a len=1 diff --git a/contrib/one-true-awk/bugs-fixed/rstart-rlength.awk b/contrib/one-true-awk/bugs-fixed/rstart-rlength.awk new file mode 100644 index 000000000000..f423f0168be3 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/rstart-rlength.awk @@ -0,0 +1,10 @@ +BEGIN { + str="\342\200\257" + print length(str) + match(str,/^/) + print RSTART, RLENGTH + match(str,/.+/) + print RSTART, RLENGTH + match(str,/$/) + print RSTART, RLENGTH +} diff --git a/contrib/one-true-awk/bugs-fixed/rstart-rlength.ok b/contrib/one-true-awk/bugs-fixed/rstart-rlength.ok new file mode 100644 index 000000000000..961cb895b51b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/rstart-rlength.ok @@ -0,0 +1,4 @@ +1 +1 0 +1 1 +2 0 diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.awk b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.awk new file mode 100644 index 000000000000..67366ec75070 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.awk @@ -0,0 +1,6 @@ +BEGIN { + FS="␟" + RS="␞" + OFS="," +} +{ print $1, $2, $3 } diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.in b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.in new file mode 100644 index 000000000000..2e882af62a2c --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.in @@ -0,0 +1,2 @@ +id␟name␟age␞1␟Bob "Billy" Smith␟42␞2␟Jane + Brown␟37 diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.ok b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.ok new file mode 100644 index 000000000000..f337302be903 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.ok @@ -0,0 +1,5 @@ +id,name,age +1,Bob "Billy" Smith,42 +2,Jane + Brown,37 + diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.awk b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.awk new file mode 100644 index 000000000000..34d77bf2c95f --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.awk @@ -0,0 +1,7 @@ +BEGIN { + FS = "א" + RS = "בב" + OFS = "," +} + +{ print $1, $2, $3 } diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.in b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.in new file mode 100644 index 000000000000..2de6e718fd3b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.in @@ -0,0 +1,2 @@ +idאnameאageא1אBob "Billy" Smithא42א2בבJane + Brownא37 diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.ok b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.ok new file mode 100644 index 000000000000..2387001bc1b2 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.ok @@ -0,0 +1,4 @@ +id,name,age +Jane + Brown,37 +, diff --git a/contrib/one-true-awk/bugs-fixed/unicode-null-match.awk b/contrib/one-true-awk/bugs-fixed/unicode-null-match.awk new file mode 100644 index 000000000000..0c056126922b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-null-match.awk @@ -0,0 +1,6 @@ +BEGIN { + # str = "\342\200\257" + str = "あ" + n = gsub(//, "X", str) + print n, str +} diff --git a/contrib/one-true-awk/bugs-fixed/unicode-null-match.bad b/contrib/one-true-awk/bugs-fixed/unicode-null-match.bad new file mode 100644 index 000000000000..7cd35ff2d932 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-null-match.bad @@ -0,0 +1 @@ +4 XXXX diff --git a/contrib/one-true-awk/bugs-fixed/unicode-null-match.ok b/contrib/one-true-awk/bugs-fixed/unicode-null-match.ok new file mode 100644 index 000000000000..1ac142f8a895 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-null-match.ok @@ -0,0 +1 @@ +2 XあX diff --git a/contrib/one-true-awk/lex.c b/contrib/one-true-awk/lex.c index d471ac2944a9..c1b892be30f5 100644 --- a/contrib/one-true-awk/lex.c +++ b/contrib/one-true-awk/lex.c @@ -1,610 +1,642 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #include #include #include #include #include "awk.h" #include "awkgram.tab.h" extern YYSTYPE yylval; extern bool infunc; int lineno = 1; int bracecnt = 0; int brackcnt = 0; int parencnt = 0; typedef struct Keyword { const char *word; int sub; int type; } Keyword; const Keyword keywords[] = { /* keep sorted: binary searched */ { "BEGIN", XBEGIN, XBEGIN }, { "END", XEND, XEND }, { "NF", VARNF, VARNF }, { "and", FAND, BLTIN }, { "atan2", FATAN, BLTIN }, { "break", BREAK, BREAK }, { "close", CLOSE, CLOSE }, { "compl", FCOMPL, BLTIN }, { "continue", CONTINUE, CONTINUE }, { "cos", FCOS, BLTIN }, { "delete", DELETE, DELETE }, { "do", DO, DO }, { "else", ELSE, ELSE }, { "exit", EXIT, EXIT }, { "exp", FEXP, BLTIN }, { "fflush", FFLUSH, BLTIN }, { "for", FOR, FOR }, { "func", FUNC, FUNC }, { "function", FUNC, FUNC }, { "gensub", GENSUB, GENSUB }, { "getline", GETLINE, GETLINE }, { "gsub", GSUB, GSUB }, { "if", IF, IF }, { "in", IN, IN }, { "index", INDEX, INDEX }, { "int", FINT, BLTIN }, { "length", FLENGTH, BLTIN }, { "log", FLOG, BLTIN }, { "lshift", FLSHIFT, BLTIN }, { "match", MATCHFCN, MATCHFCN }, { "next", NEXT, NEXT }, { "nextfile", NEXTFILE, NEXTFILE }, { "or", FFOR, BLTIN }, { "print", PRINT, PRINT }, { "printf", PRINTF, PRINTF }, { "rand", FRAND, BLTIN }, { "return", RETURN, RETURN }, { "rshift", FRSHIFT, BLTIN }, { "sin", FSIN, BLTIN }, { "split", SPLIT, SPLIT }, { "sprintf", SPRINTF, SPRINTF }, { "sqrt", FSQRT, BLTIN }, { "srand", FSRAND, BLTIN }, { "strftime", FSTRFTIME, BLTIN }, { "sub", SUB, SUB }, { "substr", SUBSTR, SUBSTR }, { "system", FSYSTEM, BLTIN }, { "systime", FSYSTIME, BLTIN }, { "tolower", FTOLOWER, BLTIN }, { "toupper", FTOUPPER, BLTIN }, { "while", WHILE, WHILE }, { "xor", FXOR, BLTIN }, }; #define RET(x) { if(dbg)printf("lex %s\n", tokname(x)); return(x); } static int peek(void) { int c = input(); unput(c); return c; } static int gettok(char **pbuf, int *psz) /* get next input token */ { int c, retc; char *buf = *pbuf; int sz = *psz; char *bp = buf; c = input(); if (c == 0) return 0; buf[0] = c; buf[1] = 0; if (!isalnum(c) && c != '.' && c != '_') return c; *bp++ = c; if (isalpha(c) || c == '_') { /* it's a varname */ for ( ; (c = input()) != 0; ) { if (bp-buf >= sz) if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok")) FATAL( "out of space for name %.10s...", buf ); if (isalnum(c) || c == '_') *bp++ = c; else { *bp = 0; unput(c); break; } } *bp = 0; retc = 'a'; /* alphanumeric */ } else { /* maybe it's a number, but could be . */ char *rem; /* read input until can't be a number */ for ( ; (c = input()) != 0; ) { if (bp-buf >= sz) if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok")) FATAL( "out of space for number %.10s...", buf ); if (isdigit(c) || c == 'e' || c == 'E' || c == '.' || c == '+' || c == '-') *bp++ = c; else { unput(c); break; } } *bp = 0; strtod(buf, &rem); /* parse the number */ if (rem == buf) { /* it wasn't a valid number at all */ buf[1] = 0; /* return one character as token */ retc = (uschar)buf[0]; /* character is its own type */ unputstr(rem+1); /* put rest back for later */ } else { /* some prefix was a number */ unputstr(rem); /* put rest back for later */ rem[0] = 0; /* truncate buf after number part */ retc = '0'; /* type is number */ } } *pbuf = buf; *psz = sz; return retc; } int word(char *); int string(void); int regexpr(void); bool sc = false; /* true => return a } right now */ bool reg = false; /* true => return a REGEXPR now */ int yylex(void) { int c; static char *buf = NULL; static int bufsize = 5; /* BUG: setting this small causes core dump! */ if (buf == NULL && (buf = (char *) malloc(bufsize)) == NULL) FATAL( "out of space in yylex" ); if (sc) { sc = false; RET('}'); } if (reg) { reg = false; return regexpr(); } for (;;) { c = gettok(&buf, &bufsize); if (c == 0) return 0; if (isalpha(c) || c == '_') return word(buf); if (isdigit(c)) { char *cp = tostring(buf); double result; if (is_number(cp, & result)) yylval.cp = setsymtab(buf, cp, result, CON|NUM, symtab); else yylval.cp = setsymtab(buf, cp, 0.0, STR, symtab); free(cp); /* should this also have STR set? */ RET(NUMBER); } yylval.i = c; switch (c) { case '\n': /* {EOL} */ lineno++; RET(NL); case '\r': /* assume \n is coming */ case ' ': /* {WS}+ */ case '\t': break; case '#': /* #.* strip comments */ while ((c = input()) != '\n' && c != 0) ; unput(c); /* * Next line is a hack, itcompensates for * unput's treatment of \n. */ lineno++; break; case ';': RET(';'); case '\\': if (peek() == '\n') { input(); lineno++; } else if (peek() == '\r') { input(); input(); /* \n */ lineno++; } else { RET(c); } break; case '&': if (peek() == '&') { input(); RET(AND); } else RET('&'); case '|': if (peek() == '|') { input(); RET(BOR); } else RET('|'); case '!': if (peek() == '=') { input(); yylval.i = NE; RET(NE); } else if (peek() == '~') { input(); yylval.i = NOTMATCH; RET(MATCHOP); } else RET(NOT); case '~': yylval.i = MATCH; RET(MATCHOP); case '<': if (peek() == '=') { input(); yylval.i = LE; RET(LE); } else { yylval.i = LT; RET(LT); } case '=': if (peek() == '=') { input(); yylval.i = EQ; RET(EQ); } else { yylval.i = ASSIGN; RET(ASGNOP); } case '>': if (peek() == '=') { input(); yylval.i = GE; RET(GE); } else if (peek() == '>') { input(); yylval.i = APPEND; RET(APPEND); } else { yylval.i = GT; RET(GT); } case '+': if (peek() == '+') { input(); yylval.i = INCR; RET(INCR); } else if (peek() == '=') { input(); yylval.i = ADDEQ; RET(ASGNOP); } else RET('+'); case '-': if (peek() == '-') { input(); yylval.i = DECR; RET(DECR); } else if (peek() == '=') { input(); yylval.i = SUBEQ; RET(ASGNOP); } else RET('-'); case '*': if (peek() == '=') { /* *= */ input(); yylval.i = MULTEQ; RET(ASGNOP); } else if (peek() == '*') { /* ** or **= */ input(); /* eat 2nd * */ if (peek() == '=') { input(); yylval.i = POWEQ; RET(ASGNOP); } else { RET(POWER); } } else RET('*'); case '/': RET('/'); case '%': if (peek() == '=') { input(); yylval.i = MODEQ; RET(ASGNOP); } else RET('%'); case '^': if (peek() == '=') { input(); yylval.i = POWEQ; RET(ASGNOP); } else RET(POWER); case '$': /* BUG: awkward, if not wrong */ c = gettok(&buf, &bufsize); if (isalpha(c)) { if (strcmp(buf, "NF") == 0) { /* very special */ unputstr("(NF)"); RET(INDIRECT); } c = peek(); if (c == '(' || c == '[' || (infunc && isarg(buf) >= 0)) { unputstr(buf); RET(INDIRECT); } yylval.cp = setsymtab(buf, "", 0.0, STR|NUM, symtab); RET(IVAR); } else if (c == 0) { /* */ SYNTAX( "unexpected end of input after $" ); RET(';'); } else { unputstr(buf); RET(INDIRECT); } case '}': if (--bracecnt < 0) SYNTAX( "extra }" ); sc = true; RET(';'); case ']': if (--brackcnt < 0) SYNTAX( "extra ]" ); RET(']'); case ')': if (--parencnt < 0) SYNTAX( "extra )" ); RET(')'); case '{': bracecnt++; RET('{'); case '[': brackcnt++; RET('['); case '(': parencnt++; RET('('); case '"': return string(); /* BUG: should be like tran.c ? */ default: RET(c); } } } +extern int runetochar(char *str, int c); + int string(void) { int c, n; char *s, *bp; static char *buf = NULL; static int bufsz = 500; if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL) FATAL("out of space for strings"); for (bp = buf; (c = input()) != '"'; ) { if (!adjbuf(&buf, &bufsz, bp-buf+2, 500, &bp, "string")) FATAL("out of space for string %.10s...", buf); switch (c) { case '\n': case '\r': case 0: *bp = '\0'; SYNTAX( "non-terminated string %.10s...", buf ); if (c == 0) /* hopeless */ FATAL( "giving up" ); lineno++; break; case '\\': c = input(); switch (c) { case '\n': break; case '"': *bp++ = '"'; break; case 'n': *bp++ = '\n'; break; case 't': *bp++ = '\t'; break; case 'f': *bp++ = '\f'; break; case 'r': *bp++ = '\r'; break; case 'b': *bp++ = '\b'; break; case 'v': *bp++ = '\v'; break; case 'a': *bp++ = '\a'; break; case '\\': *bp++ = '\\'; break; case '0': case '1': case '2': /* octal: \d \dd \ddd */ case '3': case '4': case '5': case '6': case '7': n = c - '0'; if ((c = peek()) >= '0' && c < '8') { n = 8 * n + input() - '0'; if ((c = peek()) >= '0' && c < '8') n = 8 * n + input() - '0'; } *bp++ = n; break; - case 'x': /* hex \x0-9a-fA-F + */ - { char xbuf[100], *px; - for (px = xbuf; (c = input()) != 0 && px-xbuf < 100-2; ) { - if (isdigit(c) - || (c >= 'a' && c <= 'f') - || (c >= 'A' && c <= 'F')) - *px++ = c; - else + case 'x': /* hex \x0-9a-fA-F (exactly two) */ + { + int i; + + n = 0; + for (i = 1; i <= 2; i++) { + c = input(); + if (c == 0) + break; + if (isxdigit(c)) { + c = tolower(c); + n *= 16; + if (isdigit(c)) + n += (c - '0'); + else + n += 10 + (c - 'a'); + } else + break; + } + if (n) + *bp++ = n; + else + unput(c); + break; + } + + case 'u': /* utf \u0-9a-fA-F (1..8) */ + { + int i; + + n = 0; + for (i = 0; i < 8; i++) { + c = input(); + if (!isxdigit(c) || c == 0) break; + c = tolower(c); + n *= 16; + if (isdigit(c)) + n += (c - '0'); + else + n += 10 + (c - 'a'); } - *px = 0; unput(c); - sscanf(xbuf, "%x", (unsigned int *) &n); - *bp++ = n; + bp += runetochar(bp, n); break; } default: *bp++ = c; break; } break; default: *bp++ = c; break; } } *bp = 0; s = tostring(buf); *bp++ = ' '; *bp++ = '\0'; yylval.cp = setsymtab(buf, s, 0.0, CON|STR|DONTFREE, symtab); free(s); RET(STRING); } static int binsearch(char *w, const Keyword *kp, int n) { int cond, low, mid, high; low = 0; high = n - 1; while (low <= high) { mid = (low + high) / 2; if ((cond = strcmp(w, kp[mid].word)) < 0) high = mid - 1; else if (cond > 0) low = mid + 1; else return mid; } return -1; } int word(char *w) { const Keyword *kp; int c, n; n = binsearch(w, keywords, sizeof(keywords)/sizeof(keywords[0])); if (n != -1) { /* found in table */ kp = keywords + n; yylval.i = kp->sub; switch (kp->type) { /* special handling */ case BLTIN: if (kp->sub == FSYSTEM && safe) SYNTAX( "system is unsafe" ); RET(kp->type); case FUNC: if (infunc) SYNTAX( "illegal nested function" ); RET(kp->type); case RETURN: if (!infunc) SYNTAX( "return not in function" ); RET(kp->type); case VARNF: yylval.cp = setsymtab("NF", "", 0.0, NUM, symtab); RET(VARNF); default: RET(kp->type); } } c = peek(); /* look for '(' */ if (c != '(' && infunc && (n=isarg(w)) >= 0) { yylval.i = n; RET(ARG); } else { yylval.cp = setsymtab(w, "", 0.0, STR|NUM|DONTFREE, symtab); if (c == '(') { RET(CALL); } else { RET(VAR); } } } void startreg(void) /* next call to yylex will return a regular expression */ { reg = true; } int regexpr(void) { int c; static char *buf = NULL; static int bufsz = 500; char *bp; if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL) - FATAL("out of space for rex expr"); + FATAL("out of space for reg expr"); bp = buf; for ( ; (c = input()) != '/' && c != 0; ) { if (!adjbuf(&buf, &bufsz, bp-buf+3, 500, &bp, "regexpr")) FATAL("out of space for reg expr %.10s...", buf); if (c == '\n') { *bp = '\0'; SYNTAX( "newline in regular expression %.10s...", buf ); unput('\n'); break; } else if (c == '\\') { *bp++ = '\\'; *bp++ = input(); } else { *bp++ = c; } } *bp = 0; if (c == 0) SYNTAX("non-terminated regular expression %.10s...", buf); yylval.s = tostring(buf); unput('/'); RET(REGEXPR); } /* low-level lexical stuff, sort of inherited from lex */ char ebuf[300]; char *ep = ebuf; char yysbuf[100]; /* pushback buffer */ char *yysptr = yysbuf; FILE *yyin = NULL; int input(void) /* get next lexical input character */ { int c; extern char *lexprog; if (yysptr > yysbuf) c = (uschar)*--yysptr; else if (lexprog != NULL) { /* awk '...' */ if ((c = (uschar)*lexprog) != 0) lexprog++; } else /* awk -f ... */ c = pgetc(); if (c == EOF) c = 0; if (ep >= ebuf + sizeof ebuf) ep = ebuf; *ep = c; if (c != 0) { ep++; } return (c); } void unput(int c) /* put lexical character back on input */ { if (c == '\n') lineno--; if (yysptr >= yysbuf + sizeof(yysbuf)) FATAL("pushed back too much: %.20s...", yysbuf); *yysptr++ = c; if (--ep < ebuf) ep = ebuf + sizeof(ebuf) - 1; } void unputstr(const char *s) /* put a string back on input */ { int i; for (i = strlen(s)-1; i >= 0; i--) unput(s[i]); } diff --git a/contrib/one-true-awk/lib.c b/contrib/one-true-awk/lib.c index ee77207564b7..1513b501e655 100644 --- a/contrib/one-true-awk/lib.c +++ b/contrib/one-true-awk/lib.c @@ -1,849 +1,940 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include #include #include #include #include #include #include #include "awk.h" +extern int u8_nextlen(const char *s); + char EMPTY[] = { '\0' }; FILE *infile = NULL; bool innew; /* true = infile has not been read by readrec */ char *file = EMPTY; char *record; int recsize = RECSIZE; char *fields; int fieldssize = RECSIZE; Cell **fldtab; /* pointers to Cells */ static size_t len_inputFS = 0; static char *inputFS = NULL; /* FS at time of input, for field splitting */ #define MAXFLD 2 int nfields = MAXFLD; /* last allocated slot for $i */ bool donefld; /* true = implies rec broken into fields */ bool donerec; /* true = record is valid (no flds have changed) */ int lastfld = 0; /* last used field */ int argno = 1; /* current input argument number */ extern Awkfloat *ARGC; static Cell dollar0 = { OCELL, CFLD, NULL, EMPTY, 0.0, REC|STR|DONTFREE, NULL, NULL }; static Cell dollar1 = { OCELL, CFLD, NULL, EMPTY, 0.0, FLD|STR|DONTFREE, NULL, NULL }; void recinit(unsigned int n) { if ( (record = (char *) malloc(n)) == NULL || (fields = (char *) malloc(n+1)) == NULL || (fldtab = (Cell **) calloc(nfields+2, sizeof(*fldtab))) == NULL || (fldtab[0] = (Cell *) malloc(sizeof(**fldtab))) == NULL) FATAL("out of space for $0 and fields"); *record = '\0'; *fldtab[0] = dollar0; fldtab[0]->sval = record; fldtab[0]->nval = tostring("0"); makefields(1, nfields); } void makefields(int n1, int n2) /* create $n1..$n2 inclusive */ { char temp[50]; int i; for (i = n1; i <= n2; i++) { fldtab[i] = (Cell *) malloc(sizeof(**fldtab)); if (fldtab[i] == NULL) FATAL("out of space in makefields %d", i); *fldtab[i] = dollar1; snprintf(temp, sizeof(temp), "%d", i); fldtab[i]->nval = tostring(temp); } } void initgetrec(void) { int i; char *p; for (i = 1; i < *ARGC; i++) { p = getargv(i); /* find 1st real filename */ if (p == NULL || *p == '\0') { /* deleted or zapped */ argno++; continue; } if (!isclvar(p)) { setsval(lookup("FILENAME", symtab), p); return; } setclvar(p); /* a commandline assignment before filename */ argno++; } infile = stdin; /* no filenames, so use stdin */ innew = true; } /* * POSIX specifies that fields are supposed to be evaluated as if they were * split using the value of FS at the time that the record's value ($0) was * read. * * Since field-splitting is done lazily, we save the current value of FS * whenever a new record is read in (implicitly or via getline), or when * a new value is assigned to $0. */ void savefs(void) { size_t len; if ((len = strlen(getsval(fsloc))) < len_inputFS) { strcpy(inputFS, *FS); /* for subsequent field splitting */ return; } len_inputFS = len + 1; inputFS = (char *) realloc(inputFS, len_inputFS); if (inputFS == NULL) FATAL("field separator %.10s... is too long", *FS); memcpy(inputFS, *FS, len_inputFS); } static bool firsttime = true; int getrec(char **pbuf, int *pbufsize, bool isrecord) /* get next input record */ { /* note: cares whether buf == record */ int c; char *buf = *pbuf; uschar saveb0; int bufsize = *pbufsize, savebufsize = bufsize; if (firsttime) { firsttime = false; initgetrec(); } DPRINTF("RS=<%s>, FS=<%s>, ARGC=%g, FILENAME=%s\n", *RS, *FS, *ARGC, *FILENAME); - if (isrecord) { - donefld = false; - donerec = true; - savefs(); - } saveb0 = buf[0]; buf[0] = 0; while (argno < *ARGC || infile == stdin) { DPRINTF("argno=%d, file=|%s|\n", argno, file); if (infile == NULL) { /* have to open a new file */ file = getargv(argno); if (file == NULL || *file == '\0') { /* deleted or zapped */ argno++; continue; } if (isclvar(file)) { /* a var=value arg */ setclvar(file); argno++; continue; } *FILENAME = file; DPRINTF("opening file %s\n", file); if (*file == '-' && *(file+1) == '\0') infile = stdin; else if ((infile = fopen(file, "r")) == NULL) FATAL("can't open file %s", file); innew = true; setfval(fnrloc, 0.0); } c = readrec(&buf, &bufsize, infile, innew); if (innew) innew = false; if (c != 0 || buf[0] != '\0') { /* normal record */ if (isrecord) { double result; if (freeable(fldtab[0])) xfree(fldtab[0]->sval); fldtab[0]->sval = buf; /* buf == record */ fldtab[0]->tval = REC | STR | DONTFREE; if (is_number(fldtab[0]->sval, & result)) { fldtab[0]->fval = result; fldtab[0]->tval |= NUM; } + donefld = false; + donerec = true; + savefs(); } setfval(nrloc, nrloc->fval+1); setfval(fnrloc, fnrloc->fval+1); *pbuf = buf; *pbufsize = bufsize; return 1; } /* EOF arrived on this file; set up next */ if (infile != stdin) fclose(infile); infile = NULL; argno++; } buf[0] = saveb0; *pbuf = buf; *pbufsize = savebufsize; return 0; /* true end of file */ } void nextfile(void) { if (infile != NULL && infile != stdin) fclose(infile); infile = NULL; argno++; } +extern int readcsvrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag); + int readrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag) /* read one record into buf */ { - int sep, c, isrec; - char *rr, *buf = *pbuf; + int sep, c, isrec; // POTENTIAL BUG? isrec is a macro in awk.h + char *rr = *pbuf, *buf = *pbuf; int bufsize = *pbufsize; char *rs = getsval(rsloc); - if (*rs && rs[1]) { + if (CSV) { + c = readcsvrec(pbuf, pbufsize, inf, newflag); + isrec = (c == EOF && rr == buf) ? false : true; + } else if (*rs && rs[1]) { bool found; + memset(buf, 0, bufsize); fa *pfa = makedfa(rs, 1); if (newflag) found = fnematch(pfa, inf, &buf, &bufsize, recsize); else { int tempstat = pfa->initstat; pfa->initstat = 2; found = fnematch(pfa, inf, &buf, &bufsize, recsize); pfa->initstat = tempstat; } if (found) setptr(patbeg, '\0'); isrec = (found == 0 && *buf == '\0') ? false : true; + } else { if ((sep = *rs) == 0) { sep = '\n'; while ((c=getc(inf)) == '\n' && c != EOF) /* skip leading \n's */ ; if (c != EOF) ungetc(c, inf); } for (rr = buf; ; ) { for (; (c=getc(inf)) != sep && c != EOF; ) { if (rr-buf+1 > bufsize) if (!adjbuf(&buf, &bufsize, 1+rr-buf, recsize, &rr, "readrec 1")) FATAL("input record `%.30s...' too long", buf); *rr++ = c; } if (*rs == sep || c == EOF) break; if ((c = getc(inf)) == '\n' || c == EOF) /* 2 in a row */ break; if (!adjbuf(&buf, &bufsize, 2+rr-buf, recsize, &rr, "readrec 2")) FATAL("input record `%.30s...' too long", buf); *rr++ = '\n'; *rr++ = c; } if (!adjbuf(&buf, &bufsize, 1+rr-buf, recsize, &rr, "readrec 3")) FATAL("input record `%.30s...' too long", buf); *rr = 0; isrec = (c == EOF && rr == buf) ? false : true; } *pbuf = buf; *pbufsize = bufsize; DPRINTF("readrec saw <%s>, returns %d\n", buf, isrec); return isrec; } + +/******************* + * loose ends here: + * \r\n should become \n + * what about bare \r? Excel uses that for embedded newlines + * can't have "" in unquoted fields, according to RFC 4180 +*/ + + +int readcsvrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag) /* csv can have \n's */ +{ /* so read a complete record that might be multiple lines */ + int sep, c; + char *rr = *pbuf, *buf = *pbuf; + int bufsize = *pbufsize; + bool in_quote = false; + + sep = '\n'; /* the only separator; have to skip over \n embedded in "..." */ + rr = buf; + while ((c = getc(inf)) != EOF) { + if (c == sep) { + if (! in_quote) + break; + if (rr > buf && rr[-1] == '\r') // remove \r if was \r\n + rr--; + } + + if (rr-buf+1 > bufsize) + if (!adjbuf(&buf, &bufsize, 1+rr-buf, + recsize, &rr, "readcsvrec 1")) + FATAL("input record `%.30s...' too long", buf); + *rr++ = c; + if (c == '"') + in_quote = ! in_quote; + } + if (c == '\n' && rr > buf && rr[-1] == '\r') // remove \r if was \r\n + rr--; + + if (!adjbuf(&buf, &bufsize, 1+rr-buf, recsize, &rr, "readcsvrec 4")) + FATAL("input record `%.30s...' too long", buf); + *rr = 0; + *pbuf = buf; + *pbufsize = bufsize; + DPRINTF("readcsvrec saw <%s>, returns %d\n", buf, c); + return c; +} + char *getargv(int n) /* get ARGV[n] */ { Cell *x; char *s, temp[50]; extern Array *ARGVtab; snprintf(temp, sizeof(temp), "%d", n); if (lookup(temp, ARGVtab) == NULL) return NULL; x = setsymtab(temp, "", 0.0, STR, ARGVtab); s = getsval(x); DPRINTF("getargv(%d) returns |%s|\n", n, s); return s; } void setclvar(char *s) /* set var=value from s */ { - char *p; + char *e, *p; Cell *q; double result; +/* commit f3d9187d4e0f02294fb1b0e31152070506314e67 broke T.argv test */ +/* I don't understand why it was changed. */ + for (p=s; *p != '='; p++) ; + e = p; *p++ = 0; p = qstring(p, '\0'); q = setsymtab(s, p, 0.0, STR, symtab); setsval(q, p); if (is_number(q->sval, & result)) { q->fval = result; q->tval |= NUM; } DPRINTF("command line set %s to |%s|\n", s, p); + free(p); + *e = '='; } void fldbld(void) /* create fields from current record */ { /* this relies on having fields[] the same length as $0 */ /* the fields are all stored in this one array with \0's */ /* possibly with a final trailing \0 not associated with any field */ char *r, *fr, sep; Cell *p; int i, j, n; if (donefld) return; if (!isstr(fldtab[0])) getsval(fldtab[0]); r = fldtab[0]->sval; n = strlen(r); if (n > fieldssize) { xfree(fields); if ((fields = (char *) malloc(n+2)) == NULL) /* possibly 2 final \0s */ FATAL("out of space for fields in fldbld %d", n); fieldssize = n; } fr = fields; i = 0; /* number of fields accumulated here */ if (inputFS == NULL) /* make sure we have a copy of FS */ savefs(); if (strlen(inputFS) > 1) { /* it's a regular expression */ i = refldbld(r, inputFS); - } else if ((sep = *inputFS) == ' ') { /* default whitespace */ + } else if (!CSV && (sep = *inputFS) == ' ') { /* default whitespace */ for (i = 0; ; ) { while (*r == ' ' || *r == '\t' || *r == '\n') r++; if (*r == 0) break; i++; if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); fldtab[i]->sval = fr; fldtab[i]->tval = FLD | STR | DONTFREE; do *fr++ = *r++; while (*r != ' ' && *r != '\t' && *r != '\n' && *r != '\0'); *fr++ = 0; } *fr = 0; - } else if ((sep = *inputFS) == 0) { /* new: FS="" => 1 char/field */ - for (i = 0; *r != '\0'; r += n) { - char buf[MB_LEN_MAX + 1]; - + } else if (CSV) { /* CSV processing. no error handling */ + if (*r != 0) { + for (;;) { + i++; + if (i > nfields) + growfldtab(i); + if (freeable(fldtab[i])) + xfree(fldtab[i]->sval); + fldtab[i]->sval = fr; + fldtab[i]->tval = FLD | STR | DONTFREE; + if (*r == '"' ) { /* start of "..." */ + for (r++ ; *r != '\0'; ) { + if (*r == '"' && r[1] != '\0' && r[1] == '"') { + r += 2; /* doubled quote */ + *fr++ = '"'; + } else if (*r == '"' && (r[1] == '\0' || r[1] == ',')) { + r++; /* skip over closing quote */ + break; + } else { + *fr++ = *r++; + } + } + *fr++ = 0; + } else { /* unquoted field */ + while (*r != ',' && *r != '\0') + *fr++ = *r++; + *fr++ = 0; + } + if (*r++ == 0) + break; + + } + } + *fr = 0; + } else if ((sep = *inputFS) == 0) { /* new: FS="" => 1 char/field */ + for (i = 0; *r != '\0'; ) { + char buf[10]; i++; if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); - n = mblen(r, MB_LEN_MAX); - if (n < 0) - n = 1; - memcpy(buf, r, n); - buf[n] = '\0'; + n = u8_nextlen(r); + for (j = 0; j < n; j++) + buf[j] = *r++; + buf[j] = '\0'; fldtab[i]->sval = tostring(buf); fldtab[i]->tval = FLD | STR; } *fr = 0; } else if (*r != 0) { /* if 0, it's a null field */ - /* subtlecase : if length(FS) == 1 && length(RS > 0) + /* subtle case: if length(FS) == 1 && length(RS > 0) * \n is NOT a field separator (cf awk book 61,84). * this variable is tested in the inner while loop. */ int rtest = '\n'; /* normal case */ if (strlen(*RS) > 0) rtest = '\0'; for (;;) { i++; if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); fldtab[i]->sval = fr; fldtab[i]->tval = FLD | STR | DONTFREE; while (*r != sep && *r != rtest && *r != '\0') /* \n is always a separator */ *fr++ = *r++; *fr++ = 0; if (*r++ == 0) break; } *fr = 0; } if (i > nfields) FATAL("record `%.30s...' has too many fields; can't happen", r); cleanfld(i+1, lastfld); /* clean out junk from previous record */ lastfld = i; donefld = true; for (j = 1; j <= lastfld; j++) { double result; p = fldtab[j]; if(is_number(p->sval, & result)) { p->fval = result; p->tval |= NUM; } } setfval(nfloc, (Awkfloat) lastfld); donerec = true; /* restore */ if (dbg) { for (j = 0; j <= lastfld; j++) { p = fldtab[j]; printf("field %d (%s): |%s|\n", j, p->nval, p->sval); } } } void cleanfld(int n1, int n2) /* clean out fields n1 .. n2 inclusive */ { /* nvals remain intact */ Cell *p; int i; for (i = n1; i <= n2; i++) { p = fldtab[i]; if (freeable(p)) xfree(p->sval); p->sval = EMPTY, p->tval = FLD | STR | DONTFREE; } } void newfld(int n) /* add field n after end of existing lastfld */ { if (n > nfields) growfldtab(n); cleanfld(lastfld+1, n); lastfld = n; setfval(nfloc, (Awkfloat) n); } void setlastfld(int n) /* set lastfld cleaning fldtab cells if necessary */ { if (n < 0) FATAL("cannot set NF to a negative value"); if (n > nfields) growfldtab(n); if (lastfld < n) cleanfld(lastfld+1, n); else cleanfld(n+1, lastfld); lastfld = n; } Cell *fieldadr(int n) /* get nth field */ { if (n < 0) FATAL("trying to access out of range field %d", n); if (n > nfields) /* fields after NF are empty */ growfldtab(n); /* but does not increase NF */ return(fldtab[n]); } void growfldtab(int n) /* make new fields up to at least $n */ { int nf = 2 * nfields; size_t s; if (n > nf) nf = n; s = (nf+1) * (sizeof (struct Cell *)); /* freebsd: how much do we need? */ if (s / sizeof(struct Cell *) - 1 == (size_t)nf) /* didn't overflow */ fldtab = (Cell **) realloc(fldtab, s); else /* overflow sizeof int */ xfree(fldtab); /* make it null */ if (fldtab == NULL) FATAL("out of space creating %d fields", nf); makefields(nfields+1, nf); nfields = nf; } int refldbld(const char *rec, const char *fs) /* build fields from reg expr in FS */ { /* this relies on having fields[] the same length as $0 */ /* the fields are all stored in this one array with \0's */ char *fr; int i, tempstat, n; fa *pfa; n = strlen(rec); if (n > fieldssize) { xfree(fields); if ((fields = (char *) malloc(n+1)) == NULL) FATAL("out of space for fields in refldbld %d", n); fieldssize = n; } fr = fields; *fr = '\0'; if (*rec == '\0') return 0; pfa = makedfa(fs, 1); DPRINTF("into refldbld, rec = <%s>, pat = <%s>\n", rec, fs); tempstat = pfa->initstat; for (i = 1; ; i++) { if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); fldtab[i]->tval = FLD | STR | DONTFREE; fldtab[i]->sval = fr; DPRINTF("refldbld: i=%d\n", i); if (nematch(pfa, rec)) { pfa->initstat = 2; /* horrible coupling to b.c */ DPRINTF("match %s (%d chars)\n", patbeg, patlen); strncpy(fr, rec, patbeg-rec); fr += patbeg - rec + 1; *(fr-1) = '\0'; rec = patbeg + patlen; } else { DPRINTF("no match %s\n", rec); strcpy(fr, rec); pfa->initstat = tempstat; break; } } return i; } void recbld(void) /* create $0 from $1..$NF if necessary */ { int i; char *r, *p; char *sep = getsval(ofsloc); if (donerec) return; r = record; for (i = 1; i <= *NF; i++) { p = getsval(fldtab[i]); if (!adjbuf(&record, &recsize, 1+strlen(p)+r-record, recsize, &r, "recbld 1")) FATAL("created $0 `%.30s...' too long", record); while ((*r = *p++) != 0) r++; if (i < *NF) { if (!adjbuf(&record, &recsize, 2+strlen(sep)+r-record, recsize, &r, "recbld 2")) FATAL("created $0 `%.30s...' too long", record); for (p = sep; (*r = *p++) != 0; ) r++; } } if (!adjbuf(&record, &recsize, 2+r-record, recsize, &r, "recbld 3")) FATAL("built giant record `%.30s...'", record); *r = '\0'; DPRINTF("in recbld inputFS=%s, fldtab[0]=%p\n", inputFS, (void*)fldtab[0]); if (freeable(fldtab[0])) xfree(fldtab[0]->sval); fldtab[0]->tval = REC | STR | DONTFREE; fldtab[0]->sval = record; DPRINTF("in recbld inputFS=%s, fldtab[0]=%p\n", inputFS, (void*)fldtab[0]); DPRINTF("recbld = |%s|\n", record); donerec = true; } int errorflag = 0; void yyerror(const char *s) { SYNTAX("%s", s); } void SYNTAX(const char *fmt, ...) { extern char *cmdname, *curfname; static int been_here = 0; va_list varg; if (been_here++ > 2) return; fprintf(stderr, "%s: ", cmdname); va_start(varg, fmt); vfprintf(stderr, fmt, varg); va_end(varg); fprintf(stderr, " at source line %d", lineno); if (curfname != NULL) fprintf(stderr, " in function %s", curfname); if (compile_time == COMPILING && cursource() != NULL) fprintf(stderr, " source file %s", cursource()); fprintf(stderr, "\n"); errorflag = 2; eprint(); } extern int bracecnt, brackcnt, parencnt; void bracecheck(void) { int c; static int beenhere = 0; if (beenhere++) return; while ((c = input()) != EOF && c != '\0') bclass(c); bcheck2(bracecnt, '{', '}'); bcheck2(brackcnt, '[', ']'); bcheck2(parencnt, '(', ')'); } void bcheck2(int n, int c1, int c2) { if (n == 1) fprintf(stderr, "\tmissing %c\n", c2); else if (n > 1) fprintf(stderr, "\t%d missing %c's\n", n, c2); else if (n == -1) fprintf(stderr, "\textra %c\n", c2); else if (n < -1) fprintf(stderr, "\t%d extra %c's\n", -n, c2); } void FATAL(const char *fmt, ...) { extern char *cmdname; va_list varg; fflush(stdout); fprintf(stderr, "%s: ", cmdname); va_start(varg, fmt); vfprintf(stderr, fmt, varg); va_end(varg); error(); if (dbg > 1) /* core dump if serious debugging on */ abort(); exit(2); } void WARNING(const char *fmt, ...) { extern char *cmdname; va_list varg; fflush(stdout); fprintf(stderr, "%s: ", cmdname); va_start(varg, fmt); vfprintf(stderr, fmt, varg); va_end(varg); error(); } void error() { extern Node *curnode; fprintf(stderr, "\n"); if (compile_time != ERROR_PRINTING) { if (NR && *NR > 0) { fprintf(stderr, " input record number %d", (int) (*FNR)); if (strcmp(*FILENAME, "-") != 0) fprintf(stderr, ", file %s", *FILENAME); fprintf(stderr, "\n"); } if (curnode) fprintf(stderr, " source line number %d", curnode->lineno); else if (lineno) fprintf(stderr, " source line number %d", lineno); if (compile_time == COMPILING && cursource() != NULL) fprintf(stderr, " source file %s", cursource()); fprintf(stderr, "\n"); eprint(); } } void eprint(void) /* try to print context around error */ { char *p, *q; int c; static int been_here = 0; extern char ebuf[], *ep; if (compile_time != COMPILING || been_here++ > 0 || ebuf == ep) return; if (ebuf == ep) return; p = ep - 1; if (p > ebuf && *p == '\n') p--; for ( ; p > ebuf && *p != '\n' && *p != '\0'; p--) ; while (*p == '\n') p++; fprintf(stderr, " context is\n\t"); for (q=ep-1; q>=p && *q!=' ' && *q!='\t' && *q!='\n'; q--) ; for ( ; p < q; p++) if (*p) putc(*p, stderr); fprintf(stderr, " >>> "); for ( ; p < ep; p++) if (*p) putc(*p, stderr); fprintf(stderr, " <<< "); if (*ep) while ((c = input()) != '\n' && c != '\0' && c != EOF) { putc(c, stderr); bclass(c); } putc('\n', stderr); ep = ebuf; } void bclass(int c) { switch (c) { case '{': bracecnt++; break; case '}': bracecnt--; break; case '[': brackcnt++; break; case ']': brackcnt--; break; case '(': parencnt++; break; case ')': parencnt--; break; } } double errcheck(double x, const char *s) { if (errno == EDOM) { errno = 0; WARNING("%s argument out of domain", s); x = 1; } else if (errno == ERANGE) { errno = 0; WARNING("%s result out of range", s); x = 1; } return x; } int isclvar(const char *s) /* is s of form var=something ? */ { const char *os = s; if (!isalpha((uschar) *s) && *s != '_') return 0; for ( ; *s; s++) if (!(isalnum((uschar) *s) || *s == '_')) break; return *s == '=' && s > os; } /* strtod is supposed to be a proper test of what's a valid number */ /* appears to be broken in gcc on linux: thinks 0x123 is a valid FP number */ /* wrong: violates 4.10.1.4 of ansi C standard */ /* well, not quite. As of C99, hex floating point is allowed. so this is * a bit of a mess. We work around the mess by checking for a hexadecimal * value and disallowing it. Similarly, we now follow gawk and allow only * +nan, -nan, +inf, and -inf for NaN and infinity values. */ /* * This routine now has a more complicated interface, the main point * being to avoid the double conversion of a string to double, and * also to convey out, if requested, the information that the numeric * value was a leading string or is all of the string. The latter bit * is used in getfval(). */ bool is_valid_number(const char *s, bool trailing_stuff_ok, bool *no_trailing, double *result) { double r; char *ep; bool retval = false; bool is_nan = false; bool is_inf = false; if (no_trailing) *no_trailing = false; while (isspace(*s)) s++; /* * This test, while allowed by newer POSIX standards, represents a regression * where hex strings were treated as numbers in nawk the whole time it has been * in FreeBSD (since 2001). The POSIX 2001 through 2004 standards mandated this * behavior and the current standard allows it. Deviate from upstream by restoring * the prior FreeBSD behavior. */ #if 0 - // no hex floating point, sorry + /* no hex floating point, sorry */ if (s[0] == '0' && tolower(s[1]) == 'x') return false; #endif - // allow +nan, -nan, +inf, -inf, any other letter, no + /* allow +nan, -nan, +inf, -inf, any other letter, no */ if (s[0] == '+' || s[0] == '-') { is_nan = (strncasecmp(s+1, "nan", 3) == 0); is_inf = (strncasecmp(s+1, "inf", 3) == 0); if ((is_nan || is_inf) && (isspace(s[4]) || s[4] == '\0')) goto convert; else if (! isdigit(s[1]) && s[1] != '.') return false; } else if (! isdigit(s[0]) && s[0] != '.') return false; convert: errno = 0; r = strtod(s, &ep); if (ep == s || errno == ERANGE) return false; if (isnan(r) && s[0] == '-' && signbit(r) == 0) r = -r; if (result != NULL) *result = r; /* * check for trailing stuff */ while (isspace(*ep)) ep++; if (no_trailing != NULL) *no_trailing = (*ep == '\0'); - // return true if found the end, or trailing stuff is allowed + /* return true if found the end, or trailing stuff is allowed */ retval = *ep == '\0' || trailing_stuff_ok; return retval; } diff --git a/contrib/one-true-awk/main.c b/contrib/one-true-awk/main.c index daa030000a51..6404b1d0c646 100644 --- a/contrib/one-true-awk/main.c +++ b/contrib/one-true-awk/main.c @@ -1,259 +1,270 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ -const char *version = "version 20210724"; +const char *version = "version 20231030"; #define DEBUG #include #include #include #include #include #include #include "awk.h" extern char **environ; extern int nfields; int dbg = 0; Awkfloat srand_seed = 1; char *cmdname; /* gets argv[0] for error messages */ extern FILE *yyin; /* lex input file */ char *lexprog; /* points to program argument if it exists */ extern int errorflag; /* non-zero if any syntax errors; set by yyerror */ enum compile_states compile_time = ERROR_PRINTING; static char **pfile; /* program filenames from -f's */ static size_t maxpfile; /* max program filename */ static size_t npfile; /* number of filenames */ static size_t curpfile; /* current filename */ +bool CSV = false; /* true for csv input */ + bool safe = false; /* true => "safe" mode */ +size_t awk_mb_cur_max = 1; + static noreturn void fpecatch(int n #ifdef SA_SIGINFO , siginfo_t *si, void *uc #endif ) { #ifdef SA_SIGINFO static const char *emsg[] = { [0] = "Unknown error", [FPE_INTDIV] = "Integer divide by zero", [FPE_INTOVF] = "Integer overflow", [FPE_FLTDIV] = "Floating point divide by zero", [FPE_FLTOVF] = "Floating point overflow", [FPE_FLTUND] = "Floating point underflow", [FPE_FLTRES] = "Floating point inexact result", [FPE_FLTINV] = "Invalid Floating point operation", [FPE_FLTSUB] = "Subscript out of range", }; #endif FATAL("floating point exception" #ifdef SA_SIGINFO ": %s", (size_t)si->si_code < sizeof(emsg) / sizeof(emsg[0]) && emsg[si->si_code] ? emsg[si->si_code] : emsg[0] #endif ); } /* Can this work with recursive calls? I don't think so. void segvcatch(int n) { FATAL("segfault. Do you have an unbounded recursive call?", n); } */ static const char * setfs(char *p) { /* wart: t=>\t */ if (p[0] == 't' && p[1] == '\0') { WARNING("-Ft to imply tab separator is deprecated behavior."); return "\t"; } return p; } static char * getarg(int *argc, char ***argv, const char *msg) { if ((*argv)[1][2] != '\0') { /* arg is -fsomething */ return &(*argv)[1][2]; } else { /* arg is -f something */ (*argc)--; (*argv)++; if (*argc <= 1) FATAL("%s", msg); return (*argv)[1]; } } int main(int argc, char *argv[]) { const char *fs = NULL; char *fn, *vn; setlocale(LC_CTYPE, ""); setlocale(LC_NUMERIC, "C"); /* for parsing cmdline & prog */ + awk_mb_cur_max = MB_CUR_MAX; cmdname = argv[0]; if (argc == 1) { fprintf(stderr, - "usage: %s [-F fs] [-v var=value] [-f progfile | 'prog'] [file ...]\n", + "usage: %s [-F fs | --csv] [-v var=value] [-f progfile | 'prog'] [file ...]\n", cmdname); exit(1); } #ifdef SA_SIGINFO { struct sigaction sa; sa.sa_sigaction = fpecatch; sa.sa_flags = SA_SIGINFO; sigemptyset(&sa.sa_mask); (void)sigaction(SIGFPE, &sa, NULL); } #else (void)signal(SIGFPE, fpecatch); #endif /*signal(SIGSEGV, segvcatch); experiment */ /* Set and keep track of the random seed */ srand_seed = 1; srandom((unsigned long) srand_seed); yyin = NULL; symtab = makesymtab(NSYMTAB/NSYMTAB); while (argc > 1 && argv[1][0] == '-' && argv[1][1] != '\0') { if (strcmp(argv[1], "-version") == 0 || strcmp(argv[1], "--version") == 0) { printf("awk %s\n", version); return 0; } if (strcmp(argv[1], "--") == 0) { /* explicit end of args */ argc--; argv++; break; } + if (strcmp(argv[1], "--csv") == 0) { /* turn on csv input processing */ + CSV = true; + argc--; + argv++; + continue; + } switch (argv[1][1]) { case 's': if (strcmp(argv[1], "-safe") == 0) safe = true; break; case 'f': /* next argument is program filename */ fn = getarg(&argc, &argv, "no program filename"); if (npfile >= maxpfile) { maxpfile += 20; pfile = (char **) realloc(pfile, maxpfile * sizeof(*pfile)); if (pfile == NULL) FATAL("error allocating space for -f options"); } pfile[npfile++] = fn; break; case 'F': /* set field separator */ fs = setfs(getarg(&argc, &argv, "no field separator")); break; case 'v': /* -v a=1 to be done NOW. one -v for each */ vn = getarg(&argc, &argv, "no variable name"); if (isclvar(vn)) setclvar(vn); else FATAL("invalid -v option argument: %s", vn); break; case 'd': dbg = atoi(&argv[1][2]); if (dbg == 0) dbg = 1; printf("awk %s\n", version); break; default: WARNING("unknown option %s ignored", argv[1]); break; } argc--; argv++; } /* argv[1] is now the first argument */ if (npfile == 0) { /* no -f; first argument is program */ if (argc <= 1) { if (dbg) exit(0); FATAL("no program given"); } DPRINTF("program = |%s|\n", argv[1]); lexprog = argv[1]; argc--; argv++; } recinit(recsize); syminit(); compile_time = COMPILING; argv[0] = cmdname; /* put prog name at front of arglist */ DPRINTF("argc=%d, argv[0]=%s\n", argc, argv[0]); arginit(argc, argv); if (!safe) envinit(environ); yyparse(); #if 0 // Doing this would comply with POSIX, but is not compatible with // other awks and with what most users expect. So comment it out. setlocale(LC_NUMERIC, ""); /* back to whatever it is locally */ #endif if (fs) *FS = qstring(fs, '\0'); DPRINTF("errorflag=%d\n", errorflag); if (errorflag == 0) { compile_time = RUNNING; run(winner); } else bracecheck(); return(errorflag); } int pgetc(void) /* get 1 character from awk program */ { int c; for (;;) { if (yyin == NULL) { if (curpfile >= npfile) return EOF; if (strcmp(pfile[curpfile], "-") == 0) yyin = stdin; else if ((yyin = fopen(pfile[curpfile], "r")) == NULL) FATAL("can't open file %s", pfile[curpfile]); lineno = 1; } if ((c = getc(yyin)) != EOF) return c; if (yyin != stdin) fclose(yyin); yyin = NULL; curpfile++; } } char *cursource(void) /* current source file name */ { if (npfile > 0) return pfile[curpfile < npfile ? curpfile : curpfile - 1]; else return NULL; } diff --git a/contrib/one-true-awk/makefile b/contrib/one-true-awk/makefile index 9ceaaad48af1..df966ef1b846 100644 --- a/contrib/one-true-awk/makefile +++ b/contrib/one-true-awk/makefile @@ -1,111 +1,112 @@ # /**************************************************************** # Copyright (C) Lucent Technologies 1997 # All Rights Reserved # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that the copyright notice and this # permission notice and warranty disclaimer appear in supporting # documentation, and that the name Lucent Technologies or any of # its entities not be used in advertising or publicity pertaining # to distribution of the software without specific, written prior # permission. # # LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. # IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY # SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER # IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, # ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF # THIS SOFTWARE. # ****************************************************************/ CFLAGS = -fsanitize=address -O1 -g -fno-omit-frame-pointer -fno-optimize-sibling-calls CFLAGS = -g CFLAGS = CFLAGS = -O2 # compiler options #CC = gcc -Wall -g -Wwrite-strings #CC = gcc -O4 -Wall -pedantic -fno-strict-aliasing #CC = gcc -fprofile-arcs -ftest-coverage # then gcov f1.c; cat f1.c.gcov HOSTCC = gcc -g -Wall -pedantic -Wcast-qual CC = $(HOSTCC) # change this is cross-compiling. # By fiat, to make our lives easier, yacc is now defined to be bison. # If you want something else, you're on your own. +# YACC = yacc -d -b awkgram YACC = bison -d OFILES = b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o SOURCE = awk.h awkgram.tab.c awkgram.tab.h proto.h awkgram.y lex.c b.c main.c \ maketab.c parse.c lib.c run.c tran.c proctab.c LISTING = awk.h proto.h awkgram.y lex.c b.c main.c maketab.c parse.c \ lib.c run.c tran.c SHIP = README LICENSE FIXES $(SOURCE) awkgram.tab.[ch].bak makefile \ awk.1 a.out: awkgram.tab.o $(OFILES) $(CC) $(CFLAGS) awkgram.tab.o $(OFILES) $(ALLOC) -lm $(OFILES): awk.h awkgram.tab.h proto.h awkgram.tab.c awkgram.tab.h: awk.h proto.h awkgram.y $(YACC) $(YFLAGS) awkgram.y proctab.c: maketab ./maketab awkgram.tab.h >proctab.c maketab: awkgram.tab.h maketab.c $(HOSTCC) $(CFLAGS) maketab.c -o maketab bundle: @cp awkgram.tab.h awkgram.tab.h.bak @cp awkgram.tab.c awkgram.tab.c.bak @bundle $(SHIP) tar: @cp awkgram.tab.h awkgram.tab.h.bak @cp awkgram.tab.c awkgram.tab.c.bak @bundle $(SHIP) >awk.shar @tar cf awk.tar $(SHIP) gzip awk.tar ls -l awk.tar.gz @zip awk.zip $(SHIP) ls -l awk.zip gitadd: git add README LICENSE FIXES \ awk.h proto.h awkgram.y lex.c b.c main.c maketab.c parse.c \ lib.c run.c tran.c \ makefile awk.1 testdir gitpush: # only do this once: # git remote add origin https://github.com/onetrueawk/awk.git git push -u origin master names: @echo $(LISTING) test check: ./REGRESS clean: testclean rm -f a.out *.o *.obj maketab maketab.exe *.bb *.bbg *.da *.gcov *.gcno *.gcda # proctab.c cleaner: testclean rm -f a.out *.o *.obj maketab maketab.exe *.bb *.bbg *.da *.gcov *.gcno *.gcda proctab.c awkgram.tab.* # This is a bit of a band-aid until we can invest some more time # in the test suite. testclean: cd testdir; rm -fr arnold-fixes beebe devnull echo foo* \ glop glop1 glop2 lilly.diff tempbig tempsmall time # For the habits of GNU maintainers: distclean: cleaner diff --git a/contrib/one-true-awk/parse.c b/contrib/one-true-awk/parse.c index 79b8fade83b5..2b7fd1928930 100644 --- a/contrib/one-true-awk/parse.c +++ b/contrib/one-true-awk/parse.c @@ -1,299 +1,299 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include #include "awk.h" #include "awkgram.tab.h" -Node *nodealloc(int n) +Node *nodealloc(size_t n) { Node *x; x = (Node *) malloc(sizeof(*x) + (n-1) * sizeof(x)); if (x == NULL) FATAL("out of space in nodealloc"); x->nnext = NULL; x->lineno = lineno; return(x); } Node *exptostat(Node *a) { a->ntype = NSTAT; return(a); } Node *node1(int a, Node *b) { Node *x; x = nodealloc(1); x->nobj = a; x->narg[0]=b; return(x); } Node *node2(int a, Node *b, Node *c) { Node *x; x = nodealloc(2); x->nobj = a; x->narg[0] = b; x->narg[1] = c; return(x); } Node *node3(int a, Node *b, Node *c, Node *d) { Node *x; x = nodealloc(3); x->nobj = a; x->narg[0] = b; x->narg[1] = c; x->narg[2] = d; return(x); } Node *node4(int a, Node *b, Node *c, Node *d, Node *e) { Node *x; x = nodealloc(4); x->nobj = a; x->narg[0] = b; x->narg[1] = c; x->narg[2] = d; x->narg[3] = e; return(x); } Node *node5(int a, Node *b, Node *c, Node *d, Node *e, Node *f) { Node *x; x = nodealloc(5); x->nobj = a; x->narg[0] = b; x->narg[1] = c; x->narg[2] = d; x->narg[3] = e; x->narg[4] = f; return(x); } Node *stat1(int a, Node *b) { Node *x; x = node1(a,b); x->ntype = NSTAT; return(x); } Node *stat2(int a, Node *b, Node *c) { Node *x; x = node2(a,b,c); x->ntype = NSTAT; return(x); } Node *stat3(int a, Node *b, Node *c, Node *d) { Node *x; x = node3(a,b,c,d); x->ntype = NSTAT; return(x); } Node *stat4(int a, Node *b, Node *c, Node *d, Node *e) { Node *x; x = node4(a,b,c,d,e); x->ntype = NSTAT; return(x); } Node *op1(int a, Node *b) { Node *x; x = node1(a,b); x->ntype = NEXPR; return(x); } Node *op2(int a, Node *b, Node *c) { Node *x; x = node2(a,b,c); x->ntype = NEXPR; return(x); } Node *op3(int a, Node *b, Node *c, Node *d) { Node *x; x = node3(a,b,c,d); x->ntype = NEXPR; return(x); } Node *op4(int a, Node *b, Node *c, Node *d, Node *e) { Node *x; x = node4(a,b,c,d,e); x->ntype = NEXPR; return(x); } Node *op5(int a, Node *b, Node *c, Node *d, Node *e, Node *f) { Node *x; x = node5(a,b,c,d,e,f); x->ntype = NEXPR; return(x); } Node *celltonode(Cell *a, int b) { Node *x; a->ctype = OCELL; a->csub = b; x = node1(0, (Node *) a); x->ntype = NVALUE; return(x); } Node *rectonode(void) /* make $0 into a Node */ { extern Cell *literal0; return op1(INDIRECT, celltonode(literal0, CUNK)); } Node *makearr(Node *p) { Cell *cp; if (isvalue(p)) { cp = (Cell *) (p->narg[0]); if (isfcn(cp)) SYNTAX( "%s is a function, not an array", cp->nval ); else if (!isarr(cp)) { xfree(cp->sval); cp->sval = (char *) makesymtab(NSYMTAB); cp->tval = ARR; } } return p; } #define PA2NUM 50 /* max number of pat,pat patterns allowed */ int paircnt; /* number of them in use */ int pairstack[PA2NUM]; /* state of each pat,pat */ Node *pa2stat(Node *a, Node *b, Node *c) /* pat, pat {...} */ { Node *x; x = node4(PASTAT2, a, b, c, itonp(paircnt)); if (paircnt++ >= PA2NUM) SYNTAX( "limited to %d pat,pat statements", PA2NUM ); x->ntype = NSTAT; return(x); } Node *linkum(Node *a, Node *b) { Node *c; if (errorflag) /* don't link things that are wrong */ return a; if (a == NULL) return(b); else if (b == NULL) return(a); for (c = a; c->nnext != NULL; c = c->nnext) ; c->nnext = b; return(a); } void defn(Cell *v, Node *vl, Node *st) /* turn on FCN bit in definition, */ { /* body of function, arglist */ Node *p; int n; if (isarr(v)) { SYNTAX( "`%s' is an array name and a function name", v->nval ); return; } if (isarg(v->nval) != -1) { SYNTAX( "`%s' is both function name and argument name", v->nval ); return; } v->tval = FCN; v->sval = (char *) st; n = 0; /* count arguments */ for (p = vl; p; p = p->nnext) n++; v->fval = n; DPRINTF("defining func %s (%d args)\n", v->nval, n); } int isarg(const char *s) /* is s in argument list for current function? */ { /* return -1 if not, otherwise arg # */ extern Node *arglist; Node *p = arglist; int n; for (n = 0; p != NULL; p = p->nnext, n++) if (strcmp(((Cell *)(p->narg[0]))->nval, s) == 0) return n; return -1; } int ptoi(void *p) /* convert pointer to integer */ { return (int) (long) p; /* swearing that p fits, of course */ } Node *itonp(int i) /* and vice versa */ { return (Node *) (long) i; } diff --git a/contrib/one-true-awk/proto.h b/contrib/one-true-awk/proto.h index 5cd3afb8cf85..879fbde492b5 100644 --- a/contrib/one-true-awk/proto.h +++ b/contrib/one-true-awk/proto.h @@ -1,209 +1,208 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ extern int yywrap(void); extern void setfname(Cell *); extern int constnode(Node *); extern char *strnode(Node *); extern Node *notnull(Node *); extern int yyparse(void); extern int yylex(void); extern void startreg(void); extern int input(void); extern void unput(int); extern void unputstr(const char *); extern int yylook(void); extern int yyback(int *, int); extern int yyinput(void); extern fa *makedfa(const char *, bool); extern fa *mkdfa(const char *, bool); extern int makeinit(fa *, bool); extern void penter(Node *); extern void freetr(Node *); -extern int hexstr(const uschar **); extern int quoted(const uschar **); -extern char *cclenter(const char *); +extern int *cclenter(const char *); extern noreturn void overflo(const char *); extern void cfoll(fa *, Node *); extern int first(Node *); extern void follow(Node *); -extern int member(int, const char *); +extern int member(int, int *); extern int match(fa *, const char *); extern int pmatch(fa *, const char *); extern int nematch(fa *, const char *); extern bool fnematch(fa *, FILE *, char **, int *, int); extern Node *reparse(const char *); extern Node *regexp(void); extern Node *primary(void); extern Node *concat(Node *); extern Node *alt(Node *); extern Node *unary(Node *); extern int relex(void); extern int cgoto(fa *, int, int); extern void freefa(fa *); extern int pgetc(void); extern char *cursource(void); -extern Node *nodealloc(int); +extern Node *nodealloc(size_t); extern Node *exptostat(Node *); extern Node *node1(int, Node *); extern Node *node2(int, Node *, Node *); extern Node *node3(int, Node *, Node *, Node *); extern Node *node4(int, Node *, Node *, Node *, Node *); extern Node *node5(int, Node *, Node *, Node *, Node *, Node *); extern Node *stat3(int, Node *, Node *, Node *); extern Node *op2(int, Node *, Node *); extern Node *op1(int, Node *); extern Node *stat1(int, Node *); extern Node *op3(int, Node *, Node *, Node *); extern Node *op4(int, Node *, Node *, Node *, Node *); extern Node *op5(int, Node *, Node *, Node *, Node *, Node *); extern Node *stat2(int, Node *, Node *); extern Node *stat4(int, Node *, Node *, Node *, Node *); extern Node *celltonode(Cell *, int); extern Node *rectonode(void); extern Node *makearr(Node *); extern Node *pa2stat(Node *, Node *, Node *); extern Node *linkum(Node *, Node *); extern void defn(Cell *, Node *, Node *); extern int isarg(const char *); extern const char *tokname(int); extern Cell *(*proctab[])(Node **, int); extern int ptoi(void *); extern Node *itonp(int); extern void syminit(void); extern void arginit(int, char **); extern void envinit(char **); extern Array *makesymtab(int); extern void freesymtab(Cell *); extern void freeelem(Cell *, const char *); extern Cell *setsymtab(const char *, const char *, double, unsigned int, Array *); extern int hash(const char *, int); extern void rehash(Array *); extern Cell *lookup(const char *, Array *); extern double setfval(Cell *, double); extern void funnyvar(Cell *, const char *); extern char *setsval(Cell *, const char *); extern double getfval(Cell *); extern char *getsval(Cell *); extern char *getpssval(Cell *); /* for print */ extern char *tostring(const char *); extern char *tostringN(const char *, size_t); extern char *qstring(const char *, int); extern Cell *catstr(Cell *, Cell *); extern void recinit(unsigned int); extern void initgetrec(void); extern void makefields(int, int); extern void growfldtab(int n); extern void savefs(void); extern int getrec(char **, int *, bool); extern void nextfile(void); extern int readrec(char **buf, int *bufsize, FILE *inf, bool isnew); extern char *getargv(int); extern void setclvar(char *); extern void fldbld(void); extern void cleanfld(int, int); extern void newfld(int); extern void setlastfld(int); extern int refldbld(const char *, const char *); extern void recbld(void); extern Cell *fieldadr(int); extern void yyerror(const char *); extern void bracecheck(void); extern void bcheck2(int, int, int); extern void SYNTAX(const char *, ...) __attribute__((__format__(__printf__, 1, 2))); extern noreturn void FATAL(const char *, ...) __attribute__((__format__(__printf__, 1, 2))); extern void WARNING(const char *, ...) __attribute__((__format__(__printf__, 1, 2))); extern void error(void); extern void eprint(void); extern void bclass(int); extern double errcheck(double, const char *); extern int isclvar(const char *); extern bool is_valid_number(const char *s, bool trailing_stuff_ok, bool *no_trailing, double *result); #define is_number(s, val) is_valid_number(s, false, NULL, val) extern int adjbuf(char **pb, int *sz, int min, int q, char **pbp, const char *what); extern void run(Node *); extern Cell *execute(Node *); extern Cell *program(Node **, int); extern Cell *call(Node **, int); extern Cell *copycell(Cell *); extern Cell *arg(Node **, int); extern Cell *jump(Node **, int); extern Cell *awkgetline(Node **, int); extern Cell *getnf(Node **, int); extern Cell *array(Node **, int); extern Cell *awkdelete(Node **, int); extern Cell *intest(Node **, int); extern Cell *matchop(Node **, int); extern Cell *boolop(Node **, int); extern Cell *relop(Node **, int); extern void tfree(Cell *); extern Cell *gettemp(void); extern Cell *field(Node **, int); extern Cell *indirect(Node **, int); extern Cell *substr(Node **, int); extern Cell *sindex(Node **, int); extern int format(char **, int *, const char *, Node *); extern Cell *awksprintf(Node **, int); extern Cell *awkprintf(Node **, int); extern Cell *arith(Node **, int); extern double ipow(double, int); extern Cell *incrdecr(Node **, int); extern Cell *assign(Node **, int); extern Cell *cat(Node **, int); extern Cell *pastat(Node **, int); extern Cell *dopa2(Node **, int); extern Cell *split(Node **, int); extern Cell *condexpr(Node **, int); extern Cell *ifstat(Node **, int); extern Cell *whilestat(Node **, int); extern Cell *dostat(Node **, int); extern Cell *forstat(Node **, int); extern Cell *instat(Node **, int); extern Cell *bltin(Node **, int); extern Cell *printstat(Node **, int); extern Cell *nullproc(Node **, int); extern FILE *redirect(int, Node *); extern FILE *openfile(int, const char *, bool *); extern const char *filename(FILE *); extern Cell *closefile(Node **, int); extern void closeall(void); extern Cell *sub(Node **, int); extern Cell *gsub(Node **, int); extern Cell *gensub(Node **, int); extern FILE *popen(const char *, const char *); extern int pclose(FILE *); extern const char *flags2str(int flags); diff --git a/contrib/one-true-awk/run.c b/contrib/one-true-awk/run.c index b81dafba35e6..f75dad03053e 100644 --- a/contrib/one-true-awk/run.c +++ b/contrib/one-true-awk/run.c @@ -1,2366 +1,2877 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include -#include #include #include #include #include #include #include #include #include #include #include #include "awk.h" #include "awkgram.tab.h" + static void stdinit(void); static void flush_all(void); +static char *wide_char_to_byte_str(int rune, size_t *outlen); #if 1 #define tempfree(x) do { if (istemp(x)) tfree(x); } while (/*CONSTCOND*/0) #else void tempfree(Cell *p) { if (p->ctype == OCELL && (p->csub < CUNK || p->csub > CFREE)) { WARNING("bad csub %d in Cell %d %s", p->csub, p->ctype, p->sval); } if (istemp(p)) tfree(p); } #endif /* do we really need these? */ /* #ifdef _NFILE */ /* #ifndef FOPEN_MAX */ /* #define FOPEN_MAX _NFILE */ /* #endif */ /* #endif */ /* */ /* #ifndef FOPEN_MAX */ /* #define FOPEN_MAX 40 */ /* max number of open files */ /* #endif */ /* */ /* #ifndef RAND_MAX */ /* #define RAND_MAX 32767 */ /* all that ansi guarantees */ /* #endif */ jmp_buf env; extern int pairstack[]; extern Awkfloat srand_seed; Node *winner = NULL; /* root of parse tree */ Cell *tmps; /* free temporary cells for execution */ static Cell truecell ={ OBOOL, BTRUE, 0, 0, 1.0, NUM, NULL, NULL }; Cell *True = &truecell; static Cell falsecell ={ OBOOL, BFALSE, 0, 0, 0.0, NUM, NULL, NULL }; Cell *False = &falsecell; static Cell breakcell ={ OJUMP, JBREAK, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jbreak = &breakcell; static Cell contcell ={ OJUMP, JCONT, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jcont = &contcell; static Cell nextcell ={ OJUMP, JNEXT, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jnext = &nextcell; static Cell nextfilecell ={ OJUMP, JNEXTFILE, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jnextfile = &nextfilecell; static Cell exitcell ={ OJUMP, JEXIT, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jexit = &exitcell; static Cell retcell ={ OJUMP, JRET, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jret = &retcell; static Cell tempcell ={ OCELL, CTEMP, 0, EMPTY, 0.0, NUM|STR|DONTFREE, NULL, NULL }; Node *curnode = NULL; /* the node being executed, for debugging */ /* buffer memory management */ int adjbuf(char **pbuf, int *psiz, int minlen, int quantum, char **pbptr, const char *whatrtn) /* pbuf: address of pointer to buffer being managed * psiz: address of buffer size variable * minlen: minimum length of buffer needed * quantum: buffer size quantum * pbptr: address of movable pointer into buffer, or 0 if none * whatrtn: name of the calling routine if failure should cause fatal error * * return 0 for realloc failure, !=0 for success */ { if (minlen > *psiz) { char *tbuf; int rminlen = quantum ? minlen % quantum : 0; int boff = pbptr ? *pbptr - *pbuf : 0; /* round up to next multiple of quantum */ if (rminlen) minlen += quantum - rminlen; tbuf = (char *) realloc(*pbuf, minlen); DPRINTF("adjbuf %s: %d %d (pbuf=%p, tbuf=%p)\n", whatrtn, *psiz, minlen, (void*)*pbuf, (void*)tbuf); if (tbuf == NULL) { if (whatrtn) FATAL("out of memory in %s", whatrtn); return 0; } *pbuf = tbuf; *psiz = minlen; if (pbptr) *pbptr = tbuf + boff; } return 1; } void run(Node *a) /* execution of parse tree starts here */ { stdinit(); execute(a); closeall(); } Cell *execute(Node *u) /* execute a node of the parse tree */ { Cell *(*proc)(Node **, int); Cell *x; Node *a; if (u == NULL) return(True); for (a = u; ; a = a->nnext) { curnode = a; if (isvalue(a)) { x = (Cell *) (a->narg[0]); if (isfld(x) && !donefld) fldbld(); else if (isrec(x) && !donerec) recbld(); return(x); } if (notlegal(a->nobj)) /* probably a Cell* but too risky to print */ FATAL("illegal statement"); proc = proctab[a->nobj-FIRSTTOKEN]; x = (*proc)(a->narg, a->nobj); if (isfld(x) && !donefld) fldbld(); else if (isrec(x) && !donerec) recbld(); if (isexpr(a)) return(x); if (isjump(x)) return(x); if (a->nnext == NULL) return(x); tempfree(x); } } Cell *program(Node **a, int n) /* execute an awk program */ { /* a[0] = BEGIN, a[1] = body, a[2] = END */ Cell *x; if (setjmp(env) != 0) goto ex; if (a[0]) { /* BEGIN */ x = execute(a[0]); if (isexit(x)) return(True); if (isjump(x)) FATAL("illegal break, continue, next or nextfile from BEGIN"); tempfree(x); } if (a[1] || a[2]) while (getrec(&record, &recsize, true) > 0) { x = execute(a[1]); if (isexit(x)) break; tempfree(x); } ex: if (setjmp(env) != 0) /* handles exit within END */ goto ex1; if (a[2]) { /* END */ x = execute(a[2]); if (isbreak(x) || isnext(x) || iscont(x)) FATAL("illegal break, continue, next or nextfile from END"); tempfree(x); } ex1: return(True); } struct Frame { /* stack frame for awk function calls */ int nargs; /* number of arguments in this call */ Cell *fcncell; /* pointer to Cell for function */ Cell **args; /* pointer to array of arguments after execute */ Cell *retval; /* return value */ }; #define NARGS 50 /* max args in a call */ struct Frame *frame = NULL; /* base of stack frames; dynamically allocated */ int nframe = 0; /* number of frames allocated */ struct Frame *frp = NULL; /* frame pointer. bottom level unused */ Cell *call(Node **a, int n) /* function call. very kludgy and fragile */ { static const Cell newcopycell = { OCELL, CCOPY, 0, EMPTY, 0.0, NUM|STR|DONTFREE, NULL, NULL }; int i, ncall, ndef; int freed = 0; /* handles potential double freeing when fcn & param share a tempcell */ Node *x; Cell *args[NARGS], *oargs[NARGS]; /* BUG: fixed size arrays */ Cell *y, *z, *fcn; char *s; fcn = execute(a[0]); /* the function itself */ s = fcn->nval; if (!isfcn(fcn)) FATAL("calling undefined function %s", s); if (frame == NULL) { frp = frame = (struct Frame *) calloc(nframe += 100, sizeof(*frame)); if (frame == NULL) FATAL("out of space for stack frames calling %s", s); } for (ncall = 0, x = a[1]; x != NULL; x = x->nnext) /* args in call */ ncall++; ndef = (int) fcn->fval; /* args in defn */ DPRINTF("calling %s, %d args (%d in defn), frp=%d\n", s, ncall, ndef, (int) (frp-frame)); if (ncall > ndef) WARNING("function %s called with %d args, uses only %d", s, ncall, ndef); if (ncall + ndef > NARGS) FATAL("function %s has %d arguments, limit %d", s, ncall+ndef, NARGS); for (i = 0, x = a[1]; x != NULL; i++, x = x->nnext) { /* get call args */ DPRINTF("evaluate args[%d], frp=%d:\n", i, (int) (frp-frame)); y = execute(x); oargs[i] = y; DPRINTF("args[%d]: %s %f <%s>, t=%o\n", i, NN(y->nval), y->fval, isarr(y) ? "(array)" : NN(y->sval), y->tval); if (isfcn(y)) FATAL("can't use function %s as argument in %s", y->nval, s); if (isarr(y)) args[i] = y; /* arrays by ref */ else args[i] = copycell(y); tempfree(y); } for ( ; i < ndef; i++) { /* add null args for ones not provided */ args[i] = gettemp(); *args[i] = newcopycell; } frp++; /* now ok to up frame */ if (frp >= frame + nframe) { int dfp = frp - frame; /* old index */ frame = (struct Frame *) realloc(frame, (nframe += 100) * sizeof(*frame)); if (frame == NULL) FATAL("out of space for stack frames in %s", s); frp = frame + dfp; } frp->fcncell = fcn; frp->args = args; frp->nargs = ndef; /* number defined with (excess are locals) */ frp->retval = gettemp(); DPRINTF("start exec of %s, frp=%d\n", s, (int) (frp-frame)); y = execute((Node *)(fcn->sval)); /* execute body */ DPRINTF("finished exec of %s, frp=%d\n", s, (int) (frp-frame)); for (i = 0; i < ndef; i++) { Cell *t = frp->args[i]; if (isarr(t)) { if (t->csub == CCOPY) { if (i >= ncall) { freesymtab(t); t->csub = CTEMP; tempfree(t); } else { oargs[i]->tval = t->tval; oargs[i]->tval &= ~(STR|NUM|DONTFREE); oargs[i]->sval = t->sval; tempfree(t); } } } else if (t != y) { /* kludge to prevent freeing twice */ t->csub = CTEMP; tempfree(t); } else if (t == y && t->csub == CCOPY) { t->csub = CTEMP; tempfree(t); freed = 1; } } tempfree(fcn); if (isexit(y) || isnext(y)) return y; if (freed == 0) { tempfree(y); /* don't free twice! */ } z = frp->retval; /* return value */ DPRINTF("%s returns %g |%s| %o\n", s, getfval(z), getsval(z), z->tval); frp--; return(z); } Cell *copycell(Cell *x) /* make a copy of a cell in a temp */ { Cell *y; /* copy is not constant or field */ y = gettemp(); y->tval = x->tval & ~(CON|FLD|REC); y->csub = CCOPY; /* prevents freeing until call is over */ y->nval = x->nval; /* BUG? */ if (isstr(x) /* || x->ctype == OCELL */) { y->sval = tostring(x->sval); y->tval &= ~DONTFREE; } else y->tval |= DONTFREE; y->fval = x->fval; return y; } Cell *arg(Node **a, int n) /* nth argument of a function */ { n = ptoi(a[0]); /* argument number, counting from 0 */ DPRINTF("arg(%d), frp->nargs=%d\n", n, frp->nargs); if (n+1 > frp->nargs) FATAL("argument #%d of function %s was not supplied", n+1, frp->fcncell->nval); return frp->args[n]; } Cell *jump(Node **a, int n) /* break, continue, next, nextfile, return */ { Cell *y; switch (n) { case EXIT: if (a[0] != NULL) { y = execute(a[0]); errorflag = (int) getfval(y); tempfree(y); } longjmp(env, 1); case RETURN: if (a[0] != NULL) { y = execute(a[0]); if ((y->tval & (STR|NUM)) == (STR|NUM)) { setsval(frp->retval, getsval(y)); frp->retval->fval = getfval(y); frp->retval->tval |= NUM; } else if (y->tval & STR) setsval(frp->retval, getsval(y)); else if (y->tval & NUM) setfval(frp->retval, getfval(y)); else /* can't happen */ FATAL("bad type variable %d", y->tval); tempfree(y); } return(jret); case NEXT: return(jnext); case NEXTFILE: nextfile(); return(jnextfile); case BREAK: return(jbreak); case CONTINUE: return(jcont); default: /* can't happen */ FATAL("illegal jump type %d", n); } return 0; /* not reached */ } Cell *awkgetline(Node **a, int n) /* get next line from specific input */ { /* a[0] is variable, a[1] is operator, a[2] is filename */ Cell *r, *x; extern Cell **fldtab; FILE *fp; char *buf; int bufsize = recsize; int mode; bool newflag; double result; if ((buf = (char *) malloc(bufsize)) == NULL) FATAL("out of memory in getline"); fflush(stdout); /* in case someone is waiting for a prompt */ r = gettemp(); if (a[1] != NULL) { /* getline < file */ x = execute(a[2]); /* filename */ mode = ptoi(a[1]); if (mode == '|') /* input pipe */ mode = LE; /* arbitrary flag */ fp = openfile(mode, getsval(x), &newflag); tempfree(x); if (fp == NULL) n = -1; else n = readrec(&buf, &bufsize, fp, newflag); if (n <= 0) { ; } else if (a[0] != NULL) { /* getline var sval, & result)) { x->fval = result; x->tval |= NUM; } tempfree(x); } else { /* getline sval, & result)) { fldtab[0]->fval = result; fldtab[0]->tval |= NUM; } } } else { /* bare getline; use current input */ if (a[0] == NULL) /* getline */ n = getrec(&record, &recsize, true); else { /* getline var */ n = getrec(&buf, &bufsize, false); - x = execute(a[0]); - setsval(x, buf); - if (is_number(x->sval, & result)) { - x->fval = result; - x->tval |= NUM; + if (n > 0) { + x = execute(a[0]); + setsval(x, buf); + if (is_number(x->sval, & result)) { + x->fval = result; + x->tval |= NUM; + } + tempfree(x); } - tempfree(x); } } setfval(r, (Awkfloat) n); free(buf); return r; } Cell *getnf(Node **a, int n) /* get NF */ { if (!donefld) fldbld(); return (Cell *) a[0]; } static char * makearraystring(Node *p, const char *func) { char *buf; int bufsz = recsize; size_t blen; if ((buf = (char *) malloc(bufsz)) == NULL) { FATAL("%s: out of memory", func); } blen = 0; buf[blen] = '\0'; for (; p; p = p->nnext) { Cell *x = execute(p); /* expr */ char *s = getsval(x); size_t seplen = strlen(getsval(subseploc)); size_t nsub = p->nnext ? seplen : 0; size_t slen = strlen(s); size_t tlen = blen + slen + nsub; if (!adjbuf(&buf, &bufsz, tlen + 1, recsize, 0, func)) { FATAL("%s: out of memory %s[%s...]", func, x->nval, buf); } memcpy(buf + blen, s, slen); if (nsub) { memcpy(buf + blen + slen, *SUBSEP, nsub); } buf[tlen] = '\0'; blen = tlen; tempfree(x); } return buf; } Cell *array(Node **a, int n) /* a[0] is symtab, a[1] is list of subscripts */ { Cell *x, *z; char *buf; x = execute(a[0]); /* Cell* for symbol table */ buf = makearraystring(a[1], __func__); if (!isarr(x)) { DPRINTF("making %s into an array\n", NN(x->nval)); if (freeable(x)) xfree(x->sval); x->tval &= ~(STR|NUM|DONTFREE); x->tval |= ARR; x->sval = (char *) makesymtab(NSYMTAB); } z = setsymtab(buf, "", 0.0, STR|NUM, (Array *) x->sval); z->ctype = OCELL; z->csub = CVAR; tempfree(x); free(buf); return(z); } Cell *awkdelete(Node **a, int n) /* a[0] is symtab, a[1] is list of subscripts */ { Cell *x; x = execute(a[0]); /* Cell* for symbol table */ if (x == symtabloc) { FATAL("cannot delete SYMTAB or its elements"); } if (!isarr(x)) return True; if (a[1] == NULL) { /* delete the elements, not the table */ freesymtab(x); x->tval &= ~STR; x->tval |= ARR; x->sval = (char *) makesymtab(NSYMTAB); } else { char *buf = makearraystring(a[1], __func__); freeelem(x, buf); free(buf); } tempfree(x); return True; } Cell *intest(Node **a, int n) /* a[0] is index (list), a[1] is symtab */ { Cell *ap, *k; char *buf; ap = execute(a[1]); /* array name */ if (!isarr(ap)) { DPRINTF("making %s into an array\n", ap->nval); if (freeable(ap)) xfree(ap->sval); ap->tval &= ~(STR|NUM|DONTFREE); ap->tval |= ARR; ap->sval = (char *) makesymtab(NSYMTAB); } buf = makearraystring(a[0], __func__); k = lookup(buf, (Array *) ap->sval); tempfree(ap); free(buf); if (k == NULL) return(False); else return(True); } +/* ======== utf-8 code ========== */ + +/* + * Awk strings can contain ascii, random 8-bit items (eg Latin-1), + * or utf-8. u8_isutf tests whether a string starts with a valid + * utf-8 sequence, and returns 0 if not (e.g., high bit set). + * u8_nextlen returns length of next valid sequence, which is + * 1 for ascii, 2..4 for utf-8, or 1 for high bit non-utf. + * u8_strlen returns length of string in valid utf-8 sequences + * and/or high-bit bytes. Conversion functions go between byte + * number and character number. + * + * In theory, this behaves the same as before for non-utf8 bytes. + * + * Limited checking! This is a potential security hole. + */ + +/* is s the beginning of a valid utf-8 string? */ +/* return length 1..4 if yes, 0 if no */ +int u8_isutf(const char *s) +{ + int n, ret; + unsigned char c; + + c = s[0]; + if (c < 128 || awk_mb_cur_max == 1) + return 1; /* what if it's 0? */ + + n = strlen(s); + if (n >= 2 && ((c>>5) & 0x7) == 0x6 && (s[1] & 0xC0) == 0x80) { + ret = 2; /* 110xxxxx 10xxxxxx */ + } else if (n >= 3 && ((c>>4) & 0xF) == 0xE && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80) { + ret = 3; /* 1110xxxx 10xxxxxx 10xxxxxx */ + } else if (n >= 4 && ((c>>3) & 0x1F) == 0x1E && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80 && (s[3] & 0xC0) == 0x80) { + ret = 4; /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + } else { + ret = 0; + } + return ret; +} + +/* Convert (prefix of) utf8 string to utf-32 rune. */ +/* Sets *rune to the value, returns the length. */ +/* No error checking: watch out. */ +int u8_rune(int *rune, const char *s) +{ + int n, ret; + unsigned char c; + + c = s[0]; + if (c < 128 || awk_mb_cur_max == 1) { + *rune = c; + return 1; + } + + n = strlen(s); + if (n >= 2 && ((c>>5) & 0x7) == 0x6 && (s[1] & 0xC0) == 0x80) { + *rune = ((c & 0x1F) << 6) | (s[1] & 0x3F); /* 110xxxxx 10xxxxxx */ + ret = 2; + } else if (n >= 3 && ((c>>4) & 0xF) == 0xE && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80) { + *rune = ((c & 0xF) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F); + /* 1110xxxx 10xxxxxx 10xxxxxx */ + ret = 3; + } else if (n >= 4 && ((c>>3) & 0x1F) == 0x1E && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80 && (s[3] & 0xC0) == 0x80) { + *rune = ((c & 0x7) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F); + /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + ret = 4; + } else { + *rune = c; + ret = 1; + } + return ret; /* returns one byte if sequence doesn't look like utf */ +} + +/* return length of next sequence: 1 for ascii or random, 2..4 for valid utf8 */ +int u8_nextlen(const char *s) +{ + int len; + + len = u8_isutf(s); + if (len == 0) + len = 1; + return len; +} + +/* return number of utf characters or single non-utf bytes */ +int u8_strlen(const char *s) +{ + int i, len, n, totlen; + unsigned char c; + + n = strlen(s); + totlen = 0; + for (i = 0; i < n; i += len) { + c = s[i]; + if (c < 128 || awk_mb_cur_max == 1) { + len = 1; + } else { + len = u8_nextlen(&s[i]); + } + totlen++; + if (i > n) + FATAL("bad utf count [%s] n=%d i=%d\n", s, n, i); + } + return totlen; +} + +/* convert utf-8 char number in a string to its byte offset */ +int u8_char2byte(const char *s, int charnum) +{ + int n; + int bytenum = 0; + + while (charnum > 0) { + n = u8_nextlen(s); + s += n; + bytenum += n; + charnum--; + } + return bytenum; +} + +/* convert byte offset in s to utf-8 char number that starts there */ +int u8_byte2char(const char *s, int bytenum) +{ + int i, len, b; + int charnum = 0; /* BUG: what origin? */ + /* should be 0 to match start==0 which means no match */ + + b = strlen(s); + if (bytenum > b) { + return -1; /* ??? */ + } + for (i = 0; i <= bytenum; i += len) { + len = u8_nextlen(s+i); + charnum++; + } + return charnum; +} + +/* runetochar() adapted from rune.c in the Plan 9 distributione */ + +enum +{ + Runeerror = 128, /* from somewhere else */ + Runemax = 0x10FFFF, + + Bit1 = 7, + Bitx = 6, + Bit2 = 5, + Bit3 = 4, + Bit4 = 3, + Bit5 = 2, + + T1 = ((1<<(Bit1+1))-1) ^ 0xFF, /* 0000 0000 */ + Tx = ((1<<(Bitx+1))-1) ^ 0xFF, /* 1000 0000 */ + T2 = ((1<<(Bit2+1))-1) ^ 0xFF, /* 1100 0000 */ + T3 = ((1<<(Bit3+1))-1) ^ 0xFF, /* 1110 0000 */ + T4 = ((1<<(Bit4+1))-1) ^ 0xFF, /* 1111 0000 */ + T5 = ((1<<(Bit5+1))-1) ^ 0xFF, /* 1111 1000 */ + + Rune1 = (1<<(Bit1+0*Bitx))-1, /* 0000 0000 0000 0000 0111 1111 */ + Rune2 = (1<<(Bit2+1*Bitx))-1, /* 0000 0000 0000 0111 1111 1111 */ + Rune3 = (1<<(Bit3+2*Bitx))-1, /* 0000 0000 1111 1111 1111 1111 */ + Rune4 = (1<<(Bit4+3*Bitx))-1, /* 0011 1111 1111 1111 1111 1111 */ + + Maskx = (1< 00-7F */ + if (c <= Rune1) { + str[0] = c; + return 1; + } + + /* two character sequence 00080-007FF => T2 Tx */ + if (c <= Rune2) { + str[0] = T2 | (c >> 1*Bitx); + str[1] = Tx | (c & Maskx); + return 2; + } + + /* three character sequence 00800-0FFFF => T3 Tx Tx */ + if (c > Runemax) + c = Runeerror; + if (c <= Rune3) { + str[0] = T3 | (c >> 2*Bitx); + str[1] = Tx | ((c >> 1*Bitx) & Maskx); + str[2] = Tx | (c & Maskx); + return 3; + } + + /* four character sequence 010000-1FFFFF => T4 Tx Tx Tx */ + str[0] = T4 | (c >> 3*Bitx); + str[1] = Tx | ((c >> 2*Bitx) & Maskx); + str[2] = Tx | ((c >> 1*Bitx) & Maskx); + str[3] = Tx | (c & Maskx); + return 4; +} + + +/* ========== end of utf8 code =========== */ + + + Cell *matchop(Node **a, int n) /* ~ and match() */ { Cell *x, *y; char *s, *t; int i; + int cstart, cpatlen, len; fa *pfa; int (*mf)(fa *, const char *) = match, mode = 0; if (n == MATCHFCN) { mf = pmatch; mode = 1; } x = execute(a[1]); /* a[1] = target text */ s = getsval(x); if (a[0] == NULL) /* a[1] == 0: already-compiled reg expr */ i = (*mf)((fa *) a[2], s); else { y = execute(a[2]); /* a[2] = regular expr */ t = getsval(y); pfa = makedfa(t, mode); i = (*mf)(pfa, s); tempfree(y); } tempfree(x); if (n == MATCHFCN) { - int start = patbeg - s + 1; - if (patlen < 0) - start = 0; + int start = patbeg - s + 1; /* origin 1 */ + if (patlen < 0) { + start = 0; /* not found */ + } else { + cstart = u8_byte2char(s, start-1); + cpatlen = 0; + for (i = 0; i < patlen; i += len) { + len = u8_nextlen(patbeg+i); + cpatlen++; + } + + start = cstart; + patlen = cpatlen; + } + setfval(rstartloc, (Awkfloat) start); setfval(rlengthloc, (Awkfloat) patlen); x = gettemp(); x->tval = NUM; x->fval = start; return x; } else if ((n == MATCH && i == 1) || (n == NOTMATCH && i == 0)) return(True); else return(False); } Cell *boolop(Node **a, int n) /* a[0] || a[1], a[0] && a[1], !a[0] */ { Cell *x, *y; int i; x = execute(a[0]); i = istrue(x); tempfree(x); switch (n) { case BOR: if (i) return(True); y = execute(a[1]); i = istrue(y); tempfree(y); if (i) return(True); else return(False); case AND: if ( !i ) return(False); y = execute(a[1]); i = istrue(y); tempfree(y); if (i) return(True); else return(False); case NOT: if (i) return(False); else return(True); default: /* can't happen */ FATAL("unknown boolean operator %d", n); } return 0; /*NOTREACHED*/ } Cell *relop(Node **a, int n) /* a[0 < a[1], etc. */ { int i; Cell *x, *y; Awkfloat j; + bool x_is_nan, y_is_nan; x = execute(a[0]); y = execute(a[1]); + x_is_nan = isnan(x->fval); + y_is_nan = isnan(y->fval); if (x->tval&NUM && y->tval&NUM) { + if ((x_is_nan || y_is_nan) && n != NE) + return(False); j = x->fval - y->fval; i = j<0? -1: (j>0? 1: 0); } else { i = strcmp(getsval(x), getsval(y)); } tempfree(x); tempfree(y); switch (n) { case LT: if (i<0) return(True); else return(False); case LE: if (i<=0) return(True); else return(False); - case NE: if (i!=0) return(True); + case NE: if (x_is_nan && y_is_nan) return(True); + else if (i!=0) return(True); else return(False); case EQ: if (i == 0) return(True); else return(False); case GE: if (i>=0) return(True); else return(False); case GT: if (i>0) return(True); else return(False); default: /* can't happen */ FATAL("unknown relational operator %d", n); } return 0; /*NOTREACHED*/ } void tfree(Cell *a) /* free a tempcell */ { if (freeable(a)) { DPRINTF("freeing %s %s %o\n", NN(a->nval), NN(a->sval), a->tval); xfree(a->sval); } if (a == tmps) FATAL("tempcell list is curdled"); a->cnext = tmps; tmps = a; } Cell *gettemp(void) /* get a tempcell */ { int i; Cell *x; if (!tmps) { tmps = (Cell *) calloc(100, sizeof(*tmps)); if (!tmps) FATAL("out of space for temporaries"); for (i = 1; i < 100; i++) tmps[i-1].cnext = &tmps[i]; tmps[i-1].cnext = NULL; } x = tmps; tmps = x->cnext; *x = tempcell; return(x); } Cell *indirect(Node **a, int n) /* $( a[0] ) */ { Awkfloat val; Cell *x; int m; char *s; x = execute(a[0]); val = getfval(x); /* freebsd: defend against super large field numbers */ if ((Awkfloat)INT_MAX < val) FATAL("trying to access out of range field %s", x->nval); m = (int) val; if (m == 0 && !is_number(s = getsval(x), NULL)) /* suspicion! */ FATAL("illegal field $(%s), name \"%s\"", s, x->nval); /* BUG: can x->nval ever be null??? */ tempfree(x); x = fieldadr(m); x->ctype = OCELL; /* BUG? why are these needed? */ x->csub = CFLD; return(x); } Cell *substr(Node **a, int nnn) /* substr(a[0], a[1], a[2]) */ { int k, m, n; + int mb, nb; char *s; int temp; Cell *x, *y, *z = NULL; x = execute(a[0]); y = execute(a[1]); if (a[2] != NULL) z = execute(a[2]); s = getsval(x); - k = strlen(s) + 1; + k = u8_strlen(s) + 1; if (k <= 1) { tempfree(x); tempfree(y); if (a[2] != NULL) { tempfree(z); } x = gettemp(); setsval(x, ""); return(x); } m = (int) getfval(y); if (m <= 0) m = 1; else if (m > k) m = k; tempfree(y); if (a[2] != NULL) { n = (int) getfval(z); tempfree(z); } else n = k - 1; if (n < 0) n = 0; else if (n > k - m) n = k - m; + /* m is start, n is length from there */ DPRINTF("substr: m=%d, n=%d, s=%s\n", m, n, s); y = gettemp(); - temp = s[n+m-1]; /* with thanks to John Linderman */ - s[n+m-1] = '\0'; - setsval(y, s + m - 1); - s[n+m-1] = temp; + mb = u8_char2byte(s, m-1); /* byte offset of start char in s */ + nb = u8_char2byte(s, m-1+n); /* byte offset of end+1 char in s */ + + temp = s[nb]; /* with thanks to John Linderman */ + s[nb] = '\0'; + setsval(y, s + mb); + s[nb] = temp; tempfree(x); return(y); } Cell *sindex(Node **a, int nnn) /* index(a[0], a[1]) */ { Cell *x, *y, *z; char *s1, *s2, *p1, *p2, *q; Awkfloat v = 0.0; x = execute(a[0]); s1 = getsval(x); y = execute(a[1]); s2 = getsval(y); z = gettemp(); for (p1 = s1; *p1 != '\0'; p1++) { for (q = p1, p2 = s2; *p2 != '\0' && *q == *p2; q++, p2++) continue; if (*p2 == '\0') { - v = (Awkfloat) (p1 - s1 + 1); /* origin 1 */ + /* v = (Awkfloat) (p1 - s1 + 1); origin 1 */ + + /* should be a function: used in match() as well */ + int i, len; + v = 0; + for (i = 0; i < p1-s1+1; i += len) { + len = u8_nextlen(s1+i); + v++; + } break; } } tempfree(x); tempfree(y); setfval(z, v); return(z); } +int has_utf8(char *s) /* return 1 if s contains any utf-8 (2 bytes or more) character */ +{ + int n; + + for (n = 0; *s != 0; s += n) { + n = u8_nextlen(s); + if (n > 1) + return 1; + } + return 0; +} + #define MAXNUMSIZE 50 int format(char **pbuf, int *pbufsize, const char *s, Node *a) /* printf-like conversions */ { char *fmt; char *p, *t; const char *os; Cell *x; int flag = 0, n; int fmtwd; /* format width */ int fmtsz = recsize; char *buf = *pbuf; int bufsize = *pbufsize; #define FMTSZ(a) (fmtsz - ((a) - fmt)) #define BUFSZ(a) (bufsize - ((a) - buf)) static bool first = true; static bool have_a_format = false; if (first) { char xbuf[100]; snprintf(xbuf, sizeof(xbuf), "%a", 42.0); have_a_format = (strcmp(xbuf, "0x1.5p+5") == 0); first = false; } os = s; p = buf; if ((fmt = (char *) malloc(fmtsz)) == NULL) FATAL("out of memory in format()"); while (*s) { adjbuf(&buf, &bufsize, MAXNUMSIZE+1+p-buf, recsize, &p, "format1"); if (*s != '%') { *p++ = *s++; continue; } if (*(s+1) == '%') { *p++ = '%'; s += 2; continue; } - /* have to be real careful in case this is a huge number, eg, %100000d */ fmtwd = atoi(s+1); if (fmtwd < 0) fmtwd = -fmtwd; adjbuf(&buf, &bufsize, fmtwd+1+p-buf, recsize, &p, "format2"); for (t = fmt; (*t++ = *s) != '\0'; s++) { if (!adjbuf(&fmt, &fmtsz, MAXNUMSIZE+1+t-fmt, recsize, &t, "format3")) FATAL("format item %.30s... ran format() out of memory", os); /* Ignore size specifiers */ if (strchr("hjLlqtz", *s) != NULL) { /* the ansi panoply */ t--; continue; } if (isalpha((uschar)*s)) break; if (*s == '$') { FATAL("'$' not permitted in awk formats"); } if (*s == '*') { if (a == NULL) { FATAL("not enough args in printf(%s)", os); } x = execute(a); a = a->nnext; snprintf(t - 1, FMTSZ(t - 1), "%d", fmtwd=(int) getfval(x)); if (fmtwd < 0) fmtwd = -fmtwd; adjbuf(&buf, &bufsize, fmtwd+1+p-buf, recsize, &p, "format"); t = fmt + strlen(fmt); tempfree(x); } } *t = '\0'; if (fmtwd < 0) fmtwd = -fmtwd; adjbuf(&buf, &bufsize, fmtwd+1+p-buf, recsize, &p, "format4"); switch (*s) { case 'a': case 'A': if (have_a_format) flag = *s; else flag = 'f'; break; case 'f': case 'e': case 'g': case 'E': case 'G': flag = 'f'; break; case 'd': case 'i': case 'o': case 'x': case 'X': case 'u': flag = (*s == 'd' || *s == 'i') ? 'd' : 'u'; *(t-1) = 'j'; *t = *s; *++t = '\0'; break; case 's': flag = 's'; break; case 'c': flag = 'c'; break; case '\0': FATAL("missing printf conversion specifier"); break; default: WARNING("weird printf conversion %s", fmt); flag = '?'; break; } if (a == NULL) FATAL("not enough args in printf(%s)", os); x = execute(a); a = a->nnext; n = MAXNUMSIZE; if (fmtwd > n) n = fmtwd; adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format5"); switch (flag) { - case '?': snprintf(p, BUFSZ(p), "%s", fmt); /* unknown, so dump it too */ + case '?': + snprintf(p, BUFSZ(p), "%s", fmt); /* unknown, so dump it too */ t = getsval(x); n = strlen(t); if (fmtwd > n) n = fmtwd; adjbuf(&buf, &bufsize, 1+strlen(p)+n+p-buf, recsize, &p, "format6"); p += strlen(p); snprintf(p, BUFSZ(p), "%s", t); break; case 'a': case 'A': case 'f': snprintf(p, BUFSZ(p), fmt, getfval(x)); break; case 'd': snprintf(p, BUFSZ(p), fmt, (intmax_t) getfval(x)); break; case 'u': snprintf(p, BUFSZ(p), fmt, (uintmax_t) getfval(x)); break; - case 's': + + case 's': { t = getsval(x); n = strlen(t); - if (fmtwd > n) - n = fmtwd; - if (!adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format7")) - FATAL("huge string/format (%d chars) in printf %.30s... ran format() out of memory", n, t); - snprintf(p, BUFSZ(p), fmt, t); + /* if simple format or no utf-8 in the string, sprintf works */ + if (!has_utf8(t) || strcmp(fmt,"%s") == 0) { + if (fmtwd > n) + n = fmtwd; + if (!adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format7")) + FATAL("huge string/format (%d chars) in printf %.30s..." \ + " ran format() out of memory", n, t); + snprintf(p, BUFSZ(p), fmt, t); + break; + } + + /* get here if string has utf-8 chars and fmt is not plain %s */ + /* "%-w.ps", where -, w and .p are all optional */ + /* '0' before the w is a flag character */ + /* fmt points at % */ + int ljust = 0, wid = 0, prec = n, pad = 0; + char *f = fmt+1; + if (f[0] == '-') { + ljust = 1; + f++; + } + // flags '0' and '+' are recognized but skipped + if (f[0] == '0') { + f++; + if (f[0] == '+') + f++; + } + if (f[0] == '+') { + f++; + if (f[0] == '0') + f++; + } + if (isdigit(f[0])) { /* there is a wid */ + wid = strtol(f, &f, 10); + } + if (f[0] == '.') { /* there is a .prec */ + prec = strtol(++f, &f, 10); + } + if (prec > u8_strlen(t)) + prec = u8_strlen(t); + pad = wid>prec ? wid - prec : 0; // has to be >= 0 + int i, k, n; + + if (ljust) { // print prec chars from t, then pad blanks + n = u8_char2byte(t, prec); + for (k = 0; k < n; k++) { + //putchar(t[k]); + *p++ = t[k]; + } + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + } else { // print pad blanks, then prec chars from t + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + n = u8_char2byte(t, prec); + for (k = 0; k < n; k++) { + //putchar(t[k]); + *p++ = t[k]; + } + } + *p = 0; break; - case 'c': + } + + case 'c': { + /* + * If a numeric value is given, awk should just turn + * it into a character and print it: + * BEGIN { printf("%c\n", 65) } + * prints "A". + * + * But what if the numeric value is > 128 and + * represents a valid Unicode code point?!? We do + * our best to convert it back into UTF-8. If we + * can't, we output the encoding of the Unicode + * "invalid character", 0xFFFD. + */ if (isnum(x)) { - if ((int)getfval(x)) - snprintf(p, BUFSZ(p), fmt, (int) getfval(x)); - else { + int charval = (int) getfval(x); + + if (charval != 0) { + if (charval < 128 || awk_mb_cur_max == 1) + snprintf(p, BUFSZ(p), fmt, charval); + else { + // possible unicode character + size_t count; + char *bs = wide_char_to_byte_str(charval, &count); + + if (bs == NULL) { // invalid character + // use unicode invalid character, 0xFFFD + bs = "\357\277\275"; + count = 3; + } + t = bs; + n = count; + goto format_percent_c; + } + } else { *p++ = '\0'; /* explicit null byte */ *p = '\0'; /* next output will start here */ } - } else + break; + } + t = getsval(x); + n = u8_nextlen(t); + format_percent_c: + if (n < 2) { /* not utf8 */ snprintf(p, BUFSZ(p), fmt, getsval(x)[0]); + break; + } + + // utf8 character, almost same song and dance as for %s + int ljust = 0, wid = 0, prec = n, pad = 0; + char *f = fmt+1; + if (f[0] == '-') { + ljust = 1; + f++; + } + // flags '0' and '+' are recognized but skipped + if (f[0] == '0') { + f++; + if (f[0] == '+') + f++; + } + if (f[0] == '+') { + f++; + if (f[0] == '0') + f++; + } + if (isdigit(f[0])) { /* there is a wid */ + wid = strtol(f, &f, 10); + } + if (f[0] == '.') { /* there is a .prec */ + prec = strtol(++f, &f, 10); + } + if (prec > 1) // %c --> only one character + prec = 1; + pad = wid>prec ? wid - prec : 0; // has to be >= 0 + int i; + + if (ljust) { // print one char from t, then pad blanks + for (i = 0; i < n; i++) + *p++ = t[i]; + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + } else { // print pad blanks, then prec chars from t + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + for (i = 0; i < n; i++) + *p++ = t[i]; + } + *p = 0; break; + } default: FATAL("can't happen: bad conversion %c in format()", flag); } + tempfree(x); p += strlen(p); s++; } *p = '\0'; free(fmt); - for ( ; a; a = a->nnext) /* evaluate any remaining args */ - execute(a); + for ( ; a; a = a->nnext) { /* evaluate any remaining args */ + x = execute(a); + tempfree(x); + } *pbuf = buf; *pbufsize = bufsize; return p - buf; } Cell *awksprintf(Node **a, int n) /* sprintf(a[0]) */ { Cell *x; Node *y; char *buf; int bufsz=3*recsize; if ((buf = (char *) malloc(bufsz)) == NULL) FATAL("out of memory in awksprintf"); y = a[0]->nnext; x = execute(a[0]); if (format(&buf, &bufsz, getsval(x), y) == -1) FATAL("sprintf string %.30s... too long. can't happen.", buf); tempfree(x); x = gettemp(); x->sval = buf; x->tval = STR; return(x); } Cell *awkprintf(Node **a, int n) /* printf */ { /* a[0] is list of args, starting with format string */ /* a[1] is redirection operator, a[2] is redirection file */ FILE *fp; Cell *x; Node *y; char *buf; int len; int bufsz=3*recsize; if ((buf = (char *) malloc(bufsz)) == NULL) FATAL("out of memory in awkprintf"); y = a[0]->nnext; x = execute(a[0]); if ((len = format(&buf, &bufsz, getsval(x), y)) == -1) FATAL("printf string %.30s... too long. can't happen.", buf); tempfree(x); if (a[1] == NULL) { /* fputs(buf, stdout); */ fwrite(buf, len, 1, stdout); if (ferror(stdout)) FATAL("write error on stdout"); } else { fp = redirect(ptoi(a[1]), a[2]); /* fputs(buf, fp); */ fwrite(buf, len, 1, fp); fflush(fp); if (ferror(fp)) FATAL("write error on %s", filename(fp)); } free(buf); return(True); } Cell *arith(Node **a, int n) /* a[0] + a[1], etc. also -a[0] */ { Awkfloat i, j = 0; double v; Cell *x, *y, *z; x = execute(a[0]); i = getfval(x); tempfree(x); if (n != UMINUS && n != UPLUS) { y = execute(a[1]); j = getfval(y); tempfree(y); } z = gettemp(); switch (n) { case ADD: i += j; break; case MINUS: i -= j; break; case MULT: i *= j; break; case DIVIDE: if (j == 0) FATAL("division by zero"); i /= j; break; case MOD: if (j == 0) FATAL("division by zero in mod"); modf(i/j, &v); i = i - j * v; break; case UMINUS: i = -i; break; case UPLUS: /* handled by getfval(), above */ break; case POWER: if (j >= 0 && modf(j, &v) == 0.0) /* pos integer exponent */ i = ipow(i, (int) j); else { errno = 0; i = errcheck(pow(i, j), "pow"); } break; default: /* can't happen */ FATAL("illegal arithmetic operator %d", n); } setfval(z, i); return(z); } double ipow(double x, int n) /* x**n. ought to be done by pow, but isn't always */ { double v; if (n <= 0) return 1; v = ipow(x, n/2); if (n % 2 == 0) return v * v; else return x * v * v; } Cell *incrdecr(Node **a, int n) /* a[0]++, etc. */ { Cell *x, *z; int k; Awkfloat xf; x = execute(a[0]); xf = getfval(x); k = (n == PREINCR || n == POSTINCR) ? 1 : -1; if (n == PREINCR || n == PREDECR) { setfval(x, xf + k); return(x); } z = gettemp(); setfval(z, xf); setfval(x, xf + k); tempfree(x); return(z); } Cell *assign(Node **a, int n) /* a[0] = a[1], a[0] += a[1], etc. */ { /* this is subtle; don't muck with it. */ Cell *x, *y; Awkfloat xf, yf; double v; y = execute(a[1]); x = execute(a[0]); if (n == ASSIGN) { /* ordinary assignment */ if (x == y && !(x->tval & (FLD|REC)) && x != nfloc) ; /* self-assignment: leave alone unless it's a field or NF */ else if ((y->tval & (STR|NUM)) == (STR|NUM)) { setsval(x, getsval(y)); x->fval = getfval(y); x->tval |= NUM; } else if (isstr(y)) setsval(x, getsval(y)); else if (isnum(y)) setfval(x, getfval(y)); else funnyvar(y, "read value of"); tempfree(y); return(x); } xf = getfval(x); yf = getfval(y); switch (n) { case ADDEQ: xf += yf; break; case SUBEQ: xf -= yf; break; case MULTEQ: xf *= yf; break; case DIVEQ: if (yf == 0) FATAL("division by zero in /="); xf /= yf; break; case MODEQ: if (yf == 0) FATAL("division by zero in %%="); modf(xf/yf, &v); xf = xf - yf * v; break; case POWEQ: if (yf >= 0 && modf(yf, &v) == 0.0) /* pos integer exponent */ xf = ipow(xf, (int) yf); else { errno = 0; xf = errcheck(pow(xf, yf), "pow"); } break; default: FATAL("illegal assignment operator %d", n); break; } tempfree(y); setfval(x, xf); return(x); } Cell *cat(Node **a, int q) /* a[0] cat a[1] */ { Cell *x, *y, *z; int n1, n2; char *s = NULL; int ssz = 0; x = execute(a[0]); n1 = strlen(getsval(x)); - adjbuf(&s, &ssz, n1, recsize, 0, "cat1"); + adjbuf(&s, &ssz, n1 + 1, recsize, 0, "cat1"); memcpy(s, x->sval, n1); + tempfree(x); + y = execute(a[1]); n2 = strlen(getsval(y)); adjbuf(&s, &ssz, n1 + n2 + 1, recsize, 0, "cat2"); memcpy(s + n1, y->sval, n2); s[n1 + n2] = '\0'; - tempfree(x); tempfree(y); z = gettemp(); z->sval = s; z->tval = STR; return(z); } Cell *pastat(Node **a, int n) /* a[0] { a[1] } */ { Cell *x; if (a[0] == NULL) x = execute(a[1]); else { x = execute(a[0]); if (istrue(x)) { tempfree(x); x = execute(a[1]); } } return x; } Cell *dopa2(Node **a, int n) /* a[0], a[1] { a[2] } */ { Cell *x; int pair; pair = ptoi(a[3]); if (pairstack[pair] == 0) { x = execute(a[0]); if (istrue(x)) pairstack[pair] = 1; tempfree(x); } if (pairstack[pair] == 1) { x = execute(a[1]); if (istrue(x)) pairstack[pair] = 0; tempfree(x); x = execute(a[2]); return(x); } return(False); } Cell *split(Node **a, int nnn) /* split(a[0], a[1], a[2]); a[3] is type */ { Cell *x = NULL, *y, *ap; const char *s, *origs, *t; const char *fs = NULL; char *origfs = NULL; int sep; char temp, num[50]; int n, tempstat, arg3type; + int j; double result; y = execute(a[0]); /* source string */ origs = s = strdup(getsval(y)); + tempfree(y); arg3type = ptoi(a[3]); - if (a[2] == NULL) /* fs string */ + if (a[2] == NULL) { /* BUG: CSV should override implicit fs but not explicit */ fs = getsval(fsloc); - else if (arg3type == STRING) { /* split(str,arr,"string") */ + } else if (arg3type == STRING) { /* split(str,arr,"string") */ x = execute(a[2]); fs = origfs = strdup(getsval(x)); tempfree(x); - } else if (arg3type == REGEXPR) + } else if (arg3type == REGEXPR) { fs = "(regexpr)"; /* split(str,arr,/regexpr/) */ - else + } else { FATAL("illegal type of split"); + } sep = *fs; ap = execute(a[1]); /* array name */ +/* BUG 7/26/22: this appears not to reset array: see C1/asplit */ freesymtab(ap); DPRINTF("split: s=|%s|, a=%s, sep=|%s|\n", s, NN(ap->nval), fs); ap->tval &= ~STR; ap->tval |= ARR; ap->sval = (char *) makesymtab(NSYMTAB); n = 0; if (arg3type == REGEXPR && strlen((char*)((fa*)a[2])->restr) == 0) { /* split(s, a, //); have to arrange that it looks like empty sep */ arg3type = 0; fs = ""; sep = 0; } if (*s != '\0' && (strlen(fs) > 1 || arg3type == REGEXPR)) { /* reg expr */ fa *pfa; if (arg3type == REGEXPR) { /* it's ready already */ pfa = (fa *) a[2]; } else { pfa = makedfa(fs, 1); } if (nematch(pfa,s)) { tempstat = pfa->initstat; pfa->initstat = 2; do { n++; snprintf(num, sizeof(num), "%d", n); temp = *patbeg; setptr(patbeg, '\0'); if (is_number(s, & result)) setsymtab(num, s, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, s, 0.0, STR, (Array *) ap->sval); setptr(patbeg, temp); s = patbeg + patlen; if (*(patbeg+patlen-1) == '\0' || *s == '\0') { n++; snprintf(num, sizeof(num), "%d", n); setsymtab(num, "", 0.0, STR, (Array *) ap->sval); pfa->initstat = tempstat; goto spdone; } } while (nematch(pfa,s)); pfa->initstat = tempstat; /* bwk: has to be here to reset */ /* cf gsub and refldbld */ } n++; snprintf(num, sizeof(num), "%d", n); if (is_number(s, & result)) setsymtab(num, s, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, s, 0.0, STR, (Array *) ap->sval); spdone: pfa = NULL; - } else if (sep == ' ') { + + } else if (a[2] == NULL && CSV) { /* CSV only if no explicit separator */ + char *newt = (char *) malloc(strlen(s)); /* for building new string; reuse for each field */ + for (;;) { + char *fr = newt; + n++; + if (*s == '"' ) { /* start of "..." */ + for (s++ ; *s != '\0'; ) { + if (*s == '"' && s[1] != '\0' && s[1] == '"') { + s += 2; /* doubled quote */ + *fr++ = '"'; + } else if (*s == '"' && (s[1] == '\0' || s[1] == ',')) { + s++; /* skip over closing quote */ + break; + } else { + *fr++ = *s++; + } + } + *fr++ = 0; + } else { /* unquoted field */ + while (*s != ',' && *s != '\0') + *fr++ = *s++; + *fr++ = 0; + } + snprintf(num, sizeof(num), "%d", n); + if (is_number(newt, &result)) + setsymtab(num, newt, result, STR|NUM, (Array *) ap->sval); + else + setsymtab(num, newt, 0.0, STR, (Array *) ap->sval); + if (*s++ == '\0') + break; + } + free(newt); + + } else if (!CSV && sep == ' ') { /* usual case: split on white space */ for (n = 0; ; ) { #define ISWS(c) ((c) == ' ' || (c) == '\t' || (c) == '\n') while (ISWS(*s)) s++; if (*s == '\0') break; n++; t = s; do s++; while (*s != '\0' && !ISWS(*s)); temp = *s; setptr(s, '\0'); snprintf(num, sizeof(num), "%d", n); if (is_number(t, & result)) setsymtab(num, t, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, t, 0.0, STR, (Array *) ap->sval); setptr(s, temp); if (*s != '\0') s++; } + } else if (sep == 0) { /* new: split(s, a, "") => 1 char/elem */ - for (n = 0; *s != '\0'; s++) { - char buf[2]; + for (n = 0; *s != '\0'; s += u8_nextlen(s)) { + char buf[10]; n++; snprintf(num, sizeof(num), "%d", n); - buf[0] = *s; - buf[1] = '\0'; + + for (j = 0; j < u8_nextlen(s); j++) { + buf[j] = s[j]; + } + buf[j] = '\0'; + if (isdigit((uschar)buf[0])) setsymtab(num, buf, atof(buf), STR|NUM, (Array *) ap->sval); else setsymtab(num, buf, 0.0, STR, (Array *) ap->sval); } - } else if (*s != '\0') { + + } else if (*s != '\0') { /* some random single character */ for (;;) { n++; t = s; while (*s != sep && *s != '\n' && *s != '\0') s++; temp = *s; setptr(s, '\0'); snprintf(num, sizeof(num), "%d", n); if (is_number(t, & result)) setsymtab(num, t, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, t, 0.0, STR, (Array *) ap->sval); setptr(s, temp); if (*s++ == '\0') break; } } tempfree(ap); - tempfree(y); xfree(origs); xfree(origfs); x = gettemp(); x->tval = NUM; x->fval = n; return(x); } Cell *condexpr(Node **a, int n) /* a[0] ? a[1] : a[2] */ { Cell *x; x = execute(a[0]); if (istrue(x)) { tempfree(x); x = execute(a[1]); } else { tempfree(x); x = execute(a[2]); } return(x); } Cell *ifstat(Node **a, int n) /* if (a[0]) a[1]; else a[2] */ { Cell *x; x = execute(a[0]); if (istrue(x)) { tempfree(x); x = execute(a[1]); } else if (a[2] != NULL) { tempfree(x); x = execute(a[2]); } return(x); } Cell *whilestat(Node **a, int n) /* while (a[0]) a[1] */ { Cell *x; for (;;) { x = execute(a[0]); if (!istrue(x)) return(x); tempfree(x); x = execute(a[1]); if (isbreak(x)) { x = True; return(x); } if (isnext(x) || isexit(x) || isret(x)) return(x); tempfree(x); } } Cell *dostat(Node **a, int n) /* do a[0]; while(a[1]) */ { Cell *x; for (;;) { x = execute(a[0]); if (isbreak(x)) return True; if (isnext(x) || isexit(x) || isret(x)) return(x); tempfree(x); x = execute(a[1]); if (!istrue(x)) return(x); tempfree(x); } } Cell *forstat(Node **a, int n) /* for (a[0]; a[1]; a[2]) a[3] */ { Cell *x; x = execute(a[0]); tempfree(x); for (;;) { if (a[1]!=NULL) { x = execute(a[1]); if (!istrue(x)) return(x); else tempfree(x); } x = execute(a[3]); if (isbreak(x)) /* turn off break */ return True; if (isnext(x) || isexit(x) || isret(x)) return(x); tempfree(x); x = execute(a[2]); tempfree(x); } } Cell *instat(Node **a, int n) /* for (a[0] in a[1]) a[2] */ { Cell *x, *vp, *arrayp, *cp, *ncp; Array *tp; int i; vp = execute(a[0]); arrayp = execute(a[1]); if (!isarr(arrayp)) { return True; } tp = (Array *) arrayp->sval; tempfree(arrayp); for (i = 0; i < tp->size; i++) { /* this routine knows too much */ for (cp = tp->tab[i]; cp != NULL; cp = ncp) { setsval(vp, cp->nval); ncp = cp->cnext; x = execute(a[2]); if (isbreak(x)) { tempfree(vp); return True; } if (isnext(x) || isexit(x) || isret(x)) { tempfree(vp); return(x); } tempfree(x); } } return True; } static char *nawk_convert(const char *s, int (*fun_c)(int), wint_t (*fun_wc)(wint_t)) { char *buf = NULL; char *pbuf = NULL; const char *ps = NULL; size_t n = 0; wchar_t wc; - size_t sz = MB_CUR_MAX; + const size_t sz = awk_mb_cur_max; + int unused; if (sz == 1) { buf = tostring(s); for (pbuf = buf; *pbuf; pbuf++) *pbuf = fun_c((uschar)*pbuf); return buf; } else { /* upper/lower character may be shorter/longer */ buf = tostringN(s, strlen(s) * sz + 1); (void) mbtowc(NULL, NULL, 0); /* reset internal state */ /* * Reset internal state here too. * Assign result to avoid a compiler warning. (Casting to void * doesn't work.) * Increment said variable to avoid a different warning. */ - int unused = wctomb(NULL, L'\0'); + unused = wctomb(NULL, L'\0'); unused++; ps = s; pbuf = buf; while (n = mbtowc(&wc, ps, sz), n > 0 && n != (size_t)-1 && n != (size_t)-2) { ps += n; n = wctomb(pbuf, fun_wc(wc)); if (n == (size_t)-1) FATAL("illegal wide character %s", s); pbuf += n; } *pbuf = '\0'; if (n) FATAL("illegal byte sequence %s", s); return buf; } } #ifdef __DJGPP__ static wint_t towupper(wint_t wc) { if (wc >= 0 && wc < 256) return toupper(wc & 0xFF); return wc; } static wint_t towlower(wint_t wc) { if (wc >= 0 && wc < 256) return tolower(wc & 0xFF); return wc; } #endif static char *nawk_toupper(const char *s) { return nawk_convert(s, toupper, towupper); } static char *nawk_tolower(const char *s) { return nawk_convert(s, tolower, towlower); } + + Cell *bltin(Node **a, int n) /* builtin functions. a[0] is type, a[1] is arg list */ { Cell *x, *y; Awkfloat u; int t, sz; Awkfloat tmp; char *buf, *fmt; Node *nextarg; FILE *fp; int status = 0; time_t tv; struct tm *tm; t = ptoi(a[0]); x = execute(a[1]); nextarg = a[1]->nnext; switch (t) { case FLENGTH: if (isarr(x)) u = ((Array *) x->sval)->nelem; /* GROT. should be function*/ else - u = strlen(getsval(x)); + u = u8_strlen(getsval(x)); break; case FLOG: errno = 0; u = errcheck(log(getfval(x)), "log"); break; case FINT: modf(getfval(x), &u); break; case FEXP: errno = 0; u = errcheck(exp(getfval(x)), "exp"); break; case FSQRT: errno = 0; u = errcheck(sqrt(getfval(x)), "sqrt"); break; case FSIN: u = sin(getfval(x)); break; case FCOS: u = cos(getfval(x)); break; case FATAN: if (nextarg == NULL) { WARNING("atan2 requires two arguments; returning 1.0"); u = 1.0; } else { y = execute(a[1]->nnext); u = atan2(getfval(x), getfval(y)); tempfree(y); nextarg = nextarg->nnext; } break; case FCOMPL: u = ~((int)getfval(x)); break; case FAND: if (nextarg == 0) { WARNING("and requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) & ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FFOR: if (nextarg == 0) { WARNING("or requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) | ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FXOR: if (nextarg == 0) { WARNING("xor requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) ^ ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FLSHIFT: if (nextarg == 0) { WARNING("lshift requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) << ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FRSHIFT: if (nextarg == 0) { WARNING("rshift requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) >> ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FSYSTEM: fflush(stdout); /* in case something is buffered already */ status = system(getsval(x)); u = status; if (status != -1) { if (WIFEXITED(status)) { u = WEXITSTATUS(status); } else if (WIFSIGNALED(status)) { u = WTERMSIG(status) + 256; #ifdef WCOREDUMP if (WCOREDUMP(status)) u += 256; #endif } else /* something else?!? */ u = 0; } break; case FRAND: /* random() returns numbers in [0..2^31-1] * in order to get a number in [0, 1), divide it by 2^31 */ u = (Awkfloat) random() / (0x7fffffffL + 0x1UL); break; case FSRAND: if (isrec(x)) /* no argument provided */ u = time((time_t *)0); else u = getfval(x); tmp = u; srandom((unsigned long) u); u = srand_seed; srand_seed = tmp; break; case FTOUPPER: case FTOLOWER: if (t == FTOUPPER) buf = nawk_toupper(getsval(x)); else buf = nawk_tolower(getsval(x)); tempfree(x); x = gettemp(); setsval(x, buf); free(buf); return x; case FFLUSH: if (isrec(x) || strlen(getsval(x)) == 0) { flush_all(); /* fflush() or fflush("") -> all */ u = 0; } else if ((fp = openfile(FFLUSH, getsval(x), NULL)) == NULL) u = EOF; else u = fflush(fp); break; case FSYSTIME: u = time((time_t *) 0); break; case FSTRFTIME: /* strftime([format [,timestamp]]) */ if (nextarg) { y = execute(nextarg); nextarg = nextarg->nnext; tv = (time_t) getfval(y); tempfree(y); } else tv = time((time_t *) 0); tm = localtime(&tv); if (tm == NULL) FATAL("bad time %ld", (long)tv); if (isrec(x)) { /* format argument not provided, use default */ fmt = tostring("%a %b %d %H:%M:%S %Z %Y"); } else fmt = tostring(getsval(x)); sz = 32; buf = NULL; do { if ((buf = realloc(buf, (sz *= 2))) == NULL) FATAL("out of memory in strftime"); } while (strftime(buf, sz, fmt, tm) == 0 && fmt[0] != '\0'); y = gettemp(); setsval(y, buf); free(fmt); free(buf); return y; default: /* can't happen */ FATAL("illegal function type %d", t); break; } tempfree(x); x = gettemp(); setfval(x, u); if (nextarg != NULL) { WARNING("warning: function has too many arguments"); - for ( ; nextarg; nextarg = nextarg->nnext) - execute(nextarg); + for ( ; nextarg; nextarg = nextarg->nnext) { + y = execute(nextarg); + tempfree(y); + } } return(x); } Cell *printstat(Node **a, int n) /* print a[0] */ { Node *x; Cell *y; FILE *fp; if (a[1] == NULL) /* a[1] is redirection operator, a[2] is file */ fp = stdout; else fp = redirect(ptoi(a[1]), a[2]); for (x = a[0]; x != NULL; x = x->nnext) { y = execute(x); fputs(getpssval(y), fp); tempfree(y); if (x->nnext == NULL) fputs(getsval(orsloc), fp); else fputs(getsval(ofsloc), fp); } if (a[1] != NULL) fflush(fp); if (ferror(fp)) FATAL("write error on %s", filename(fp)); return(True); } Cell *nullproc(Node **a, int n) { return 0; } FILE *redirect(int a, Node *b) /* set up all i/o redirections */ { FILE *fp; Cell *x; char *fname; x = execute(b); fname = getsval(x); fp = openfile(a, fname, NULL); if (fp == NULL) FATAL("can't open file %s", fname); tempfree(x); return fp; } struct files { FILE *fp; const char *fname; int mode; /* '|', 'a', 'w' => LE/LT, GT */ } *files; size_t nfiles; static void stdinit(void) /* in case stdin, etc., are not constants */ { nfiles = FOPEN_MAX; files = (struct files *) calloc(nfiles, sizeof(*files)); if (files == NULL) FATAL("can't allocate file memory for %zu files", nfiles); files[0].fp = stdin; - files[0].fname = "/dev/stdin"; + files[0].fname = tostring("/dev/stdin"); files[0].mode = LT; files[1].fp = stdout; - files[1].fname = "/dev/stdout"; + files[1].fname = tostring("/dev/stdout"); files[1].mode = GT; files[2].fp = stderr; - files[2].fname = "/dev/stderr"; + files[2].fname = tostring("/dev/stderr"); files[2].mode = GT; } FILE *openfile(int a, const char *us, bool *pnewflag) { const char *s = us; size_t i; int m; FILE *fp = NULL; if (*s == '\0') FATAL("null file name in print or getline"); for (i = 0; i < nfiles; i++) if (files[i].fname && strcmp(s, files[i].fname) == 0 && (a == files[i].mode || (a==APPEND && files[i].mode==GT) || a == FFLUSH)) { if (pnewflag) *pnewflag = false; return files[i].fp; } if (a == FFLUSH) /* didn't find it, so don't create it! */ return NULL; for (i = 0; i < nfiles; i++) if (files[i].fp == NULL) break; if (i >= nfiles) { struct files *nf; size_t nnf = nfiles + FOPEN_MAX; nf = (struct files *) realloc(files, nnf * sizeof(*nf)); if (nf == NULL) FATAL("cannot grow files for %s and %zu files", s, nnf); memset(&nf[nfiles], 0, FOPEN_MAX * sizeof(*nf)); nfiles = nnf; files = nf; } fflush(stdout); /* force a semblance of order */ m = a; if (a == GT) { fp = fopen(s, "w"); } else if (a == APPEND) { fp = fopen(s, "a"); m = GT; /* so can mix > and >> */ } else if (a == '|') { /* output pipe */ fp = popen(s, "w"); } else if (a == LE) { /* input pipe */ fp = popen(s, "r"); } else if (a == LT) { /* getline sval, files[i].fname) != 0) continue; - if (ferror(files[i].fp)) - FATAL("i/o error occurred on %s", files[i].fname); + if (files[i].mode == GT || files[i].mode == '|') + fflush(files[i].fp); + if (ferror(files[i].fp)) { + if ((files[i].mode == GT && files[i].fp != stderr) + || files[i].mode == '|') + FATAL("write error on %s", files[i].fname); + else + WARNING("i/o error occurred on %s", files[i].fname); + } if (files[i].fp == stdin || files[i].fp == stdout || files[i].fp == stderr) stat = freopen("/dev/null", "r+", files[i].fp) == NULL; else if (files[i].mode == '|' || files[i].mode == LE) stat = pclose(files[i].fp) == -1; else stat = fclose(files[i].fp) == EOF; if (stat) - FATAL("i/o error occurred closing %s", files[i].fname); - if (i > 2) /* don't do /dev/std... */ - xfree(files[i].fname); + WARNING("i/o error occurred closing %s", files[i].fname); + xfree(files[i].fname); files[i].fname = NULL; /* watch out for ref thru this */ files[i].fp = NULL; break; } tempfree(x); x = gettemp(); setfval(x, (Awkfloat) (stat ? -1 : 0)); return(x); - } +} void closeall(void) { size_t i; bool stat = false; for (i = 0; i < nfiles; i++) { if (! files[i].fp) continue; - if (ferror(files[i].fp)) - FATAL( "i/o error occurred on %s", files[i].fname ); - if (files[i].fp == stdin) + if (files[i].mode == GT || files[i].mode == '|') + fflush(files[i].fp); + if (ferror(files[i].fp)) { + if ((files[i].mode == GT && files[i].fp != stderr) + || files[i].mode == '|') + FATAL("write error on %s", files[i].fname); + else + WARNING("i/o error occurred on %s", files[i].fname); + } + if (files[i].fp == stdin || files[i].fp == stdout || + files[i].fp == stderr) continue; if (files[i].mode == '|' || files[i].mode == LE) stat = pclose(files[i].fp) == -1; - else if (files[i].fp == stdout || files[i].fp == stderr) - stat = fflush(files[i].fp) == EOF; else stat = fclose(files[i].fp) == EOF; if (stat) - FATAL( "i/o error occurred while closing %s", files[i].fname ); + WARNING("i/o error occurred while closing %s", files[i].fname); } } static void flush_all(void) { size_t i; for (i = 0; i < nfiles; i++) if (files[i].fp) fflush(files[i].fp); } void backsub(char **pb_ptr, const char **sptr_ptr); Cell *sub(Node **a, int nnn) /* substitute command */ { const char *sptr, *q; Cell *x, *y, *result; char *t, *buf, *pb; fa *pfa; int bufsz = recsize; if ((buf = (char *) malloc(bufsz)) == NULL) FATAL("out of memory in sub"); x = execute(a[3]); /* target string */ t = getsval(x); if (a[0] == NULL) /* 0 => a[1] is already-compiled regexpr */ pfa = (fa *) a[1]; /* regular expression */ else { y = execute(a[1]); pfa = makedfa(getsval(y), 1); tempfree(y); } y = execute(a[2]); /* replacement string */ result = False; if (pmatch(pfa, t)) { sptr = t; adjbuf(&buf, &bufsz, 1+patbeg-sptr, recsize, 0, "sub"); pb = buf; while (sptr < patbeg) *pb++ = *sptr++; sptr = getsval(y); while (*sptr != '\0') { adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "sub"); if (*sptr == '\\') { backsub(&pb, &sptr); } else if (*sptr == '&') { sptr++; adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "sub"); for (q = patbeg; q < patbeg+patlen; ) *pb++ = *q++; } else *pb++ = *sptr++; } *pb = '\0'; if (pb > buf + bufsz) FATAL("sub result1 %.30s too big; can't happen", buf); sptr = patbeg + patlen; if ((patlen == 0 && *patbeg) || (patlen && *(sptr-1))) { adjbuf(&buf, &bufsz, 1+strlen(sptr)+pb-buf, 0, &pb, "sub"); while ((*pb++ = *sptr++) != '\0') continue; } if (pb > buf + bufsz) FATAL("sub result2 %.30s too big; can't happen", buf); setsval(x, buf); /* BUG: should be able to avoid copy */ result = True; } tempfree(x); tempfree(y); free(buf); return result; } Cell *gsub(Node **a, int nnn) /* global substitute */ { Cell *x, *y; char *rptr, *pb; const char *q, *t, *sptr; char *buf; fa *pfa; int mflag, tempstat, num; int bufsz = recsize; + int charlen = 0; if ((buf = (char *) malloc(bufsz)) == NULL) FATAL("out of memory in gsub"); mflag = 0; /* if mflag == 0, can replace empty string */ num = 0; x = execute(a[3]); /* target string */ t = getsval(x); if (a[0] == NULL) /* 0 => a[1] is already-compiled regexpr */ pfa = (fa *) a[1]; /* regular expression */ else { y = execute(a[1]); pfa = makedfa(getsval(y), 1); tempfree(y); } y = execute(a[2]); /* replacement string */ if (pmatch(pfa, t)) { tempstat = pfa->initstat; pfa->initstat = 2; pb = buf; rptr = getsval(y); do { if (patlen == 0 && *patbeg != '\0') { /* matched empty string */ if (mflag == 0) { /* can replace empty */ num++; sptr = rptr; while (*sptr != '\0') { adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gsub"); if (*sptr == '\\') { backsub(&pb, &sptr); } else if (*sptr == '&') { sptr++; adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gsub"); for (q = patbeg; q < patbeg+patlen; ) *pb++ = *q++; } else *pb++ = *sptr++; } } if (*t == '\0') /* at end */ goto done; adjbuf(&buf, &bufsz, 2+pb-buf, recsize, &pb, "gsub"); - *pb++ = *t++; + charlen = u8_nextlen(t); + while (charlen-- > 0) + *pb++ = *t++; if (pb > buf + bufsz) /* BUG: not sure of this test */ FATAL("gsub result0 %.30s too big; can't happen", buf); mflag = 0; } else { /* matched nonempty string */ num++; sptr = t; adjbuf(&buf, &bufsz, 1+(patbeg-sptr)+pb-buf, recsize, &pb, "gsub"); while (sptr < patbeg) *pb++ = *sptr++; sptr = rptr; while (*sptr != '\0') { adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gsub"); if (*sptr == '\\') { backsub(&pb, &sptr); } else if (*sptr == '&') { sptr++; adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gsub"); for (q = patbeg; q < patbeg+patlen; ) *pb++ = *q++; } else *pb++ = *sptr++; } t = patbeg + patlen; if (patlen == 0 || *t == '\0' || *(t-1) == '\0') goto done; if (pb > buf + bufsz) FATAL("gsub result1 %.30s too big; can't happen", buf); mflag = 1; } } while (pmatch(pfa,t)); sptr = t; adjbuf(&buf, &bufsz, 1+strlen(sptr)+pb-buf, 0, &pb, "gsub"); while ((*pb++ = *sptr++) != '\0') continue; done: if (pb < buf + bufsz) *pb = '\0'; else if (*(pb-1) != '\0') FATAL("gsub result2 %.30s truncated; can't happen", buf); setsval(x, buf); /* BUG: should be able to avoid copy + free */ pfa->initstat = tempstat; } tempfree(x); tempfree(y); x = gettemp(); x->tval = NUM; x->fval = num; free(buf); return(x); } Cell *gensub(Node **a, int nnn) /* global selective substitute */ /* XXX incomplete - doesn't support backreferences \0 ... \9 */ { Cell *x, *y, *res, *h; char *rptr; const char *sptr; char *buf, *pb; const char *t, *q; fa *pfa; int mflag, tempstat, num, whichm; int bufsz = recsize; if ((buf = malloc(bufsz)) == NULL) FATAL("out of memory in gensub"); mflag = 0; /* if mflag == 0, can replace empty string */ num = 0; x = execute(a[4]); /* source string */ t = getsval(x); res = copycell(x); /* target string - initially copy of source */ res->csub = CTEMP; /* result values are temporary */ if (a[0] == 0) /* 0 => a[1] is already-compiled regexpr */ pfa = (fa *) a[1]; /* regular expression */ else { y = execute(a[1]); pfa = makedfa(getsval(y), 1); tempfree(y); } y = execute(a[2]); /* replacement string */ h = execute(a[3]); /* which matches should be replaced */ sptr = getsval(h); if (sptr[0] == 'g' || sptr[0] == 'G') whichm = -1; else { /* * The specified number is index of replacement, starting * from 1. GNU awk treats index lower than 0 same as * 1, we do same for compatibility. */ whichm = (int) getfval(h) - 1; if (whichm < 0) whichm = 0; } tempfree(h); if (pmatch(pfa, t)) { char *sl; tempstat = pfa->initstat; pfa->initstat = 2; pb = buf; rptr = getsval(y); /* * XXX if there are any backreferences in subst string, * complain now. */ for (sl = rptr; (sl = strchr(sl, '\\')) && sl[1]; sl++) { if (strchr("0123456789", sl[1])) { FATAL("gensub doesn't support backreferences (subst \"%s\")", rptr); } } do { if (whichm >= 0 && whichm != num) { num++; adjbuf(&buf, &bufsz, (pb - buf) + (patbeg - t) + patlen, recsize, &pb, "gensub"); /* copy the part of string up to and including * match to output buffer */ while (t < patbeg + patlen) *pb++ = *t++; continue; } if (patlen == 0 && *patbeg != 0) { /* matched empty string */ if (mflag == 0) { /* can replace empty */ num++; sptr = rptr; while (*sptr != 0) { adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gensub"); if (*sptr == '\\') { backsub(&pb, &sptr); } else if (*sptr == '&') { sptr++; adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gensub"); for (q = patbeg; q < patbeg+patlen; ) *pb++ = *q++; } else *pb++ = *sptr++; } } if (*t == 0) /* at end */ goto done; adjbuf(&buf, &bufsz, 2+pb-buf, recsize, &pb, "gensub"); *pb++ = *t++; if (pb > buf + bufsz) /* BUG: not sure of this test */ FATAL("gensub result0 %.30s too big; can't happen", buf); mflag = 0; } else { /* matched nonempty string */ num++; sptr = t; adjbuf(&buf, &bufsz, 1+(patbeg-sptr)+pb-buf, recsize, &pb, "gensub"); while (sptr < patbeg) *pb++ = *sptr++; sptr = rptr; while (*sptr != 0) { adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gensub"); if (*sptr == '\\') { backsub(&pb, &sptr); } else if (*sptr == '&') { sptr++; adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gensub"); for (q = patbeg; q < patbeg+patlen; ) *pb++ = *q++; } else *pb++ = *sptr++; } t = patbeg + patlen; if (patlen == 0 || *t == 0 || *(t-1) == 0) goto done; if (pb > buf + bufsz) FATAL("gensub result1 %.30s too big; can't happen", buf); mflag = 1; } } while (pmatch(pfa,t)); sptr = t; adjbuf(&buf, &bufsz, 1+strlen(sptr)+pb-buf, 0, &pb, "gensub"); while ((*pb++ = *sptr++) != 0) ; done: if (pb > buf + bufsz) FATAL("gensub result2 %.30s too big; can't happen", buf); *pb = '\0'; setsval(res, buf); pfa->initstat = tempstat; } tempfree(x); tempfree(y); free(buf); return(res); } void backsub(char **pb_ptr, const char **sptr_ptr) /* handle \\& variations */ { /* sptr[0] == '\\' */ char *pb = *pb_ptr; const char *sptr = *sptr_ptr; static bool first = true; static bool do_posix = false; if (first) { first = false; do_posix = (getenv("POSIXLY_CORRECT") != NULL); } if (sptr[1] == '\\') { if (sptr[2] == '\\' && sptr[3] == '&') { /* \\\& -> \& */ *pb++ = '\\'; *pb++ = '&'; sptr += 4; } else if (sptr[2] == '&') { /* \\& -> \ + matched */ *pb++ = '\\'; sptr += 2; } else if (do_posix) { /* \\x -> \x */ sptr++; *pb++ = *sptr++; } else { /* \\x -> \\x */ *pb++ = *sptr++; *pb++ = *sptr++; } } else if (sptr[1] == '&') { /* literal & */ sptr++; *pb++ = *sptr++; } else /* literal \ */ *pb++ = *sptr++; *pb_ptr = pb; *sptr_ptr = sptr; } + +static char *wide_char_to_byte_str(int rune, size_t *outlen) +{ + static char buf[5]; + int len; + + if (rune < 0 || rune > 0x10FFFF) + return NULL; + + memset(buf, 0, sizeof(buf)); + + len = 0; + if (rune <= 0x0000007F) { + buf[len++] = rune; + } else if (rune <= 0x000007FF) { + // 110xxxxx 10xxxxxx + buf[len++] = 0xC0 | (rune >> 6); + buf[len++] = 0x80 | (rune & 0x3F); + } else if (rune <= 0x0000FFFF) { + // 1110xxxx 10xxxxxx 10xxxxxx + buf[len++] = 0xE0 | (rune >> 12); + buf[len++] = 0x80 | ((rune >> 6) & 0x3F); + buf[len++] = 0x80 | (rune & 0x3F); + + } else { + // 0x00010000 - 0x10FFFF + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + buf[len++] = 0xF0 | (rune >> 18); + buf[len++] = 0x80 | ((rune >> 12) & 0x3F); + buf[len++] = 0x80 | ((rune >> 6) & 0x3F); + buf[len++] = 0x80 | (rune & 0x3F); + } + + *outlen = len; + buf[len++] = '\0'; + + return buf; +} diff --git a/contrib/one-true-awk/testdir/T.argv b/contrib/one-true-awk/testdir/T.argv index b9a67ef39099..55e2754cd8b5 100755 --- a/contrib/one-true-awk/testdir/T.argv +++ b/contrib/one-true-awk/testdir/T.argv @@ -1,144 +1,150 @@ echo T.argv: misc tests of argc and argv awk=${awk-../a.out} echo >foo1 echo >foo2 $awk ' BEGIN { for (i = 1; i < ARGC-1; i++) printf "%s ", ARGV[i] if (ARGC > 1) printf "%s", ARGV[i] printf "\n" exit }' * >foo1 echo * >foo2 diff foo1 foo2 || echo 'BAD: T.argv (echo1 *)' $awk ' BEGIN { for (i = 1; i < ARGC; i++) { printf "%s", ARGV[i] if (i < ARGC-1) printf " " } printf "\n" exit }' * >foo1 echo * >foo2 diff foo1 foo2 || echo 'BAD: T.argv (echo2 *)' $awk ' BEGIN { print ARGC ARGV[ARGC-1] = "" for (i=0; i < ARGC; i++) print ARGV[i] exit } ' a bc def gh >foo1 echo "5 $awk a bc def " >foo2 diff foo1 foo2 || echo 'BAD: T.argv (argc *)' echo '1 2 3' >foo0 echo 'foo1 foo2 foo3' >foo1 $awk '{print L $0}' L=foo foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo foo0 echo 'foo1 foo2 foo3' >foo1 $awk '{print L $0}' L=foo foo0 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo foo1)' echo '1 2 3' >foo0 echo 'foo1 foo2 foo3' >foo1 cat foo0 | $awk '{print L $0}' L=foo - >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo -)' echo '1 2 3' >foo0 echo 'foo1 foo2 foo3 glop1 glop2 glop3' >foo1 $awk '{print L $0}' L=foo foo0 L=glop foo0 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo L=glop)' echo '1 2 3' >foo0 echo '111 112 113 221 222 223' >foo1 $awk '{print L $0}' L=11 foo0 L=22 foo0 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=11 L=22)' +echo >foo0 +echo 'name=value +name=value' >foo1 +$awk 'BEGIN { print ARGV[1] } { print ARGV[1] }' name=value foo0 >foo2 +diff foo1 foo2 || echo 'BAD: T.argv assignment operand modified' + echo 3.345 >foo1 $awk 'BEGIN { print ARGV[1] + ARGV[2]}' 1 2.345 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (ARGV[1] + ARGV[2])' echo 3.345 >foo1 x1=1 x2=2.345 $awk 'BEGIN { print ENVIRON["x1"] + ENVIRON["x2"]}' 1 2.345 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (ENVIRON[x1] + ENVIRON[x2])' echo 'foo1' >foo1 echo 'foo2' >foo2 echo 'foo3' >foo3 $awk 'BEGIN { ARGV[2] = "" } { print }' foo1 foo2 foo3 >foo4 echo 'foo1 foo3' >foo5 diff foo4 foo5 || echo 'BAD: T.argv zap ARGV[2]' echo hi > foo1 ; mv foo1 foo2 $awk 'BEGIN { ARGV[1] = "foo2" ; print FILENAME } { print FILENAME }' foo1 >foo3 echo ' foo2' >foo4 diff foo3 foo4 || echo 'BAD: T.argv startup FILENAME' # assumes that startup FILENAME is "" # test data balanced on pinhead... echo 'ARGV[3] is /dev/null ARGV[0] is ../a.out ARGV[1] is /dev/null' >foo1 $awk 'BEGIN { # this is a variant of arnolds original example ARGV[1] = "/dev/null" ARGV[2] = "glotch" # file open must skipped deleted argv ARGV[3] = "/dev/null" ARGC = 4 delete ARGV[2] } # note that input is read here END { for (i in ARGV) printf("ARGV[%d] is %s\n", i, ARGV[i]) }' >foo2 diff foo1 foo2 || echo 'BAD: T.argv delete ARGV[2]' diff --git a/contrib/one-true-awk/testdir/T.csv b/contrib/one-true-awk/testdir/T.csv new file mode 100755 index 000000000000..10da1ea90b8b --- /dev/null +++ b/contrib/one-true-awk/testdir/T.csv @@ -0,0 +1,81 @@ +#!/bin/sh + +echo T.csv: tests of csv field splitting, no embedded newlines + +awk=${awk-../a.out} + +$awk ' +BEGIN { + FS = "\t" + awk = "../a.out --csv" +} +NF == 0 || $1 ~ /^#/ { + next +} +$1 ~ /try/ { # new test + nt++ + sub(/try /, "") + prog = $0 + printf("%3d %s\n", nt, prog) + prog = sprintf("%s -F\"\\t\" '"'"'%s'"'"'", awk, prog) + # print "prog is", prog + nt2 = 0 + while (getline > 0) { + if (NF == 0) # blank line terminates a sequence + break + input = $1 + for (i = 2; i < NF; i++) # input data + input = input "\t" $i + test = sprintf("./echo '"'"'%s'"'"' | %s >foo1; ", + input, prog) + if ($NF == "\"\"") + output = ">foo2;" + else + output = sprintf("./echo '"'"'%s'"'"' >foo2; ", $NF) + gsub(/\\t/, "\t", output) + gsub(/\\n/, "\n", output) + run = sprintf("cmp foo1 foo2 || echo test %d.%d failed", + nt, ++nt2) + # print "input is", input + # print "test is", test + # print "output is", output + # print "run is", run + system(test output run) + } + tt += nt2 +} +END { print tt, "tests" } +' <<\!!!! +# General format: +# try program as rest of line +# $1 $2 $3 output1 (\t for tab, \n for newline, +# $1 $2 $3 output2 ("" for null) +# ... terminated by blank line + + +try { for (i=1; i<=NF; i++) printf("[%s]", $i); printf("\n") } +a [a] + a [ a] +,a [][a] + , a [ ][ a] +a,b [a][b] +a,b,c [a][b][c] +"" [] +"abc" [abc] +"a""b" [a"b] +"a","b" [a][b] +a""b [a""b] +"a,b" [a,b] +"""" ["] +"""""" [""] +"""x""" ["x"] +""",""" [","] +,,"" [][][] +a""b [a""b] +a''b [a''b] +,, [][][] +a, [a][] +"", [][] +, [][] +a"b [a"b] +!!!! diff --git a/contrib/one-true-awk/testdir/T.misc b/contrib/one-true-awk/testdir/T.misc index ad34ab8fdd8b..1e5c3c553d9e 100755 --- a/contrib/one-true-awk/testdir/T.misc +++ b/contrib/one-true-awk/testdir/T.misc @@ -1,506 +1,512 @@ #!/bin/sh echo T.misc: miscellaneous buglets now watched for awk=${awk-../a.out} rm -f core echo 'The big brown over the lazy doe The big brown over the lazy dog x The big brown over the lazy dog' >foo echo 'failed succeeded failed succeeded' >foo1 $awk '{ if (match($0, /^The big brown over the lazy dog/) == 0) { printf("failed\n") } else { printf("succeeded\n") } } ' foo >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc ghosh RE bug' echo '123 1234567890 12345678901' >foo echo '12345678901' >foo1 $awk 'length($0) > 10' foo >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc last number bug' # check some \ sequences in strings (ascii) echo HIJKL >foo1 echo foo | $awk '{ print "H\x49\x4a\x4BL" }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc hex string cvt' echo 012x45 >foo1 $awk 'BEGIN { print "0\061\62x\0645" }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc oct string cvt' # $i++ means ($i)++ echo 3 5 | $awk '{ i = 1; print $i++ ; print $1, i }' >foo1 echo '3 4 1' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc bad field increment' # makes sure that fields are recomputed even if self-assignment # take into account that subtracting from NF now rebuilds the record echo 'a b c s p q r x y z' >foo echo 'a s p x' >foo1 $awk '{ NF -= 2; $1 = $1; print }' foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad field self-assignment" echo '1 1' >foo1 $awk 'BEGIN {x = 1; print x; x = x; print x}' >foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad self-assignment" echo 573109312 | $awk '{print $1*4}' >foo1 echo 2292437248 >foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad overflow" # note that there are 8-bit characters in the echo # some shells will probably screw this up. echo '# code € 1 code € 2' | $awk '/^#/' >foo1 echo '#' >foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad match of 8-bit char" echo hello | $awk 'BEGIN { FILENAME = "/etc/passwd" } { print $0 }' >/dev/null if test -r core; then echo 1>&2 "BAD: T.misc /etc/passwd dropped core"; fi echo hello | $awk ' function foo(foo) { foo = 1 foo() } { foo(bar) } ' >/dev/null 2>&1 if test -r core; then echo 1>&2 "BAD: T.misc function foo(foo) dropped core" rm -f core fi echo '2 10' | $awk '{ x[NR] = $0 } # test whether $0 is NUM as well as STR END { if (x[1] > x[2]) print "BAD: T.misc: $0 is not NUM" }' $awk 'BEGIN { npad = substr("alexander" " ",1,15) print npad }' >foo grep '\\' foo && echo 1>&2 "BAD: T.misc alexander fails" # This should give an error about function arguments $awk ' function foo(x) { print "x is" x } BEGIN { foo(foo) } ' 2>foo grep "can't use function foo" foo >/dev/null || echo 1>&2 "BAD: T.misc fcn args" # gawk defref test; should give error about undefined function $awk 'BEGIN { foo() }' 2>foo grep "calling undefined function foo" foo >/dev/null || echo 1>&2 "BAD: T.misc undefined function" # gawk arrayparm test; should give error about function $awk ' BEGIN { foo[1]=1; foo[2]=2; bug1(foo); } function bug1(i) { for (i in foo) { bug2(i); delete foo[i]; print i,1,bot[1]; } } function bug2(arg) { bot[arg]=arg; } ' 2>foo grep "can.t assign to foo" foo >/dev/null || echo 1>&2 "BAD: T.misc foo bug" # This should be a syntax error $awk ' !x = y ' 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error !x=y fails" # This should print bbb $awk ' BEGIN { up[1] = "a" for (i in up) gsub("a", "A", x) print x x "bbb" exit } ' >foo grep bbb foo >/dev/null || echo 1>&2 "BAD: T.misc gsub failed" echo yes | $awk ' BEGIN { printf "push return" >"/dev/null" getline ans <"/dev/null" } ' if test -r core; then echo 1>&2 "BAD: T.misc getline ans dropped core"; fi $awk 'BEGIN { unireghf() } function unireghf(hfeed) { hfeed[1] = 0 }' if test -r core; then echo 1>&2 "BAD: T.misc unireghf dropped core"; fi echo x | $awk '/[/]/' 2>foo grep 'nonterminated character class' foo >/dev/null || error 'BAD: T.misc nonterminated fails' if test -r core; then echo 1>&2 "BAD: T.misc nonterminated dropped core"; fi $awk ' function f() { return 12345 } BEGIN { printf "<%s>\n", f() } ' >foo grep '<12345>' foo >/dev/null || echo 'BAD: T.misc <12345> fails' echo 'abc def ghi jkl' >foo $awk ' BEGIN { RS = "" while (getline <"foo") print }' >foo1 $awk 'END {print NR}' foo1 | grep 4 >/dev/null || echo 'BAD: T.misc abcdef fails' # Test for RS regex matching an empty record at EOF echo a | $awk 1 RS='a\n' > foo1 cat << \EOF > foo2 EOF diff foo1 foo2 || echo 'BAD: T.misc RS regex matching an empty record at EOF fails' # Test for RS regex being reapplied echo aaa1a2a | $awk 1 RS='^a' >foo1 cat << \EOF > foo2 aa1a2a EOF diff foo1 foo2 || echo 'BAD: T.misc ^regex reapplied fails' # ^-anchored RS matching should be active at the start of each input file tee foo1 foo2 >foo3 << \EOF aaa EOF $awk 1 RS='^a' foo1 foo2 foo3 >foo4 cat << \EOF > foo5 aa aa aa EOF diff foo4 foo5 || echo 'BAD: T.misc ^RS matches the start of every input file fails' # The following should not produce a warning about changing a constant # nor about a curdled tempcell list $awk 'function f(x) { x = 2 } BEGIN { f(1) }' >foo grep '^' foo && echo 'BAD: test constant change fails' # The following should not produce a warning about a curdled tempcell list $awk 'function f(x) { x } BEGIN { f(1) }' >foo grep '^' foo && echo 'BAD: test tempcell list fails' $awk 'BEGIN { print 9, a=10, 11; print a; exit }' >foo1 echo '9 10 11 10' >foo2 diff foo1 foo2 || echo 'BAD: T.misc (embedded expression)' echo "abc defgh ijkl" | $awk ' { $1 = ""; line = $0; print line; print $0; $0 = line; print $0 }' >foo1 echo " defgh ijkl defgh ijkl defgh ijkl" >foo2 diff foo1 foo2 || echo 'BAD: T.misc (assignment to $0)' $awk ' function min(a, b) { if (a < b) return a else return b } BEGIN { exit } ' if test -r core; then echo 1>&2 "BAD: T.misc function min dropped core"; fi # The following should not give a syntax error message: $awk ' function expand(chart) { getline chart < "CHAR.ticks" } ' >foo grep '^' foo >/dev/null && echo 'BAD: T.misc expand error' $awk 'BEGIN { print 1e40 }' >/dev/null if test -r core; then echo 1>&2 "BAD: T.misc 1E40 dropped core"; fi # The following syntax error should not dump core: $awk ' $NF==3 {first=1} $NF==2 && first==0 && (abs($1-o1)>120||abs($2-o2)>120) {print $0} $NF==2 {o1=%1; o2=$2; first=0} ' 2>/dev/null if test -r core; then echo 1>&2 "BAD: T.misc first/abs dropped core"; fi # The following syntax error should not dump core: $awk '{ n = split($1, address, !); print address[1] }' 2>foo grep 'illegal statement' foo >/dev/null || echo 'BAD: T.misc split error' if test -r core; then echo 1>&2 "BAD: T.misc split! dropped core"; fi # The following should cause a syntax error message $awk 'BEGIN {"hello"}' 2>foo grep 'illegal statement' foo >/dev/null || echo 'BAD: T.misc hello error' # The following should give a syntax error message: $awk ' function pile(c, r) { r = ++pile[c] } { pile($1) } ' 2>foo grep 'context is' foo >/dev/null || echo 'BAD: T.misc pile error' # This should complain about missing atan2 argument: $awk 'BEGIN { atan2(1) }' 2>foo grep 'requires two arg' foo >/dev/null || echo 'BAD: T.misc atan2 error' # This should not core dump: $awk 'BEGIN { f() } function f(A) { delete A[1] } ' if test -r core; then echo 1>&2 "BAD: T.misc delete dropped core"; fi # nasty one: should not be able to overwrite constants $awk 'BEGIN { gsub(/ana/,"anda","banana") printf "the monkey ate a %s\n", "banana" } ' >/dev/null 2>foo grep 'syntax error' foo >/dev/null || echo 'BAD: T.misc gsub banana error' # nasty one: should not be able to overwrite constants $awk 'BEGIN { sub(/ana/,"anda","banana") printf "the monkey ate a %s\n", "banana" } ' >/dev/null 2>foo grep 'syntax error' foo >/dev/null || echo 'BAD: T.misc sub banana error' # line numbers used to double-count comments $awk '# # # /x ' >/dev/null 2>foo grep 'line [45]' foo >/dev/null || echo 'BAD: T.misc lineno' echo 'x  \y' >foo1 $awk 'BEGIN { print "x\f\r\b\v\a\\y" }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc weird chars' echo 0 >foo1 $awk ' BEGIN { exit } { print } END { print NR }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc BEGIN exit' echo 1 >foo1 $awk ' { exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit' echo 1 >foo1 $awk ' {i = 1; while (i <= NF) {if (i == NF) exit; i++ } } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 2' echo 1 >foo1 $awk ' function f() { i = 1; while (i <= NF) {if (i == NF) return NR; i++ } } { if (f() == 1) exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc while return' echo 1 >foo1 $awk ' function f() { split("a b c", arr) for (i in arr) {if (i == 3) return NR; i++ } } { if (f() == 1) exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc while return' echo 1 >foo1 $awk ' {i = 1; do { if (i == NF) exit; i++ } while (i <= NF) } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 3' echo 1 >foo1 $awk ' function f() { i = 1; do { if (i == NF) return NR; i++ } while (i <= NF) } { if (f() == 1) exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc do return' echo 1 >foo1 $awk ' {i = 1; do { if (i == NF) break; i++ } while (i <= NF); exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 4' echo 1 >foo1 $awk ' { n = split($0, x) for (i in x) { if (i == 1) exit } } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 5' echo XXXXXXXX >foo1 $awk 'BEGIN { s = "ab\fc\rd\be" t = s; gsub("[" s "]", "X", t); print t }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc weird escapes in char class' $awk '{}' /etc/passwd glop/glop >foo 2>foo2 grep "can't open.*glop" foo2 >/dev/null || echo "BAD: T.misc can't open" echo ' a aa b c ' >foo echo 3 >foo1 $awk 'BEGIN { RS = "" }; END { print NR }' foo >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc RS botch' $awk 'BEGIN \ { print "hello, world" } }}}' >foo1 2>foo2 grep 'source line 4' foo2 >/dev/null 2>&1 || echo 'BAD: T.misc continuation line number' echo 111 222 333 >foo $awk '{ f[1]=1; f[2]=2; print $f[1], $f[1]++, $f[2], f[1], f[2] }' foo >foo2 echo 111 111 222 2 2 >foo1 cmp -s foo1 foo2 || echo 'BAD: T.misc $f[1]++' # These should be syntax errors $awk . 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error . fails" $awk .. 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error .. fails" $awk .E. 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error .E. fails" $awk .++. 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error .++. fails" # These should be syntax errors $awk '$' 2>foo grep "unexpected" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error $ fails" $awk '{print $' 2>foo grep "unexpected" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error $2 fails" $awk '"' 2>foo grep "non-terminated" foo >/dev/null || echo 1>&2 "BAD: T.misc bare quote fails" # %c of 0 is explicit null byte ./echo '3' >foo1 $awk 'BEGIN {printf("%c%c\n", 0, 0) }' | wc | $awk '{print $3}' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc null byte' # non-terminated RE $awk /xyz >foo 2>&1 grep "non-terminated" foo >/dev/null || echo 1>&2 "BAD: T.misc non-terminated RE" # next several were infinite loops, found by brian tsang. # this is his example: $awk 'BEGIN { switch (substr("x",1,1)) { case /ask.com/: break case "google": break } }' >foo 2>&1 grep "illegal statement" foo >/dev/null || echo 1>&2 "BAD: T.misc looping syntax error 1" $awk 'BEGIN { s { c /./ } }' >foo 2>&1 grep "illegal statement" foo >/dev/null || echo 1>&2 "BAD: T.misc looping syntax error 2" $awk 'BEGIN { s { c /../ } }' >foo 2>&1 grep "illegal statement" foo >/dev/null || echo 1>&2 "BAD: T.misc looping syntax error 3" $awk 'BEGIN {printf "%2$s %1$s\n", "a", "b"}' >foo 2>&1 grep "'$' not permitted in awk formats" foo >/dev/null || echo 1>&2 "BAD: T.misc '$' not permitted in formats" echo 'a b c de fg hi' >foo0 $awk 'END { print NF, $0 }' foo0 >foo1 awk '{ print NF, $0 }' foo0| tail -1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc END must preserve $0' echo 'fg hi' >foo0 $awk 'END { print NF, $0 }' foo0 >foo1 awk '{ print NF, $0 }' foo0| tail -1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc END must preserve $0' echo '' >foo0 $awk 'END { print NF, $0 }' foo0 >foo1 awk '{ print NF, $0 }' foo0| tail -1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc END must preserve $0' # Check for nonzero exit status on I/O error. echo 'E 2' >foo1 (trap '' PIPE; "$awk" 'BEGIN { print "hi"; }' 2>/dev/null; echo "E $?" >foo2) | : cmp -s foo1 foo2 || echo 'BAD: T.misc exit status on I/O error' + +# Check for clobbering of the lexer's regular expression buffer. +# If the output is "a1" instead of "1b", /b/ clobbered /a/. +echo 1b >foo1 +echo ab | $awk '{ sub(/a/, "b" ~ /b/); print }' >foo2 +cmp -s foo1 foo2 || echo 'BAD: T.misc lexer regex buffer clobbered' diff --git a/contrib/one-true-awk/testdir/T.utf b/contrib/one-true-awk/testdir/T.utf new file mode 100755 index 000000000000..18f2b9c355cf --- /dev/null +++ b/contrib/one-true-awk/testdir/T.utf @@ -0,0 +1,194 @@ +#!/bin/sh + +echo T.utf: tests of utf functions + +awk=${awk-../a.out} + +$awk ' +BEGIN { + FS = "\t" + awk = "../a.out" +} +NF == 0 || $1 ~ /^#/ { + next +} +$1 ~ /try/ { # new test + nt++ + sub(/try [a-zA-Z_0-9]+ /, "") + prog = $0 + printf("try %3d %s\n", nt, prog) + prog = sprintf("%s -F\"\\t\" '"'"'%s'"'"'", awk, prog) + # print "prog is", prog + nt2 = 0 + while (getline > 0) { + if (NF == 0) # blank line terminates a sequence + break + input = $1 + for (i = 2; i < NF; i++) # input data + input = input "\t" $i + test = sprintf("./echo '"'"'%s'"'"' | %s >foo1; ", + input, prog) + if ($NF == "\"\"") + output = ">foo2;" + else + output = sprintf("./echo '"'"'%s'"'"' >foo2; ", $NF) + gsub(/\\t/, "\t", output) + gsub(/\\n/, "\n", output) + run = sprintf("diff foo1 foo2 || echo test %d.%d failed", + nt, ++nt2) + # print "input is", input + # print "test is", test + # print "output is", output + # print "run is", run + system(test output run) + } + tt += nt2 +} +END { print tt, "tests" } +' <<\!!!! +# General format: +# try program as rest of line +# $1 $2 $3 output1 (\t for tab, \n for newline, +# $1 $2 $3 output2 ("" for null) +# ... terminated by blank line + +# try another program... + +try length { print length($1) } + 0 +a 1 +の今がその時だ 7 +Сейчас 6 +现在是时候了 6 +给所有的好男 6 +来参加聚会。 6 +😀 1 +🖕 finger 8 +Τωρα 4 +για 3 +να 2 +עכשיו 5 +לכל 3 +לבוא 4 +の今がその時だ 7 +지금이 3 +모든 2 +파티에 3 +Сейчас 6 +для 3 +прийти 6 + +try index { print index($1, $2) } +abc a 1 +abc b 2 +abc x 0 +现在是时候了 "" 0 +现在是时候了 了 6 +现在是时候了 在是 2 +现在是时候了 x 0 +现x在是时候了 x 2 +🖕 fingerすべての善人のためにすべての善人のために f 3 +🖕 finger🖕 r🖕 8 + +try substr { print substr($0, 2, 3) } +abcdef bcd +Τωρα ειναι η ωρα +Τω ω +지금 이절호의 금 이 +xпyрийти пyр + +try rematch { print $1 ~ $2 } +abc a 1 +abc x 0 +すべての善人のために の 1 +すべての善人のために の.*の 1 +すべての善人のために の.*て 0 +Τωρα ω+ 1 + +# replace first occurrence of $2 by $3 in $1 +try sub { n = sub($2, $3, $1); print n, $1 } +abcdef bc XYZ 1 aXYZdef +abcdef xy XYZ 0 abcdef +の今がその時だ の NO 1 NO今がその時だ +🖕 finger 🖕.*g FING 1 FINGer +Сейчас . x 1 xейчас + +# replace all occurrences of $2 by $3 in $1 +try gsub { n = gsub($2, $3, $1); print n, $1 } +abcdef bc XYZ 1 aXYZdef +abcdef xy XYZ 0 abcdef +の今がその時だ の NO 2 NO今がそNO時だ +🖕 finger 🖕.*g FING 1 FINGer +Сейчас . x 6 xxxxxx + +try match { print match($1, $2), RSTART, RLENGTH } +abc [^a] 2 2 1 +abc [^ab] 3 3 1 +すべての善人のために [^す] 2 2 1 +すべての善人のために [^ぁ-ゖ] 5 5 1 +abc a 1 1 1 +abc x 0 0 -1 +すべての善人のために の 4 4 1 +すべての善人のために の.*の 4 4 4 +すべての善人のために の.*て 0 0 -1 +Τωρα ω+ 2 2 1 +Τωρα x+ 0 0 -1 +Τωρα ω. 2 2 2 +すべての善人のために [の] 4 4 1 +すべての善人のために [ぁ-え] 0 0 -1 +すべての善人のために [^ぁ-え] 1 1 1 +Τωρα ειναι η [α-ω] 2 2 1 +Τωρα ειναι η [α-ω]+ 2 2 3 +xxxΤωρα ειναι η [Α-Ω] 4 4 1 +για όλους τους καλούς ά α.*α 3 3 15 +να έρθει στο πά [^ν] 2 2 1 + +# FS="" should split into unicode chars +try emptyFS BEGIN {FS=""} {print NF} +すべての善人のために 10 +の今がその時だ 7 +Сейчас 6 +现在是时候了 6 +给所有的好男 6 +来参加聚会。 6 +😀 1 +🖕 finger 8 + +# printf(%N.Ns) for utf8 strings +try printfs1 {printf("[%5.2s][%-5.2s]\n"), $1, $1} +abcd [ ab][ab ] +现在abc [ 现在][现在 ] +现ωabc [ 现ω][现ω ] +ωabc [ ωa][ωa ] +Сейчас [ Се][Се ] +Сейxyz [ Се][Се ] +😀 [ 😀][😀 ] + +# printf(%N.Ns) for utf8 strings +try printfs2 {printf("[%5s][%-5s]\n"), $1, $1} +abcd [ abcd][abcd ] +现在ab [ 现在ab][现在ab ] +a现在ab [a现在ab][a现在ab] +a现在abc [a现在abc][a现在abc] +现ωab [ 现ωab][现ωab ] +ωabc [ ωabc][ωabc ] +Сейчас [Сейчас][Сейчас] +😀 [ 😀][😀 ] + +# printf(%N.Ns) for utf8 strings +try printfs3 {printf("[%.2s][%-.2s]\n"), $1, $1} +abcd [ab][ab] +现在abc [现在][现在] +现ωabc [现ω][现ω] +ω [ω][ω] +😀 [😀][😀] + +# printf(%c) for utf +try printfc {printf("%c %c\n", $1, substr($1,2,1))} +すべての善人のために す べ +の今がその時だ の 今 +Сейчас С е +现在是时候了 现 在 +😀🖕 😀 🖕 + +!!!! diff --git a/contrib/one-true-awk/testdir/T.utfre b/contrib/one-true-awk/testdir/T.utfre new file mode 100755 index 000000000000..20e66cbde9a5 --- /dev/null +++ b/contrib/one-true-awk/testdir/T.utfre @@ -0,0 +1,234 @@ +echo T.utfre: tests of regular expression code for Unicode/utf-8 +# adapted from T.re + +awk ' +BEGIN { + FS = "\t" + awk = "../a.out" +} +NF == 0 { + next +} +$1 != "" { # new test + re = $1 +} +$2 != "" { # either ~ or !~ + op = $2 + if (op == "~") + neg = "!" + else if (op == "!~") + neg = "" +} +$3 != "" { # new test string + str = $3 +} +$3 == "\"\"" { # explicit empty line + $3 = "" +} +NF > 2 { # generate a test + input = $3 + test = sprintf("./echo '"'"'%s'"'"' | %s '"'"'%s/%s/ {print \"%d fails %s %s %s\"}'"'"'", + input, awk, neg, re, NR, re, op, input) + # printf(" %3d %s %s %s:\n", NR, re, op, input) + # print "test is |" test "|" + system(test) + # system("bprint -c ../a.out") + nt++ +} +END { print " " nt, "tests" } +' <<\!!!! + ~ 🖕 + 🖕🖕 + 🖕🖕🖕 + "" +🖕 ~ 🖕 + b🖕 + b🖕b + !~ "" + 时 + xxxxx +. ~ 时 + x时x + 🙂 + !~ "" +.の ~ xの + xxの + xのx + !~ の + のx + "" +$ ~ x + 🙂 + "" +.$ ~ 모 + xx모 + x모x + !~ "" +д$ ~ д + bд + bbbд + !~ дb + x + "" +^ ~ и + "" + ^ +^λ$ ~ λ + !~ xλ + λx + xλx + "" +^λ.$ ~ λx + λλ + !~ xλ + λλλ + λxy + "" +^$ ~ "" + !~ に + ^ +^.해 ~ め해 + め해해 + !~ 해 + "" +^.*해 ~ 해 + め해 + めめめめめめ해 + !~ "" +^.+해 ~ め해 + めめめめめめ해 + !~ "" + 해 + 해め +해* ~ "" + 해 + 해해해해 + め해 + めめめめ +해해* ~ 해 + 해해해 + め해 + !~ めめめめ + "" +\$ ~ 🖕$ + $ + $🖕 + 🖕$🖕 + !~ "" + 🖕 +\. ~ . + !~ 🖕 + "" +xθ+y ~ xθy + xθθy + xθθθθθθy + !~ θy + xy + xθ +xθ?y ~ xy + xθy + !~ xθθy +θ?b?の? ~ "" + x +^a?b?め ~ め + aめ + bめ + abめ + めa + !~ "" + ab + aba +[Α-Ω] ~ Α + aΔb + xyΩ + !~ abc + β + "" +[^Α-Ω] ~ δ + aΔb + xyΩ + !~ Α + Δ + "" +[Α-ΔΦ-Ω] ~ Α + Β + Δ + Φ + Ω + !~ Π + Σ +Π[[:lower:]]+ ~ Πa + Πab + !~ Π + ΠX + Π: + Π[ + Π] +の[0-9]+に ~ の0に + の23に + の12345に + !~ 0に + のに +の[0-9]?に ~ のに + の1に + !~ の23に +の[[]に ~ の[に + !~ のに + の[[]に + の]に +の[[-]に ~ の[に + の-に + !~ のに + の[[]に + の]に +の[[-a]に ~ の[に + のaに + の]に + !~ のに + の[[]に + の-に +の[]-a]に ~ の]に + のaに + !~ のに + の[に + の-に +の[]]に ~ の]に + !~ のに + の[]]に + の[に +の[^[]に ~ のaに + !~ の[に +の[-]に ~ の-に + !~ のに + の+に +の[^-]に ~ の+に + !~ の-に + のに +の[][]に ~ の[に + の]に + !~ のに + の][に + の[]に +の[z-a]に ~ のに + !~ の + に + のaに + のzに + の-に +に|だ ~ だ + に + だに + !~ a + "" +^στο|τους$ ~ στο + στοd + aτους + τους + !~ xστο + τουςa +^(στο|τους)$ ~ στο + τους + !~ στοτους + στοx + cτους +!!!! diff --git a/contrib/one-true-awk/testdir/tt.15 b/contrib/one-true-awk/testdir/tt.15 index c05e61f5aa12..cd48064f0d6c 100644 --- a/contrib/one-true-awk/testdir/tt.15 +++ b/contrib/one-true-awk/testdir/tt.15 @@ -1,33 +1,33 @@ # fmt - format # input: text # output: text formatted into lines of <= 72 characters BEGIN { maxlen = 72 } /^[ \t]/ { printline(); print; next } # verbatim ###/^ +/ { printline(); } # whitespace == break /./ { for (i = 1; i <= NF; i++) addword($i); next } /^$/ { printline(); print "" } END { printline() } function addword(w) { - ## print "adding [", w, "] ", length(w), length(line), maxlen + print "adding [", w, "] ", length(w), length(line), maxlen if (length(line) + length(w) > maxlen) printline() if (length(w) > 2 && ( w ~ /[\.!]["?)]?$/ || w ~ /[?!]"?$/) && w !~ /^(Mr|Dr|Ms|Mrs|vs|Ph.D)\.$/) w = w " " line = line " " w } function printline() { if (length(line) > 0) { sub(/ +$/, "", line) print substr(line, 2) # removes leading blank line = "" } } diff --git a/contrib/one-true-awk/testdir/xc b/contrib/one-true-awk/testdir/xc deleted file mode 100755 index 57eb5210eede..000000000000 --- a/contrib/one-true-awk/testdir/xc +++ /dev/null @@ -1,17 +0,0 @@ -for i in $* -do - echo $i >/dev/tty - echo $i '<<<' - cd .. - echo testdir/$i: - ind drek.c - cat drek.c - make drek || ( echo $i ' ' bad compile; echo $i ' ' bad compile >/dev/tty; continue ) - cd testdir - time awk -f $i test.countries >foo1 - time ../drek test.countries >foo2 - cmp foo1 foo2 || ( echo $i ' ' bad; echo $i ' ' bad >/dev/tty; diff foo1 foo2 ) - echo '>>>' $i - echo -done diff --git a/contrib/one-true-awk/tran.c b/contrib/one-true-awk/tran.c index c6ae890c8a91..482eede1de4c 100644 --- a/contrib/one-true-awk/tran.c +++ b/contrib/one-true-awk/tran.c @@ -1,658 +1,648 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include #include #include #include "awk.h" #define FULLTAB 2 /* rehash when table gets this x full */ #define GROWTAB 4 /* grow table by this factor */ Array *symtab; /* main symbol table */ char **FS; /* initial field sep */ char **RS; /* initial record sep */ char **OFS; /* output field sep */ char **ORS; /* output record sep */ char **OFMT; /* output format for numbers */ char **CONVFMT; /* format for conversions in getsval */ Awkfloat *NF; /* number of fields in current record */ Awkfloat *NR; /* number of current record */ Awkfloat *FNR; /* number of current record in current file */ char **FILENAME; /* current filename argument */ Awkfloat *ARGC; /* number of arguments from command line */ char **SUBSEP; /* subscript separator for a[i,j,k]; default \034 */ Awkfloat *RSTART; /* start of re matched with ~; origin 1 (!) */ Awkfloat *RLENGTH; /* length of same */ Cell *fsloc; /* FS */ Cell *nrloc; /* NR */ Cell *nfloc; /* NF */ Cell *fnrloc; /* FNR */ Cell *ofsloc; /* OFS */ Cell *orsloc; /* ORS */ Cell *rsloc; /* RS */ Array *ARGVtab; /* symbol table containing ARGV[...] */ Array *ENVtab; /* symbol table containing ENVIRON[...] */ Cell *rstartloc; /* RSTART */ Cell *rlengthloc; /* RLENGTH */ Cell *subseploc; /* SUBSEP */ Cell *symtabloc; /* SYMTAB */ Cell *nullloc; /* a guaranteed empty cell */ Node *nullnode; /* zero&null, converted into a node for comparisons */ Cell *literal0; extern Cell **fldtab; -static void -setfree(Cell *vp) -{ - if (&vp->sval == FS || &vp->sval == RS || - &vp->sval == OFS || &vp->sval == ORS || - &vp->sval == OFMT || &vp->sval == CONVFMT || - &vp->sval == FILENAME || &vp->sval == SUBSEP) - vp->tval |= DONTFREE; - else - vp->tval &= ~DONTFREE; -} - void syminit(void) /* initialize symbol table with builtin vars */ { literal0 = setsymtab("0", "0", 0.0, NUM|STR|CON|DONTFREE, symtab); /* this is used for if(x)... tests: */ nullloc = setsymtab("$zero&null", "", 0.0, NUM|STR|CON|DONTFREE, symtab); nullnode = celltonode(nullloc, CCON); fsloc = setsymtab("FS", " ", 0.0, STR|DONTFREE, symtab); FS = &fsloc->sval; rsloc = setsymtab("RS", "\n", 0.0, STR|DONTFREE, symtab); RS = &rsloc->sval; ofsloc = setsymtab("OFS", " ", 0.0, STR|DONTFREE, symtab); OFS = &ofsloc->sval; orsloc = setsymtab("ORS", "\n", 0.0, STR|DONTFREE, symtab); ORS = &orsloc->sval; OFMT = &setsymtab("OFMT", "%.6g", 0.0, STR|DONTFREE, symtab)->sval; CONVFMT = &setsymtab("CONVFMT", "%.6g", 0.0, STR|DONTFREE, symtab)->sval; FILENAME = &setsymtab("FILENAME", "", 0.0, STR|DONTFREE, symtab)->sval; nfloc = setsymtab("NF", "", 0.0, NUM, symtab); NF = &nfloc->fval; nrloc = setsymtab("NR", "", 0.0, NUM, symtab); NR = &nrloc->fval; fnrloc = setsymtab("FNR", "", 0.0, NUM, symtab); FNR = &fnrloc->fval; subseploc = setsymtab("SUBSEP", "\034", 0.0, STR|DONTFREE, symtab); SUBSEP = &subseploc->sval; rstartloc = setsymtab("RSTART", "", 0.0, NUM, symtab); RSTART = &rstartloc->fval; rlengthloc = setsymtab("RLENGTH", "", 0.0, NUM, symtab); RLENGTH = &rlengthloc->fval; symtabloc = setsymtab("SYMTAB", "", 0.0, ARR, symtab); free(symtabloc->sval); symtabloc->sval = (char *) symtab; } void arginit(int ac, char **av) /* set up ARGV and ARGC */ { Cell *cp; int i; char temp[50]; ARGC = &setsymtab("ARGC", "", (Awkfloat) ac, NUM, symtab)->fval; cp = setsymtab("ARGV", "", 0.0, ARR, symtab); ARGVtab = makesymtab(NSYMTAB); /* could be (int) ARGC as well */ free(cp->sval); cp->sval = (char *) ARGVtab; for (i = 0; i < ac; i++) { double result; sprintf(temp, "%d", i); if (is_number(*av, & result)) setsymtab(temp, *av, result, STR|NUM, ARGVtab); else setsymtab(temp, *av, 0.0, STR, ARGVtab); av++; } } void envinit(char **envp) /* set up ENVIRON variable */ { Cell *cp; char *p; cp = setsymtab("ENVIRON", "", 0.0, ARR, symtab); ENVtab = makesymtab(NSYMTAB); free(cp->sval); cp->sval = (char *) ENVtab; for ( ; *envp; envp++) { double result; if ((p = strchr(*envp, '=')) == NULL) continue; if( p == *envp ) /* no left hand side name in env string */ continue; *p++ = 0; /* split into two strings at = */ if (is_number(p, & result)) setsymtab(*envp, p, result, STR|NUM, ENVtab); else setsymtab(*envp, p, 0.0, STR, ENVtab); p[-1] = '='; /* restore in case env is passed down to a shell */ } } Array *makesymtab(int n) /* make a new symbol table */ { Array *ap; Cell **tp; ap = (Array *) malloc(sizeof(*ap)); tp = (Cell **) calloc(n, sizeof(*tp)); if (ap == NULL || tp == NULL) FATAL("out of space in makesymtab"); ap->nelem = 0; ap->size = n; ap->tab = tp; return(ap); } void freesymtab(Cell *ap) /* free a symbol table */ { Cell *cp, *temp; Array *tp; int i; if (!isarr(ap)) return; tp = (Array *) ap->sval; if (tp == NULL) return; for (i = 0; i < tp->size; i++) { for (cp = tp->tab[i]; cp != NULL; cp = temp) { xfree(cp->nval); if (freeable(cp)) xfree(cp->sval); temp = cp->cnext; /* avoids freeing then using */ free(cp); tp->nelem--; } tp->tab[i] = NULL; } if (tp->nelem != 0) WARNING("can't happen: inconsistent element count freeing %s", ap->nval); free(tp->tab); free(tp); } void freeelem(Cell *ap, const char *s) /* free elem s from ap (i.e., ap["s"] */ { Array *tp; Cell *p, *prev = NULL; int h; tp = (Array *) ap->sval; h = hash(s, tp->size); for (p = tp->tab[h]; p != NULL; prev = p, p = p->cnext) if (strcmp(s, p->nval) == 0) { if (prev == NULL) /* 1st one */ tp->tab[h] = p->cnext; else /* middle somewhere */ prev->cnext = p->cnext; if (freeable(p)) xfree(p->sval); free(p->nval); free(p); tp->nelem--; return; } } Cell *setsymtab(const char *n, const char *s, Awkfloat f, unsigned t, Array *tp) { int h; Cell *p; if (n != NULL && (p = lookup(n, tp)) != NULL) { DPRINTF("setsymtab found %p: n=%s s=\"%s\" f=%g t=%o\n", (void*)p, NN(p->nval), NN(p->sval), p->fval, p->tval); return(p); } p = (Cell *) malloc(sizeof(*p)); if (p == NULL) FATAL("out of space for symbol table at %s", n); p->nval = tostring(n); p->sval = s ? tostring(s) : tostring(""); p->fval = f; p->tval = t; p->csub = CUNK; p->ctype = OCELL; tp->nelem++; if (tp->nelem > FULLTAB * tp->size) rehash(tp); h = hash(n, tp->size); p->cnext = tp->tab[h]; tp->tab[h] = p; DPRINTF("setsymtab set %p: n=%s s=\"%s\" f=%g t=%o\n", (void*)p, p->nval, p->sval, p->fval, p->tval); return(p); } int hash(const char *s, int n) /* form hash value for string s */ { unsigned hashval; for (hashval = 0; *s != '\0'; s++) hashval = (*s + 31 * hashval); return hashval % n; } void rehash(Array *tp) /* rehash items in small table into big one */ { int i, nh, nsz; Cell *cp, *op, **np; nsz = GROWTAB * tp->size; np = (Cell **) calloc(nsz, sizeof(*np)); if (np == NULL) /* can't do it, but can keep running. */ return; /* someone else will run out later. */ for (i = 0; i < tp->size; i++) { for (cp = tp->tab[i]; cp; cp = op) { op = cp->cnext; nh = hash(cp->nval, nsz); cp->cnext = np[nh]; np[nh] = cp; } } free(tp->tab); tp->tab = np; tp->size = nsz; } Cell *lookup(const char *s, Array *tp) /* look for s in tp */ { Cell *p; int h; h = hash(s, tp->size); for (p = tp->tab[h]; p != NULL; p = p->cnext) if (strcmp(s, p->nval) == 0) return(p); /* found it */ return(NULL); /* not found */ } Awkfloat setfval(Cell *vp, Awkfloat f) /* set float val of a Cell */ { int fldno; f += 0.0; /* normalise negative zero to positive zero */ if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "assign to"); if (isfld(vp)) { donerec = false; /* mark $0 invalid */ fldno = atoi(vp->nval); if (fldno > *NF) newfld(fldno); DPRINTF("setting field %d to %g\n", fldno, f); } else if (&vp->fval == NF) { donerec = false; /* mark $0 invalid */ setlastfld(f); - DPRINTF("setting NF to %g\n", f); + DPRINTF("setfval: setting NF to %g\n", f); } else if (isrec(vp)) { donefld = false; /* mark $1... invalid */ donerec = true; savefs(); } else if (vp == ofsloc) { if (!donerec) recbld(); } if (freeable(vp)) xfree(vp->sval); /* free any previous string */ vp->tval &= ~(STR|CONVC|CONVO); /* mark string invalid */ vp->fmt = NULL; vp->tval |= NUM; /* mark number ok */ if (f == -0) /* who would have thought this possible? */ f = 0; DPRINTF("setfval %p: %s = %g, t=%o\n", (void*)vp, NN(vp->nval), f, vp->tval); return vp->fval = f; } void funnyvar(Cell *vp, const char *rw) { if (isarr(vp)) FATAL("can't %s %s; it's an array name.", rw, vp->nval); if (vp->tval & FCN) FATAL("can't %s %s; it's a function.", rw, vp->nval); WARNING("funny variable %p: n=%s s=\"%s\" f=%g t=%o", (void *)vp, vp->nval, vp->sval, vp->fval, vp->tval); } char *setsval(Cell *vp, const char *s) /* set string val of a Cell */ { char *t; int fldno; Awkfloat f; DPRINTF("starting setsval %p: %s = \"%s\", t=%o, r,f=%d,%d\n", (void*)vp, NN(vp->nval), s, vp->tval, donerec, donefld); if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "assign to"); + if (CSV && (vp == rsloc)) + WARNING("danger: don't set RS when --csv is in effect"); + if (CSV && (vp == fsloc)) + WARNING("danger: don't set FS when --csv is in effect"); if (isfld(vp)) { donerec = false; /* mark $0 invalid */ fldno = atoi(vp->nval); if (fldno > *NF) newfld(fldno); DPRINTF("setting field %d to %s (%p)\n", fldno, s, (const void*)s); } else if (isrec(vp)) { donefld = false; /* mark $1... invalid */ donerec = true; savefs(); } else if (vp == ofsloc) { if (!donerec) recbld(); } t = s ? tostring(s) : tostring(""); /* in case it's self-assign */ if (freeable(vp)) xfree(vp->sval); - vp->tval &= ~(NUM|CONVC|CONVO); + vp->tval &= ~(NUM|DONTFREE|CONVC|CONVO); vp->tval |= STR; vp->fmt = NULL; - setfree(vp); DPRINTF("setsval %p: %s = \"%s (%p) \", t=%o r,f=%d,%d\n", (void*)vp, NN(vp->nval), t, (void*)t, vp->tval, donerec, donefld); vp->sval = t; if (&vp->fval == NF) { donerec = false; /* mark $0 invalid */ f = getfval(vp); setlastfld(f); - DPRINTF("setting NF to %g\n", f); + DPRINTF("setsval: setting NF to %g\n", f); } return(vp->sval); } Awkfloat getfval(Cell *vp) /* get float val of a Cell */ { if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "read value of"); if (isfld(vp) && !donefld) fldbld(); else if (isrec(vp) && !donerec) recbld(); if (!isnum(vp)) { /* not a number */ double fval; bool no_trailing; if (is_valid_number(vp->sval, true, & no_trailing, & fval)) { vp->fval = fval; if (no_trailing && !(vp->tval&CON)) vp->tval |= NUM; /* make NUM only sparingly */ } else vp->fval = 0.0; } DPRINTF("getfval %p: %s = %g, t=%o\n", (void*)vp, NN(vp->nval), vp->fval, vp->tval); return(vp->fval); } static const char *get_inf_nan(double d) { if (isinf(d)) { return (d < 0 ? "-inf" : "+inf"); } else if (isnan(d)) { return (signbit(d) != 0 ? "-nan" : "+nan"); } else return NULL; } static char *get_str_val(Cell *vp, char **fmt) /* get string val of a Cell */ { char s[256]; double dtemp; const char *p; if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "read value of"); if (isfld(vp) && ! donefld) fldbld(); else if (isrec(vp) && ! donerec) recbld(); /* * ADR: This is complicated and more fragile than is desirable. * Retrieving a string value for a number associates the string * value with the scalar. Previously, the string value was * sticky, meaning if converted via OFMT that became the value * (even though POSIX wants it to be via CONVFMT). Or if CONVFMT * changed after a string value was retrieved, the original value * was maintained and used. Also not per POSIX. * * We work around this design by adding two additional flags, * CONVC and CONVO, indicating how the string value was * obtained (via CONVFMT or OFMT) and _also_ maintaining a copy * of the pointer to the xFMT format string used for the * conversion. This pointer is only read, **never** dereferenced. * The next time we do a conversion, if it's coming from the same * xFMT as last time, and the pointer value is different, we * know that the xFMT format string changed, and we need to * redo the conversion. If it's the same, we don't have to. * * There are also several cases where we don't do a conversion, * such as for a field (see the checks below). */ /* Don't duplicate the code for actually updating the value */ #define update_str_val(vp) \ { \ if (freeable(vp)) \ xfree(vp->sval); \ if ((p = get_inf_nan(vp->fval)) != NULL) \ strcpy(s, p); \ else if (modf(vp->fval, &dtemp) == 0) /* it's integral */ \ snprintf(s, sizeof (s), "%.30g", vp->fval); \ else \ snprintf(s, sizeof (s), *fmt, vp->fval); \ vp->sval = tostring(s); \ vp->tval &= ~DONTFREE; \ vp->tval |= STR; \ } if (isstr(vp) == 0) { update_str_val(vp); if (fmt == OFMT) { vp->tval &= ~CONVC; vp->tval |= CONVO; } else { /* CONVFMT */ vp->tval &= ~CONVO; vp->tval |= CONVC; } vp->fmt = *fmt; } else if ((vp->tval & DONTFREE) != 0 || ! isnum(vp) || isfld(vp)) { goto done; } else if (isstr(vp)) { if (fmt == OFMT) { if ((vp->tval & CONVC) != 0 || ((vp->tval & CONVO) != 0 && vp->fmt != *fmt)) { update_str_val(vp); vp->tval &= ~CONVC; vp->tval |= CONVO; vp->fmt = *fmt; } } else { /* CONVFMT */ if ((vp->tval & CONVO) != 0 || ((vp->tval & CONVC) != 0 && vp->fmt != *fmt)) { update_str_val(vp); vp->tval &= ~CONVO; vp->tval |= CONVC; vp->fmt = *fmt; } } } done: DPRINTF("getsval %p: %s = \"%s (%p)\", t=%o\n", (void*)vp, NN(vp->nval), vp->sval, (void*)vp->sval, vp->tval); return(vp->sval); } char *getsval(Cell *vp) /* get string val of a Cell */ { return get_str_val(vp, CONVFMT); } char *getpssval(Cell *vp) /* get string val of a Cell for print */ { return get_str_val(vp, OFMT); } char *tostring(const char *s) /* make a copy of string s */ { char *p = strdup(s); if (p == NULL) FATAL("out of space in tostring on %s", s); return(p); } char *tostringN(const char *s, size_t n) /* make a copy of string s */ { char *p; p = (char *) malloc(n); if (p == NULL) FATAL("out of space in tostring on %s", s); strcpy(p, s); return(p); } Cell *catstr(Cell *a, Cell *b) /* concatenate a and b */ { Cell *c; char *p; char *sa = getsval(a); char *sb = getsval(b); size_t l = strlen(sa) + strlen(sb) + 1; p = (char *) malloc(l); if (p == NULL) FATAL("out of space concatenating %s and %s", sa, sb); snprintf(p, l, "%s%s", sa, sb); l++; // add room for ' ' char *newbuf = (char *) malloc(l); if (newbuf == NULL) FATAL("out of space concatenating %s and %s", sa, sb); // See string() in lex.c; a string "xx" is stored in the symbol // table as "xx ". snprintf(newbuf, l, "%s ", p); c = setsymtab(newbuf, p, 0.0, CON|STR|DONTFREE, symtab); free(p); free(newbuf); return c; } char *qstring(const char *is, int delim) /* collect string up to next delim */ { - const char *os = is; int c, n; const uschar *s = (const uschar *) is; uschar *buf, *bp; if ((buf = (uschar *) malloc(strlen(is)+3)) == NULL) FATAL( "out of space in qstring(%s)", s); for (bp = buf; (c = *s) != delim; s++) { if (c == '\n') - SYNTAX( "newline in string %.20s...", os ); + SYNTAX( "newline in string %.20s...", is ); else if (c != '\\') *bp++ = c; else { /* \something */ c = *++s; if (c == 0) { /* \ at end */ *bp++ = '\\'; break; /* for loop */ } switch (c) { case '\\': *bp++ = '\\'; break; case 'n': *bp++ = '\n'; break; case 't': *bp++ = '\t'; break; case 'b': *bp++ = '\b'; break; case 'f': *bp++ = '\f'; break; case 'r': *bp++ = '\r'; break; case 'v': *bp++ = '\v'; break; case 'a': *bp++ = '\a'; break; default: if (!isdigit(c)) { *bp++ = c; break; } n = c - '0'; if (isdigit(s[1])) { n = 8 * n + *++s - '0'; if (isdigit(s[1])) n = 8 * n + *++s - '0'; } *bp++ = n; break; } } } *bp++ = 0; return (char *) buf; } const char *flags2str(int flags) { static const struct ftab { const char *name; int value; } flagtab[] = { { "NUM", NUM }, { "STR", STR }, { "DONTFREE", DONTFREE }, { "CON", CON }, { "ARR", ARR }, { "FCN", FCN }, { "FLD", FLD }, { "REC", REC }, { "CONVC", CONVC }, { "CONVO", CONVO }, { NULL, 0 } }; static char buf[100]; int i; char *cp = buf; for (i = 0; flagtab[i].name != NULL; i++) { if ((flags & flagtab[i].value) != 0) { if (cp > buf) *cp++ = '|'; strcpy(cp, flagtab[i].name); cp += strlen(cp); } } return buf; }